From b3a108fb8b0ce870db4f80bb6096d5477105f4bf Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Mon, 30 Mar 2026 13:41:10 -0700 Subject: [PATCH 01/44] fix: restore queue health recovery progression (#2054) Co-authored-by: svelderrainruiz --- .../__tests__/queue-supervisor.test.mjs | 49 +++++++++++++++++++ tools/priority/queue-supervisor.mjs | 43 +++++++++++++--- 2 files changed, 85 insertions(+), 7 deletions(-) diff --git a/tools/priority/__tests__/queue-supervisor.test.mjs b/tools/priority/__tests__/queue-supervisor.test.mjs index 1c9c6a00b..7c1124b49 100644 --- a/tools/priority/__tests__/queue-supervisor.test.mjs +++ b/tools/priority/__tests__/queue-supervisor.test.mjs @@ -405,6 +405,25 @@ test('evaluateAdaptiveInflight applies hysteresis before upgrading from stabiliz assert.equal(secondRecovery.hysteresis.transition, 'upgrade-applied'); }); +test('evaluateAdaptiveInflight restores upgrade streak from persisted hysteresis state', () => { + const recovered = evaluateAdaptiveInflight({ + maxInflight: 5, + minInflight: 2, + adaptiveCap: true, + health: { successRate: 0.95, minSuccessRate: 0.8 }, + runtimeFleet: { totals: { queued: 0, inProgress: 0, stalled: 0 }, thresholds: { maxQueuedRuns: 6, maxInProgressRuns: 8 } }, + retryPressure: { retryRatio: 0, quarantineRatio: 0 }, + previousControllerState: { + mode: 'stabilize', + hysteresis: { + upgradeStreak: 1 + } + } + }); + assert.equal(recovered.tier, 'guarded'); + assert.equal(recovered.hysteresis.transition, 'upgrade-applied'); +}); + test('evaluateBurstWindow activates on release triggers and carries refill cycles', () => { const initial = evaluateBurstWindow({ burstMode: 'auto', @@ -616,6 +635,36 @@ test('evaluateHealthGate pauses when success rate drops or red window exceeds th assert.ok(redWindow.reasons.includes('trunk-red-window-exceeded')); }); +test('evaluateHealthGate ignores stale workflow failures outside the health lookback window', () => { + const now = new Date('2026-03-30T22:00:00.000Z'); + const result = evaluateHealthGate({ + workflowRunsByName: { + Validate: [ + { conclusion: 'success', status: 'completed', created_at: '2026-03-30T18:57:23Z', updated_at: '2026-03-30T19:13:12Z' }, + { conclusion: 'success', status: 'completed', created_at: '2026-03-29T22:01:19Z', updated_at: '2026-03-29T22:17:18Z' } + ], + 'Policy Guard (Upstream)': [ + { conclusion: 'success', status: 'completed', created_at: '2026-03-30T18:57:23Z', updated_at: '2026-03-30T18:58:03Z' } + ], + 'Fixture Drift Validation': [ + { conclusion: 'failure', status: 'completed', created_at: '2026-03-29T22:13:26Z', updated_at: '2026-03-29T22:15:13Z' }, + { conclusion: 'failure', status: 'completed', created_at: '2025-10-07T08:32:27Z', updated_at: '2025-10-07T08:35:06Z' }, + { conclusion: 'failure', status: 'completed', created_at: '2025-10-07T08:12:14Z', updated_at: '2025-10-07T08:14:27Z' }, + { conclusion: 'failure', status: 'completed', created_at: '2025-10-07T07:58:25Z', updated_at: '2025-10-07T08:00:46Z' } + ], + 'commit-integrity': [ + { conclusion: 'success', status: 'completed', created_at: '2026-03-29T22:24:27Z', updated_at: '2026-03-29T22:24:44Z' } + ] + }, + now + }); + + assert.equal(result.paused, false); + assert.equal(result.sampleSize, 5); + assert.equal(result.successful, 4); + assert.equal(result.successRate, 0.8); +}); + test('evaluateRuntimeFleetHealth pauses on saturation and stalled runs', () => { const now = new Date('2026-03-05T22:00:00.000Z'); const runtime = evaluateRuntimeFleetHealth({ diff --git a/tools/priority/queue-supervisor.mjs b/tools/priority/queue-supervisor.mjs index 492cd47d1..f4c530f98 100644 --- a/tools/priority/queue-supervisor.mjs +++ b/tools/priority/queue-supervisor.mjs @@ -36,6 +36,7 @@ const DEFAULT_MIN_INFLIGHT = 2; const DEFAULT_HEALTH_SAMPLE = 10; const DEFAULT_HEALTH_MIN_SUCCESS_RATE = 0.8; const DEFAULT_HEALTH_MAX_RED_MINUTES = 30; +const DEFAULT_HEALTH_LOOKBACK_DAYS = 30; const DEFAULT_MAX_QUEUED_RUNS = 6; const DEFAULT_MAX_IN_PROGRESS_RUNS = 8; const DEFAULT_STALL_THRESHOLD_MINUTES = 45; @@ -820,7 +821,8 @@ export function evaluateHealthGate({ workflowRunsByName, now = new Date(), minSuccessRate = DEFAULT_HEALTH_MIN_SUCCESS_RATE, - maxRedMinutes = DEFAULT_HEALTH_MAX_RED_MINUTES + maxRedMinutes = DEFAULT_HEALTH_MAX_RED_MINUTES, + lookbackDays = DEFAULT_HEALTH_LOOKBACK_DAYS }) { const runs = []; for (const [workflow, workflowRuns] of Object.entries(workflowRunsByName ?? {})) { @@ -836,12 +838,24 @@ export function evaluateHealthGate({ } } - runs.sort((a, b) => Date.parse(b.createdAt || 0) - Date.parse(a.createdAt || 0)); - const sampleSize = runs.length; - const successful = runs.filter((run) => run.conclusion === 'success').length; + runs.sort((a, b) => runTimestampMs(b) - runTimestampMs(a)); + const lookbackMs = + Number.isFinite(lookbackDays) && lookbackDays > 0 + ? lookbackDays * 24 * 60 * 60 * 1000 + : null; + const filteredRuns = + lookbackMs == null + ? runs + : runs.filter((run) => { + const stamp = runTimestampMs(run); + return Number.isFinite(stamp) && now.valueOf() - stamp <= lookbackMs; + }); + + const sampleSize = filteredRuns.length; + const successful = filteredRuns.filter((run) => run.conclusion === 'success').length; const successRate = sampleSize === 0 ? 0 : successful / sampleSize; - const latest = runs[0] ?? null; - const lastSuccess = runs.find((run) => run.conclusion === 'success') ?? null; + const latest = filteredRuns[0] ?? null; + const lastSuccess = filteredRuns.find((run) => run.conclusion === 'success') ?? null; let redMinutes = 0; if (latest && latest.conclusion !== 'success') { @@ -1208,6 +1222,20 @@ function applyControllerHysteresis({ }; } +function readPreviousControllerUpgradeStreak(previousControllerState) { + const direct = Number(previousControllerState?.upgradeStreak); + if (Number.isFinite(direct) && direct >= 0) { + return Math.trunc(direct); + } + + const nested = Number(previousControllerState?.hysteresis?.upgradeStreak); + if (Number.isFinite(nested) && nested >= 0) { + return Math.trunc(nested); + } + + return 0; +} + export function evaluateAdaptiveInflight({ maxInflight, minInflight = DEFAULT_MIN_INFLIGHT, @@ -1277,7 +1305,7 @@ export function evaluateAdaptiveInflight({ const hysteresis = applyControllerHysteresis({ desiredMode: desired.desiredMode, previousMode: previousControllerState?.mode, - previousUpgradeStreak: previousControllerState?.upgradeStreak, + previousUpgradeStreak: readPreviousControllerUpgradeStreak(previousControllerState), requiredUpgradeStreak: CONTROLLER_REQUIRED_UPGRADE_STREAK }); const effectiveMaxInflight = caps[hysteresis.mode] ?? configuredMin; @@ -1786,6 +1814,7 @@ export async function runQueueSupervisor(options = {}) { reasons: adaptiveInflight.reasons, metrics: adaptiveInflight.metrics, thresholds: adaptiveInflight.thresholds, + upgradeStreak: adaptiveInflight.hysteresis?.upgradeStreak ?? 0, hysteresis: adaptiveInflight.hysteresis, retryPressure, paused: uniquePausedReasons.length > 0, From b17db63ff5a24ff634356214d5f981f78a4ff24a Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Mon, 30 Mar 2026 13:58:27 -0700 Subject: [PATCH 02/44] policy: shrink develop required merge checks (#2055) Co-authored-by: svelderrainruiz --- docs/DEVELOPER_GUIDE.md | 6 +- docs/RELEASE_PROMOTION_CONTRACT.md | 12 + ...RUNBOOK_CONTAINER_LANE_PROMOTION_POLICY.md | 4 +- docs/knowledgebase/FEATURE_BRANCH_POLICY.md | 11 +- tools/policy/branch-required-checks.json | 10 - tools/policy/promotion-contract.json | 5 - .../__tests__/check-policy-apply.test.mjs | 268 ++++++++++++++---- tools/priority/policy.json | 10 - 8 files changed, 239 insertions(+), 87 deletions(-) diff --git a/docs/DEVELOPER_GUIDE.md b/docs/DEVELOPER_GUIDE.md index 23c200beb..df1423f90 100644 --- a/docs/DEVELOPER_GUIDE.md +++ b/docs/DEVELOPER_GUIDE.md @@ -463,7 +463,8 @@ For each cut: - Branch protection requires a linear history: use the **Squash and merge** button (or rebase-and-merge) so no merge commits land on `develop`/`main`. - Keep PRs focused and include the standing issue reference (`#`) in the commit subject and PR description. -- Ensure required checks (`validate`, `fixtures`, `session-index`) are green before merging; rerun as needed. +- Ensure required checks (`lint`, `fixtures`, `Policy Guard (Upstream) / policy-guard`, + `vi-history-scenarios-linux`, `commit-integrity`) are green before merging; rerun as needed. - `Validate` now computes a `validate-scope-plan` artifact before the heavy lanes fan out. Standard `pull_request` and `merge_group` runs classify changed paths with an allow-list into: `docs-metadata-only`, `tests-only`, `tools-policy-only`, `ci-control-plane`, `mixed-lightweight`, `compare-engine-history`, @@ -495,6 +496,9 @@ For each cut: the local CLI/orchestrator path, and `ready_for_review` is reserved for final hosted validation on the current head. - `agent-review-policy` is the hosted concentrator for local review evidence and promotion validation. It should not be used to acquire a second GitHub-side Copilot review after `ready_for_review`. +- `session-index`, `issue-snapshot`, `semver`, `agent-review-policy`, and `hook-parity` + remain valuable supporting lanes, but they are not part of the minimal merge-queue + required-check surface for `develop`. - Run `node tools/npm/run-script.mjs priority:policy` (or `node tools/npm/run-script.mjs priority:policy:sync`) if you need to audit merge settings locally; the command also runs during `priority:handoff-tests` and fails when repo/branch policy drifts. diff --git a/docs/RELEASE_PROMOTION_CONTRACT.md b/docs/RELEASE_PROMOTION_CONTRACT.md index 5cb590a64..c5b217cd6 100644 --- a/docs/RELEASE_PROMOTION_CONTRACT.md +++ b/docs/RELEASE_PROMOTION_CONTRACT.md @@ -61,6 +61,18 @@ Canonical required-check lists for `develop` and `release/*` must remain in sync `release/*`; `priority:policy` resolves it to the current live GitHub ruleset) +For `develop`, the required set is intentionally limited to the smallest merge-safety +surface that still preserves trunk truth: + +- `lint` +- `fixtures` +- `Policy Guard (Upstream) / policy-guard` +- `vi-history-scenarios-linux` +- `commit-integrity` + +Other Validate or platform lanes can still run and remain useful evidence, but they +must not be treated as queue-required unless they prove merge safety for `develop`. + The workflow context `Promotion Contract / promotion-contract` remains an operational evidence check, but it is not a branch-protection required status on `develop` or `release/*` because the workflow is intentionally path-scoped for pull requests. diff --git a/docs/RUNBOOK_CONTAINER_LANE_PROMOTION_POLICY.md b/docs/RUNBOOK_CONTAINER_LANE_PROMOTION_POLICY.md index 0269fa795..0310e60c6 100644 --- a/docs/RUNBOOK_CONTAINER_LANE_PROMOTION_POLICY.md +++ b/docs/RUNBOOK_CONTAINER_LANE_PROMOTION_POLICY.md @@ -73,8 +73,8 @@ Rollback behavior was validated against the canary state for #662/#663: - Canary lane remains non-required during this policy phase. - Current develop required-check contract remains satisfiable with no runbook-container context required. - Expected branch-protection behavior after rollback: merges continue using canonical required contexts (`lint`, - `fixtures`, `session-index`, `issue-snapshot`, `semver`, `Policy Guard (Upstream) / policy-guard`, - `vi-history-scenarios-linux`, `agent-review-policy`, `hook-parity`, `commit-integrity`). + `fixtures`, `Policy Guard (Upstream) / policy-guard`, `vi-history-scenarios-linux`, + `commit-integrity`). ## Decision for #663 diff --git a/docs/knowledgebase/FEATURE_BRANCH_POLICY.md b/docs/knowledgebase/FEATURE_BRANCH_POLICY.md index e70aeaf86..d37edde8c 100644 --- a/docs/knowledgebase/FEATURE_BRANCH_POLICY.md +++ b/docs/knowledgebase/FEATURE_BRANCH_POLICY.md @@ -1,7 +1,7 @@ # Feature Branch Enforcement & Merge Queue -| `develop` (live id may drift) | `refs/heads/develop` | Merge queue enabled (`merge_method=SQUASH`, `grouping=ALLGREEN`, build queue <=20 entries, 1-minute quiet window). Required checks: `lint`, `fixtures`, `session-index`, `issue-snapshot`, `semver`, `Policy Guard (Upstream) / policy-guard`, `vi-history-scenarios-linux`, `agent-review-policy`, `hook-parity`, `commit-integrity`. Non-required hosted proof lanes may run alongside the queue contract, including `vi-history-scenarios-windows` on GitHub-hosted `windows-2022`. Copilot review settings are no longer enforced through policy; draft/ready review semantics are repo-owned and validated by `agent-review-policy`. | +| `develop` (live id may drift) | `refs/heads/develop` | Merge queue enabled (`merge_method=SQUASH`, `grouping=ALLGREEN`, build queue <=20 entries, 1-minute quiet window). Required checks: `lint`, `fixtures`, `Policy Guard (Upstream) / policy-guard`, `vi-history-scenarios-linux`, `commit-integrity`. Non-required hosted proof lanes may run alongside the queue contract, including `session-index`, `issue-snapshot`, `semver`, `agent-review-policy`, `hook-parity`, and `vi-history-scenarios-windows` on GitHub-hosted `windows-2022`. Copilot review settings are no longer enforced through policy; draft/ready review semantics are repo-owned and validated by `agent-review-policy`. | ## Purpose @@ -84,7 +84,7 @@ promotion behavior, not the branch-class source of truth. ### GitHub rulesets | Manifest identity | Scope | Highlights | |-------------------|----------------------|----------------------------------------------------------------------------------------------| -| `develop` (live id may drift) | `refs/heads/develop` | Merge queue enabled (`merge_method=SQUASH`, `grouping=ALLGREEN`, build queue <=20 entries, 1-minute quiet window). Required checks: `lint`, `fixtures`, `session-index`, `issue-snapshot`, `semver`, `Policy Guard (Upstream) / policy-guard`, `vi-history-scenarios-linux`, `agent-review-policy`, `hook-parity`, `commit-integrity`. Non-required hosted proof lanes may run alongside the queue contract, including `vi-history-scenarios-windows` on GitHub-hosted `windows-2022`. Copilot review settings are no longer enforced through policy; draft/ready review semantics are repo-owned and validated by `agent-review-policy`. | +| `develop` (live id may drift) | `refs/heads/develop` | Merge queue enabled (`merge_method=SQUASH`, `grouping=ALLGREEN`, build queue <=20 entries, 1-minute quiet window). Required checks: `lint`, `fixtures`, `Policy Guard (Upstream) / policy-guard`, `vi-history-scenarios-linux`, `commit-integrity`. Non-required hosted proof lanes may run alongside the queue contract, including `session-index`, `issue-snapshot`, `semver`, `agent-review-policy`, `hook-parity`, and `vi-history-scenarios-windows` on GitHub-hosted `windows-2022`. Copilot review settings are no longer enforced through policy; draft/ready review semantics are repo-owned and validated by `agent-review-policy`. | | `main` (`tools/priority/policy.json` key `8614140`) | `refs/heads/main` | Merge queue enabled (`merge_method=SQUASH`, `grouping=ALLGREEN`, build queue <=5 entries, 1-minute quiet window). Required checks: `lint`, `pester`, `vi-binary-check`, `vi-compare`, `Policy Guard (Upstream) / policy-guard`, `commit-integrity`. Required approving reviews: `0`. | | `release` (`tools/priority/policy.json` key `8614172`) | `refs/heads/release/*` | No merge queue; protects against force-push/deletion. Required checks: `lint`, `pester / normalize`, `smoke-gate`, `Policy Guard (Upstream) / policy-guard`, `commit-integrity`. Required approving reviews: `0`. | @@ -126,9 +126,10 @@ checked into `tools/priority/policy.json` so `priority:policy` stays authoritati ### `develop` - **Merge strategy**: queue-managed squash with linear history enforced; merge commits disabled. -- **Required checks**: `lint`, `fixtures`, `session-index`, `issue-snapshot`, `semver`, - `Policy Guard (Upstream) / policy-guard`, - `vi-history-scenarios-linux`, `agent-review-policy`, `hook-parity`, `commit-integrity`. +- **Required checks**: `lint`, `fixtures`, `Policy Guard (Upstream) / policy-guard`, + `vi-history-scenarios-linux`, `commit-integrity`. +- **Supporting non-required lanes**: `session-index`, `issue-snapshot`, `semver`, + `agent-review-policy`, `hook-parity`, plus hosted Windows proving. - **Non-required hosted proof**: `vi-history-scenarios-windows` may run on GitHub-hosted `windows-2022` to validate `nationalinstruments/labview:2026q1-windows`. Agents may dispatch that hosted lane while manually running the Linux or Windows Docker Desktop/WSL2 lanes on this host. diff --git a/tools/policy/branch-required-checks.json b/tools/policy/branch-required-checks.json index 0e792f0d4..c37a2cdc9 100644 --- a/tools/policy/branch-required-checks.json +++ b/tools/policy/branch-required-checks.json @@ -11,13 +11,8 @@ "upstream-integration": [ "lint", "fixtures", - "session-index", - "issue-snapshot", - "semver", "Policy Guard (Upstream) / policy-guard", "vi-history-scenarios-linux", - "agent-review-policy", - "hook-parity", "commit-integrity" ], "downstream-consumer-proving-rail": [ @@ -43,13 +38,8 @@ "develop": [ "lint", "fixtures", - "session-index", - "issue-snapshot", - "semver", "Policy Guard (Upstream) / policy-guard", "vi-history-scenarios-linux", - "agent-review-policy", - "hook-parity", "commit-integrity" ], "downstream/develop": [ diff --git a/tools/policy/promotion-contract.json b/tools/policy/promotion-contract.json index 9afb6e275..7ee2e25c8 100644 --- a/tools/policy/promotion-contract.json +++ b/tools/policy/promotion-contract.json @@ -48,13 +48,8 @@ "develop": [ "lint", "fixtures", - "session-index", - "issue-snapshot", - "semver", "Policy Guard (Upstream) / policy-guard", "vi-history-scenarios-linux", - "agent-review-policy", - "hook-parity", "commit-integrity" ], "release/*": [ diff --git a/tools/priority/__tests__/check-policy-apply.test.mjs b/tools/priority/__tests__/check-policy-apply.test.mjs index bf47cad37..4dade15c0 100644 --- a/tools/priority/__tests__/check-policy-apply.test.mjs +++ b/tools/priority/__tests__/check-policy-apply.test.mjs @@ -25,16 +25,15 @@ function createResponse(data, status = 200, statusText = 'OK') { const EXPECTED_DEVELOP_CHECKS = [ 'lint', 'fixtures', - 'session-index', - 'issue-snapshot', - 'semver', 'Policy Guard (Upstream) / policy-guard', 'vi-history-scenarios-linux', - 'agent-review-policy', - 'hook-parity', 'commit-integrity' ]; +const EXPECTED_DOWNSTREAM_DEVELOP_CHECKS = [ + 'Downstream Promotion / downstream-promotion' +]; + const EXPECTED_MAIN_CHECKS = [ 'lint', 'pester', @@ -186,7 +185,7 @@ function createRuleset({ }; } -function createAlignedRulesets(ids = { develop: 8811898, main: 8614140, release: 8614172 }) { +function createAlignedRulesets(ids = { develop: 8811898, main: 8614140, release: 8614172, downstream: 8811901 }) { return { develop: createRuleset({ id: ids.develop, @@ -215,6 +214,16 @@ function createAlignedRulesets(ids = { develop: 8811898, main: 8614140, release: dismissStaleReviewsOnPush: true, allowedMergeMethods: ['rebase'] }) + }), + downstreamDevelop: createRuleset({ + id: ids.downstream, + name: 'downstream-develop', + includes: ['refs/heads/downstream/develop'], + requiredStatusChecks: EXPECTED_DOWNSTREAM_DEVELOP_CHECKS, + requiredLinearHistory: true, + pullRequestRule: createPullRequestRule({ + allowedMergeMethods: ['rebase'] + }) }) }; } @@ -263,6 +272,7 @@ test('priority:policy --apply updates rulesets for develop/main/release', async const rulesetDevelopUrl = `${repoUrl}/rulesets/8811898`; const rulesetMainUrl = `${repoUrl}/rulesets/8614140`; const rulesetReleaseUrl = `${repoUrl}/rulesets/8614172`; + const rulesetDownstreamDevelopUrl = `${repoUrl}/rulesets/8811901`; const repoState = { allow_squash_merge: true, @@ -409,7 +419,19 @@ test('priority:policy --apply updates rulesets for develop/main/release', async ] }; + const rulesetDownstreamDevelop = createRuleset({ + id: 8811901, + name: 'downstream-develop', + includes: ['refs/heads/downstream/develop'], + requiredStatusChecks: EXPECTED_DOWNSTREAM_DEVELOP_CHECKS, + requiredLinearHistory: true, + pullRequestRule: createPullRequestRule({ + allowedMergeMethods: ['rebase'] + }) + }); + const branchDevelopUrl = `${repoUrl}/branches/develop/protection`; + const branchDownstreamDevelopUrl = `${repoUrl}/branches/downstream%2Fdevelop/protection`; const branchMainUrl = `${repoUrl}/branches/main/protection`; let branchDevelopProtection = { @@ -458,6 +480,8 @@ test('priority:policy --apply updates rulesets for develop/main/release', async allow_fork_syncing: { enabled: false } }; + const branchDownstreamDevelopProtection = createAlignedBranchProtection(EXPECTED_DOWNSTREAM_DEVELOP_CHECKS); + const wrapEnabled = (value) => ({ enabled: Boolean(value) }); const requests = []; const fetchMock = async (url, options = {}) => { @@ -524,11 +548,16 @@ test('priority:policy --apply updates rulesets for develop/main/release', async } } + if (method === 'GET' && url === branchDownstreamDevelopUrl) { + return createResponse(branchDownstreamDevelopProtection); + } + if (method === 'GET' && url === listUrl) { return createResponse([ toRulesetSummary(rulesetDevelop), toRulesetSummary(rulesetMain), - toRulesetSummary(rulesetRelease) + toRulesetSummary(rulesetRelease), + toRulesetSummary(rulesetDownstreamDevelop) ]); } @@ -567,6 +596,10 @@ test('priority:policy --apply updates rulesets for develop/main/release', async } } + if (method === 'GET' && url === rulesetDownstreamDevelopUrl) { + return createResponse(rulesetDownstreamDevelop); + } + throw new Error(`Unexpected request ${method} ${url}`); }; @@ -707,6 +740,7 @@ test('priority:policy verifies fork-local rulesets by stable identity when manif const repoUrl = 'https://api.github.com/repos/test-org/test-repo'; const listUrl = `${repoUrl}/rulesets`; const branchDevelopUrl = `${repoUrl}/branches/develop/protection`; + const branchDownstreamDevelopUrl = `${repoUrl}/branches/downstream%2Fdevelop/protection`; const branchMainUrl = `${repoUrl}/branches/main/protection`; const expectedRulesetUrls = { main: `${repoUrl}/rulesets/8614140`, @@ -715,7 +749,8 @@ test('priority:policy verifies fork-local rulesets by stable identity when manif const rulesets = createAlignedRulesets({ develop: 99001, main: 99002, - release: 99003 + release: 99003, + downstream: 99004 }); const fetchMock = async (url, options = {}) => { @@ -729,6 +764,9 @@ test('priority:policy verifies fork-local rulesets by stable identity when manif if (url === branchDevelopUrl) { return createResponse(createAlignedBranchProtection(EXPECTED_DEVELOP_CHECKS)); } + if (url === branchDownstreamDevelopUrl) { + return createResponse(createAlignedBranchProtection(EXPECTED_DOWNSTREAM_DEVELOP_CHECKS)); + } if (url === branchMainUrl) { return createResponse(createAlignedBranchProtection(EXPECTED_MAIN_CHECKS)); } @@ -739,7 +777,8 @@ test('priority:policy verifies fork-local rulesets by stable identity when manif return createResponse([ toRulesetSummary(rulesets.develop), toRulesetSummary(rulesets.main), - toRulesetSummary(rulesets.release) + toRulesetSummary(rulesets.release), + toRulesetSummary(rulesets.downstreamDevelop) ]); } if (url === `${repoUrl}/rulesets/${rulesets.develop.id}`) { @@ -751,6 +790,9 @@ test('priority:policy verifies fork-local rulesets by stable identity when manif if (url === `${repoUrl}/rulesets/${rulesets.release.id}`) { return createResponse(rulesets.release); } + if (url === `${repoUrl}/rulesets/${rulesets.downstreamDevelop.id}`) { + return createResponse(rulesets.downstreamDevelop); + } throw new Error(`Unexpected request ${method} ${url}`); }; @@ -787,6 +829,7 @@ test('priority:policy --apply updates fork-local rulesets resolved by stable ide const repoUrl = 'https://api.github.com/repos/test-org/test-repo'; const listUrl = `${repoUrl}/rulesets`; const branchDevelopUrl = `${repoUrl}/branches/develop/protection`; + const branchDownstreamDevelopUrl = `${repoUrl}/branches/downstream%2Fdevelop/protection`; const branchMainUrl = `${repoUrl}/branches/main/protection`; const expectedRulesetUrls = { main: `${repoUrl}/rulesets/8614140`, @@ -795,7 +838,8 @@ test('priority:policy --apply updates fork-local rulesets resolved by stable ide const rulesets = createAlignedRulesets({ develop: 99101, main: 99102, - release: 99103 + release: 99103, + downstream: 99104 }); rulesets.develop.conditions.ref_name.include = ['refs/heads/develop-drifted']; rulesets.develop.rules = rulesets.develop.rules.filter( @@ -838,6 +882,9 @@ test('priority:policy --apply updates fork-local rulesets resolved by stable ide if (method === 'GET' && url === branchDevelopUrl) { return createResponse(createAlignedBranchProtection(EXPECTED_DEVELOP_CHECKS, FORK_MIRROR_BRANCH_PROTECTION)); } + if (method === 'GET' && url === branchDownstreamDevelopUrl) { + return createResponse(createAlignedBranchProtection(EXPECTED_DOWNSTREAM_DEVELOP_CHECKS)); + } if (method === 'GET' && url === branchMainUrl) { return createResponse(createAlignedBranchProtection(EXPECTED_MAIN_CHECKS)); } @@ -848,7 +895,8 @@ test('priority:policy --apply updates fork-local rulesets resolved by stable ide return createResponse([ toRulesetSummary(rulesets.develop), toRulesetSummary(rulesets.main), - toRulesetSummary(rulesets.release) + toRulesetSummary(rulesets.release), + toRulesetSummary(rulesets.downstreamDevelop) ]); } if (url === `${repoUrl}/rulesets/${rulesets.develop.id}`) { @@ -878,6 +926,15 @@ test('priority:policy --apply updates fork-local rulesets resolved by stable ide return createResponse(rulesets.release); } } + if (url === `${repoUrl}/rulesets/${rulesets.downstreamDevelop.id}`) { + if (method === 'GET') { + return createResponse(rulesets.downstreamDevelop); + } + if (method === 'PUT') { + rulesets.downstreamDevelop = updateRuleset(rulesets.downstreamDevelop, JSON.parse(options.body)); + return createResponse(rulesets.downstreamDevelop); + } + } throw new Error(`Unexpected request ${method} ${url}`); }; @@ -1021,6 +1078,7 @@ test('priority:policy --apply creates missing fork-local rulesets when no identi const repoUrl = 'https://api.github.com/repos/test-org/test-repo'; const listUrl = `${repoUrl}/rulesets`; const branchDevelopUrl = `${repoUrl}/branches/develop/protection`; + const branchDownstreamDevelopUrl = `${repoUrl}/branches/downstream%2Fdevelop/protection`; const branchMainUrl = `${repoUrl}/branches/main/protection`; const createdRulesets = []; let nextRulesetId = 99200; @@ -1038,6 +1096,9 @@ test('priority:policy --apply creates missing fork-local rulesets when no identi if (method === 'GET' && url === branchDevelopUrl) { return createResponse(createAlignedBranchProtection(EXPECTED_DEVELOP_CHECKS, FORK_MIRROR_BRANCH_PROTECTION)); } + if (method === 'GET' && url === branchDownstreamDevelopUrl) { + return createResponse(createAlignedBranchProtection(EXPECTED_DOWNSTREAM_DEVELOP_CHECKS)); + } if (method === 'GET' && url === branchMainUrl) { return createResponse(createAlignedBranchProtection(EXPECTED_MAIN_CHECKS)); } @@ -1089,8 +1150,8 @@ test('priority:policy --apply creates missing fork-local rulesets when no identi assert.equal(createdRulesets.length, 3, 'three fork-local rulesets should be created'); assert.deepEqual( createdRulesets.map((ruleset) => ruleset.name).sort(), - ['develop', 'main', 'release'], - 'created rulesets should cover develop/main/release identities' + ['develop', 'downstream-develop', 'main'], + 'created rulesets should cover develop/main/downstream identities' ); const createdDevelopRuleset = createdRulesets.find((ruleset) => ruleset.name === 'develop'); assert.ok( @@ -1103,6 +1164,7 @@ test('priority:policy verify passes on user-owned throughput forks while still e const repoUrl = 'https://api.github.com/repos/test-user/test-repo'; const listUrl = `${repoUrl}/rulesets`; const branchDevelopUrl = `${repoUrl}/branches/develop/protection`; + const branchDownstreamDevelopUrl = `${repoUrl}/branches/downstream%2Fdevelop/protection`; const branchMainUrl = `${repoUrl}/branches/main/protection`; const rulesetReleaseUrl = `${repoUrl}/rulesets/8614172`; const reportDir = await mkdtemp(path.join(os.tmpdir(), 'priority-policy-throughput-verify-')); @@ -1130,6 +1192,9 @@ test('priority:policy verify passes on user-owned throughput forks while still e if (method === 'GET' && url === branchDevelopUrl) { return createResponse(createAlignedBranchProtection(EXPECTED_DEVELOP_CHECKS, FORK_MIRROR_BRANCH_PROTECTION)); } + if (method === 'GET' && url === branchDownstreamDevelopUrl) { + return createResponse(createAlignedBranchProtection(EXPECTED_DOWNSTREAM_DEVELOP_CHECKS)); + } if (method === 'GET' && url === branchMainUrl) { return createResponse(createAlignedBranchProtection(EXPECTED_MAIN_CHECKS)); } @@ -1140,10 +1205,13 @@ test('priority:policy verify passes on user-owned throughput forks while still e return createResponse({ message: 'Not Found', status: '404' }, 404, 'Not Found'); } if (method === 'GET' && url === rulesetReleaseUrl) { - return createResponse(alignedRulesets.release); + return createResponse({ message: 'Not Found', status: '404' }, 404, 'Not Found'); } if (method === 'GET' && url === listUrl) { - return createResponse([]); + return createResponse([toRulesetSummary(alignedRulesets.downstreamDevelop)]); + } + if (method === 'GET' && url === `${repoUrl}/rulesets/${alignedRulesets.downstreamDevelop.id}`) { + return createResponse(alignedRulesets.downstreamDevelop); } throw new Error(`Unexpected request ${method} ${url}`); }; @@ -1185,6 +1253,7 @@ test('priority:policy verify honors repo portability overrides for forks that ca const repoUrl = 'https://api.github.com/repos/test-org/test-fork'; const listUrl = `${repoUrl}/rulesets`; const branchDevelopUrl = `${repoUrl}/branches/develop/protection`; + const branchDownstreamDevelopUrl = `${repoUrl}/branches/downstream%2Fdevelop/protection`; const branchMainUrl = `${repoUrl}/branches/main/protection`; const rulesetReleaseUrl = `${repoUrl}/rulesets/8614172`; const reportDir = await mkdtemp(path.join(os.tmpdir(), 'priority-policy-throughput-override-')); @@ -1212,6 +1281,9 @@ test('priority:policy verify honors repo portability overrides for forks that ca if (method === 'GET' && url === branchDevelopUrl) { return createResponse(createAlignedBranchProtection(EXPECTED_DEVELOP_CHECKS, FORK_MIRROR_BRANCH_PROTECTION)); } + if (method === 'GET' && url === branchDownstreamDevelopUrl) { + return createResponse(createAlignedBranchProtection(EXPECTED_DOWNSTREAM_DEVELOP_CHECKS)); + } if (method === 'GET' && url === branchMainUrl) { return createResponse(createAlignedBranchProtection(EXPECTED_MAIN_CHECKS)); } @@ -1222,10 +1294,13 @@ test('priority:policy verify honors repo portability overrides for forks that ca return createResponse({ message: 'Not Found', status: '404' }, 404, 'Not Found'); } if (method === 'GET' && url === rulesetReleaseUrl) { - return createResponse(alignedRulesets.release); + return createResponse({ message: 'Not Found', status: '404' }, 404, 'Not Found'); } if (method === 'GET' && url === listUrl) { - return createResponse([]); + return createResponse([toRulesetSummary(alignedRulesets.downstreamDevelop)]); + } + if (method === 'GET' && url === `${repoUrl}/rulesets/${alignedRulesets.downstreamDevelop.id}`) { + return createResponse(alignedRulesets.downstreamDevelop); } throw new Error(`Unexpected request ${method} ${url}`); }; @@ -1401,10 +1476,11 @@ test('priority:policy --apply skips queue-managed rulesets on user-owned through const repoUrl = 'https://api.github.com/repos/test-user/test-repo'; const listUrl = `${repoUrl}/rulesets`; const branchDevelopUrl = `${repoUrl}/branches/develop/protection`; + const branchDownstreamDevelopUrl = `${repoUrl}/branches/downstream%2Fdevelop/protection`; const branchMainUrl = `${repoUrl}/branches/main/protection`; const rulesetReleaseUrl = `${repoUrl}/rulesets/8614172`; const requests = []; - let createdReleaseRuleset = null; + let createdDownstreamRuleset = null; const manifestOverride = JSON.parse(await readFile(new URL('../policy.json', import.meta.url), 'utf8')); manifestOverride.repoProfiles = { ...(manifestOverride.repoProfiles ?? {}), @@ -1428,6 +1504,9 @@ test('priority:policy --apply skips queue-managed rulesets on user-owned through if (method === 'GET' && url === branchDevelopUrl) { return createResponse(createAlignedBranchProtection(EXPECTED_DEVELOP_CHECKS, FORK_MIRROR_BRANCH_PROTECTION)); } + if (method === 'GET' && url === branchDownstreamDevelopUrl) { + return createResponse(createAlignedBranchProtection(EXPECTED_DOWNSTREAM_DEVELOP_CHECKS)); + } if (method === 'GET' && url === branchMainUrl) { return createResponse(createAlignedBranchProtection(EXPECTED_MAIN_CHECKS)); } @@ -1438,24 +1517,27 @@ test('priority:policy --apply skips queue-managed rulesets on user-owned through return createResponse({ message: 'Not Found', status: '404' }, 404, 'Not Found'); } if (method === 'GET' && url === rulesetReleaseUrl) { - if (createdReleaseRuleset) { - return createResponse(createdReleaseRuleset); - } return createResponse({ message: 'Not Found', status: '404' }, 404, 'Not Found'); } if (method === 'GET' && url === listUrl) { - return createResponse([]); + return createResponse(createdDownstreamRuleset ? [toRulesetSummary(createdDownstreamRuleset)] : []); + } + if (method === 'GET' && url === `${repoUrl}/rulesets/99201`) { + if (createdDownstreamRuleset) { + return createResponse(createdDownstreamRuleset); + } + return createResponse({ message: 'Not Found', status: '404' }, 404, 'Not Found'); } if (method === 'POST' && url === listUrl) { const payload = JSON.parse(options.body); - createdReleaseRuleset = { + createdDownstreamRuleset = { ...payload, - id: 8614172, + id: 99201, enforcement: payload.enforcement ?? 'active', target: payload.target ?? 'branch', bypass_actors: payload.bypass_actors ?? [] }; - return createResponse(createdReleaseRuleset, 201, 'Created'); + return createResponse(createdDownstreamRuleset, 201, 'Created'); } throw new Error(`Unexpected request ${method} ${url}`); }; @@ -1478,8 +1560,9 @@ test('priority:policy --apply skips queue-managed rulesets on user-owned through assert.equal(code, 0, 'apply mode should succeed on user-owned throughput forks'); const createRequests = requests.filter((entry) => entry.method === 'POST' && entry.url === listUrl); - assert.equal(createRequests.length, 1, 'apply mode should only create the non-queue release ruleset'); + assert.equal(createRequests.length, 1, 'apply mode should only create the non-queue downstream-develop ruleset'); const createdPayload = JSON.parse(createRequests[0].body); + assert.equal(createdPayload.name, 'downstream-develop'); assert.equal( createdPayload.rules.some((rule) => rule?.type === 'merge_queue'), false, @@ -1491,12 +1574,13 @@ test('priority:policy --apply downgrades queue-managed rulesets when a fork reje const repoUrl = 'https://api.github.com/repos/test-org/test-fork'; const listUrl = `${repoUrl}/rulesets`; const branchDevelopUrl = `${repoUrl}/branches/develop/protection`; + const branchDownstreamDevelopUrl = `${repoUrl}/branches/downstream%2Fdevelop/protection`; const branchMainUrl = `${repoUrl}/branches/main/protection`; const rulesetReleaseUrl = `${repoUrl}/rulesets/8614172`; const reportDir = await mkdtemp(path.join(os.tmpdir(), 'priority-policy-throughput-apply-')); const reportPath = path.join(reportDir, 'policy-report.json'); const requests = []; - let createdReleaseRuleset = null; + const createdRulesets = []; const fetchMock = async (url, options = {}) => { const method = options.method ?? 'GET'; @@ -1512,6 +1596,9 @@ test('priority:policy --apply downgrades queue-managed rulesets when a fork reje if (method === 'GET' && url === branchDevelopUrl) { return createResponse(createAlignedBranchProtection(EXPECTED_DEVELOP_CHECKS, FORK_MIRROR_BRANCH_PROTECTION)); } + if (method === 'GET' && url === branchDownstreamDevelopUrl) { + return createResponse(createAlignedBranchProtection(EXPECTED_DOWNSTREAM_DEVELOP_CHECKS)); + } if (method === 'GET' && url === branchMainUrl) { return createResponse(createAlignedBranchProtection(EXPECTED_MAIN_CHECKS)); } @@ -1522,13 +1609,18 @@ test('priority:policy --apply downgrades queue-managed rulesets when a fork reje return createResponse({ message: 'Not Found', status: '404' }, 404, 'Not Found'); } if (method === 'GET' && url === rulesetReleaseUrl) { - if (createdReleaseRuleset) { - return createResponse(createdReleaseRuleset); - } return createResponse({ message: 'Not Found', status: '404' }, 404, 'Not Found'); } if (method === 'GET' && url === listUrl) { - return createResponse([]); + return createResponse(createdRulesets.map((ruleset) => toRulesetSummary(ruleset))); + } + if (method === 'GET' && url.startsWith(`${repoUrl}/rulesets/`)) { + const id = Number(url.split('/').at(-1)); + const match = createdRulesets.find((ruleset) => ruleset.id === id); + if (match) { + return createResponse(match); + } + return createResponse({ message: 'Not Found', status: '404' }, 404, 'Not Found'); } if (method === 'POST' && url === listUrl) { const payload = JSON.parse(options.body); @@ -1542,14 +1634,15 @@ test('priority:policy --apply downgrades queue-managed rulesets when a fork reje 'Unprocessable Entity' ); } - createdReleaseRuleset = { + const created = { ...payload, - id: 8614172, + id: 99210 + createdRulesets.length, enforcement: payload.enforcement ?? 'active', target: payload.target ?? 'branch', bypass_actors: payload.bypass_actors ?? [] }; - return createResponse(createdReleaseRuleset, 201, 'Created'); + createdRulesets.push(created); + return createResponse(created, 201, 'Created'); } throw new Error(`Unexpected request ${method} ${url}`); }; @@ -1595,6 +1688,7 @@ test('priority:policy --apply keeps fork develop on the mirror-rail override for const repoUrl = 'https://api.github.com/repos/LabVIEW-Community-CI-CD/compare-vi-cli-action-fork'; const listUrl = `${repoUrl}/rulesets`; const branchDevelopUrl = `${repoUrl}/branches/develop/protection`; + const branchDownstreamDevelopUrl = `${repoUrl}/branches/downstream%2Fdevelop/protection`; const branchMainUrl = `${repoUrl}/branches/main/protection`; const alignedRulesets = createAlignedRulesets(); let developProtection = createAlignedBranchProtection(EXPECTED_DEVELOP_CHECKS); @@ -1613,6 +1707,9 @@ test('priority:policy --apply keeps fork develop on the mirror-rail override for if (method === 'GET' && url === branchDevelopUrl) { return createResponse(developProtection); } + if (method === 'GET' && url === branchDownstreamDevelopUrl) { + return createResponse(createAlignedBranchProtection(EXPECTED_DOWNSTREAM_DEVELOP_CHECKS)); + } if (method === 'GET' && url === branchMainUrl) { return createResponse(createAlignedBranchProtection(EXPECTED_MAIN_CHECKS)); } @@ -1636,10 +1733,17 @@ test('priority:policy --apply keeps fork develop on the mirror-rail override for return createResponse(alignedRulesets.main); } if (method === 'GET' && url === `${repoUrl}/rulesets/8614172`) { - return createResponse(alignedRulesets.release); + return createResponse({ message: 'Not Found', status: '404' }, 404, 'Not Found'); + } + if (method === 'GET' && url === `${repoUrl}/rulesets/${alignedRulesets.downstreamDevelop.id}`) { + return createResponse(alignedRulesets.downstreamDevelop); } if (method === 'GET' && url === listUrl) { - return createResponse(Object.values(alignedRulesets).map(toRulesetSummary)); + return createResponse([ + toRulesetSummary(alignedRulesets.develop), + toRulesetSummary(alignedRulesets.main), + toRulesetSummary(alignedRulesets.downstreamDevelop) + ]); } throw new Error(`Unexpected request ${method} ${url}`); }; @@ -1675,11 +1779,13 @@ test('priority:policy --apply enforces required checks after merge_queue portabi const repoUrl = 'https://api.github.com/repos/test-org/test-fork'; const listUrl = `${repoUrl}/rulesets`; const branchDevelopUrl = `${repoUrl}/branches/develop/protection`; + const branchDownstreamDevelopUrl = `${repoUrl}/branches/downstream%2Fdevelop/protection`; const branchMainUrl = `${repoUrl}/branches/main/protection`; const rulesetReleaseUrl = `${repoUrl}/rulesets/8614172`; const requests = []; - let createdReleaseRuleset = null; + const createdRulesets = []; let developProtection = createAlignedBranchProtection([], FORK_MIRROR_BRANCH_PROTECTION); + let downstreamDevelopProtection = createAlignedBranchProtection(EXPECTED_DOWNSTREAM_DEVELOP_CHECKS); let mainProtection = createAlignedBranchProtection([]); const fetchMock = async (url, options = {}) => { @@ -1696,6 +1802,9 @@ test('priority:policy --apply enforces required checks after merge_queue portabi if (method === 'GET' && url === branchDevelopUrl) { return createResponse(developProtection); } + if (method === 'GET' && url === branchDownstreamDevelopUrl) { + return createResponse(downstreamDevelopProtection); + } if (method === 'GET' && url === branchMainUrl) { return createResponse(mainProtection); } @@ -1728,13 +1837,18 @@ test('priority:policy --apply enforces required checks after merge_queue portabi return createResponse({ message: 'Not Found', status: '404' }, 404, 'Not Found'); } if (method === 'GET' && url === rulesetReleaseUrl) { - if (createdReleaseRuleset) { - return createResponse(createdReleaseRuleset); - } return createResponse({ message: 'Not Found', status: '404' }, 404, 'Not Found'); } if (method === 'GET' && url === listUrl) { - return createResponse([]); + return createResponse(createdRulesets.map((ruleset) => toRulesetSummary(ruleset))); + } + if (method === 'GET' && url.startsWith(`${repoUrl}/rulesets/`)) { + const id = Number(url.split('/').at(-1)); + const match = createdRulesets.find((ruleset) => ruleset.id === id); + if (match) { + return createResponse(match); + } + return createResponse({ message: 'Not Found', status: '404' }, 404, 'Not Found'); } if (method === 'POST' && url === listUrl) { const payload = JSON.parse(options.body); @@ -1748,14 +1862,15 @@ test('priority:policy --apply enforces required checks after merge_queue portabi 'Unprocessable Entity' ); } - createdReleaseRuleset = { + const created = { ...payload, - id: 8614172, + id: 99310 + createdRulesets.length, enforcement: payload.enforcement ?? 'active', target: payload.target ?? 'branch', bypass_actors: payload.bypass_actors ?? [] }; - return createResponse(createdReleaseRuleset, 201, 'Created'); + createdRulesets.push(created); + return createResponse(created, 201, 'Created'); } throw new Error(`Unexpected request ${method} ${url}`); }; @@ -2410,6 +2525,7 @@ test('priority:policy verify uses queue-managed rulesets as required-check sourc const rulesetMainUrl = `${repoUrl}/rulesets/8614140`; const rulesetReleaseUrl = `${repoUrl}/rulesets/8614172`; const branchDevelopUrl = `${repoUrl}/branches/develop/protection`; + const branchDownstreamDevelopUrl = `${repoUrl}/branches/downstream%2Fdevelop/protection`; const branchMainUrl = `${repoUrl}/branches/main/protection`; const repoState = { @@ -2432,15 +2548,7 @@ test('priority:policy verify uses queue-managed rulesets as required-check sourc 'Policy Guard (Upstream) / policy-guard', 'commit-integrity' ]; - const releaseChecksExpected = [ - 'lint', - 'pester', - 'publish', - 'vi-binary-check', - 'vi-compare', - 'mock-cli', - 'Policy Guard (Upstream) / policy-guard' - ]; + const releaseChecksExpected = [...EXPECTED_RELEASE_CHECKS]; const branchDevelopProtection = { required_status_checks: { @@ -2480,6 +2588,8 @@ test('priority:policy verify uses queue-managed rulesets as required-check sourc allow_fork_syncing: { enabled: false } }; + const branchDownstreamDevelopProtection = createAlignedBranchProtection(EXPECTED_DOWNSTREAM_DEVELOP_CHECKS); + const developMergeQueueParams = { merge_method: 'SQUASH', grouping_strategy: 'ALLGREEN', @@ -2586,6 +2696,33 @@ test('priority:policy verify uses queue-managed rulesets as required-check sourc ] }; + const rulesetDownstreamDevelop = { + id: 99004, + name: 'downstream-develop', + target: 'branch', + conditions: { ref_name: { include: ['refs/heads/downstream/develop'], exclude: [] } }, + rules: [ + { type: 'required_linear_history' }, + { + type: 'required_status_checks', + parameters: { + required_status_checks: EXPECTED_DOWNSTREAM_DEVELOP_CHECKS.map((context) => ({ context })) + } + }, + { + type: 'pull_request', + parameters: { + required_approving_review_count: 0, + dismiss_stale_reviews_on_push: false, + require_code_owner_review: false, + require_last_push_approval: false, + required_review_thread_resolution: false, + allowed_merge_methods: ['rebase'] + } + } + ] + }; + const fetchMock = async (url, options = {}) => { const method = options.method ?? 'GET'; if (method !== 'GET') { @@ -2597,6 +2734,9 @@ test('priority:policy verify uses queue-managed rulesets as required-check sourc if (url === branchDevelopUrl) { return createResponse(branchDevelopProtection); } + if (url === branchDownstreamDevelopUrl) { + return createResponse(branchDownstreamDevelopProtection); + } if (url === branchMainUrl) { return createResponse(branchMainProtection); } @@ -2604,12 +2744,16 @@ test('priority:policy verify uses queue-managed rulesets as required-check sourc return createResponse([ toRulesetSummary(rulesetDevelop), toRulesetSummary(rulesetMain), - toRulesetSummary(rulesetRelease) + toRulesetSummary(rulesetRelease), + toRulesetSummary(rulesetDownstreamDevelop) ]); } if (url === `${repoUrl}/rulesets/${rulesetDevelop.id}`) { return createResponse(rulesetDevelop); } + if (url === `${repoUrl}/rulesets/${rulesetDownstreamDevelop.id}`) { + return createResponse(rulesetDownstreamDevelop); + } if (url === rulesetMainUrl) { return createResponse(rulesetMain); } @@ -2652,6 +2796,7 @@ test('priority:policy --apply preserves branch required checks when queue-manage const repoUrl = 'https://api.github.com/repos/test-org/test-repo'; const listUrl = `${repoUrl}/rulesets`; const branchDevelopUrl = `${repoUrl}/branches/develop/protection`; + const branchDownstreamDevelopUrl = `${repoUrl}/branches/downstream%2Fdevelop/protection`; const branchMainUrl = `${repoUrl}/branches/main/protection`; const rulesets = createAlignedRulesets(); let developProtection = { @@ -2672,6 +2817,9 @@ test('priority:policy --apply preserves branch required checks when queue-manage if (method === 'GET' && url === branchDevelopUrl) { return createResponse(developProtection); } + if (method === 'GET' && url === branchDownstreamDevelopUrl) { + return createResponse(createAlignedBranchProtection(EXPECTED_DOWNSTREAM_DEVELOP_CHECKS)); + } if (method === 'GET' && url === branchMainUrl) { return createResponse(mainProtection); } @@ -2696,6 +2844,9 @@ test('priority:policy --apply preserves branch required checks when queue-manage if (method === 'GET' && url === `${repoUrl}/rulesets/8614172`) { return createResponse(rulesets.release); } + if (method === 'GET' && url === `${repoUrl}/rulesets/${rulesets.downstreamDevelop.id}`) { + return createResponse(rulesets.downstreamDevelop); + } if (method === 'GET' && url === listUrl) { return createResponse(Object.values(rulesets).map(toRulesetSummary)); } @@ -3114,14 +3265,17 @@ test('priority:policy build branch-protection payload honors explicit disabled s test('priority:policy --apply projects required checks from the branch-class contract when the manifest omits copied lists', async () => { const manifestOverride = JSON.parse(await readFile(new URL('../policy.json', import.meta.url), 'utf8')); delete manifestOverride.branches.develop.required_status_checks; + delete manifestOverride.branches['downstream/develop'].required_status_checks; delete manifestOverride.branches.main.required_status_checks; delete manifestOverride.branches['release/*'].required_status_checks; delete manifestOverride.rulesets.develop.required_status_checks; + delete manifestOverride.rulesets['downstream-develop'].required_status_checks; delete manifestOverride.rulesets['8614140'].required_status_checks; delete manifestOverride.rulesets['8614172'].required_status_checks; const repoUrl = 'https://api.github.com/repos/test-org/test-repo'; const branchDevelopUrl = `${repoUrl}/branches/develop/protection`; + const branchDownstreamDevelopUrl = `${repoUrl}/branches/downstream%2Fdevelop/protection`; const branchMainUrl = `${repoUrl}/branches/main/protection`; const listUrl = `${repoUrl}/rulesets`; const rulesets = createAlignedRulesets(); @@ -3142,6 +3296,9 @@ test('priority:policy --apply projects required checks from the branch-class con if (method === 'GET' && url === branchDevelopUrl) { return createResponse(createAlignedBranchProtection([])); } + if (method === 'GET' && url === branchDownstreamDevelopUrl) { + return createResponse(createAlignedBranchProtection(EXPECTED_DOWNSTREAM_DEVELOP_CHECKS)); + } if (method === 'GET' && url === branchMainUrl) { return createResponse(createAlignedBranchProtection(EXPECTED_MAIN_CHECKS)); } @@ -3166,6 +3323,9 @@ test('priority:policy --apply projects required checks from the branch-class con if (method === 'GET' && url === `${repoUrl}/rulesets/8614172`) { return createResponse(rulesets.release); } + if (method === 'GET' && url === `${repoUrl}/rulesets/${rulesets.downstreamDevelop.id}`) { + return createResponse(rulesets.downstreamDevelop); + } if (method === 'GET' && url === listUrl) { return createResponse(Object.values(rulesets).map(toRulesetSummary)); } diff --git a/tools/priority/policy.json b/tools/priority/policy.json index f45341753..2cd4d0ec8 100644 --- a/tools/priority/policy.json +++ b/tools/priority/policy.json @@ -47,13 +47,8 @@ "required_status_checks": [ "lint", "fixtures", - "session-index", - "issue-snapshot", - "semver", "Policy Guard (Upstream) / policy-guard", "vi-history-scenarios-linux", - "agent-review-policy", - "hook-parity", "commit-integrity" ] }, @@ -150,13 +145,8 @@ "required_status_checks": [ "lint", "fixtures", - "session-index", - "issue-snapshot", - "semver", "Policy Guard (Upstream) / policy-guard", "vi-history-scenarios-linux", - "agent-review-policy", - "hook-parity", "commit-integrity" ] }, From e3b725902e5980abc3e2070a66b817a9f758f738 Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Mon, 30 Mar 2026 14:58:43 -0700 Subject: [PATCH 03/44] Use touch-history semantics in VI history proofs (#2056) Co-authored-by: svelderrainruiz --- tests/CompareVI.History.Tests.ps1 | 143 +++++++++++++- tests/Render-VIHistoryReport.Tests.ps1 | 6 +- tools/Compare-RefsToTemp.ps1 | 25 ++- tools/Compare-VIHistory.ps1 | 264 +++++++++++++++++++------ tools/Render-VIHistoryReport.ps1 | 3 + 5 files changed, 378 insertions(+), 63 deletions(-) diff --git a/tests/CompareVI.History.Tests.ps1 b/tests/CompareVI.History.Tests.ps1 index 0a638c927..a18f58342 100644 --- a/tests/CompareVI.History.Tests.ps1 +++ b/tests/CompareVI.History.Tests.ps1 @@ -627,6 +627,68 @@ exit 0 } } + It 'does not treat compare identity banners as diff categories or highlights' { + if (-not $_pairs) { Set-ItResult -Skipped -Because 'Missing commit data'; return } + $previousDiff = $env:STUB_COMPARE_DIFF + $previousFixture = $env:STUB_COMPARE_REPORT_FIXTURE + $fixtureRoot = Join-Path $TestDrive 'history-identity-banner-fixture' + New-Item -ItemType Directory -Path $fixtureRoot -Force | Out-Null +@' + + +
+
First VI: /compare/m0/Base.vi
Second VI: /compare/m0/Head.vi
 + Block Diagram - Diagram +
    +
  • Block Diagram objects
    • +
+
+ + +'@ | Set-Content -LiteralPath (Join-Path $fixtureRoot 'compare-report.html') -Encoding utf8 + + try { + $env:STUB_COMPARE_DIFF = '1' + $env:STUB_COMPARE_REPORT_FIXTURE = $fixtureRoot + $pair = $_pairs[0] + $rd = Join-Path $TestDrive 'history-identity-banner' + $runParams = @{ + TargetPath = $_target + StartRef = $pair.Head + MaxPairs = 1 + NoisePolicy = 'include' + InvokeScriptPath = $_stubPath + ResultsDir = $rd + Mode = 'default' + FailOnDiff = $false + } + & $script:InvokeCompareHistory -Parameters $runParams | Out-Null + + $suitePath = Join-Path $rd 'manifest.json' + Test-Path -LiteralPath $suitePath | Should -BeTrue + $aggregate = Get-Content -LiteralPath $suitePath -Raw | ConvertFrom-Json + $modeEntry = $aggregate.modes | Where-Object { $_.slug -eq 'default' } + $modeManifest = Get-Content -LiteralPath $modeEntry.manifestPath -Raw | ConvertFrom-Json + $comparison = @($modeManifest.comparisons)[0] + + ($comparison.result.categories -join "`n") | Should -Not -Match 'First VI:|Second VI:|compare/m0/Base\.vi|compare/m0/Head\.vi' + ($comparison.result.highlights -join "`n") | Should -Not -Match 'First VI:|Second VI:|compare/m0/Base\.vi|compare/m0/Head\.vi' + $comparison.result.categories | Should -Contain 'Block Diagram' + $aggregate.stats.categoryCounts.PSObject.Properties.Name | Should -Contain 'Block Diagram' + } finally { + if ($null -eq $previousDiff) { + Remove-Item Env:STUB_COMPARE_DIFF -ErrorAction SilentlyContinue + } else { + $env:STUB_COMPARE_DIFF = $previousDiff + } + if ($null -eq $previousFixture) { + Remove-Item Env:STUB_COMPARE_REPORT_FIXTURE -ErrorAction SilentlyContinue + } else { + $env:STUB_COMPARE_REPORT_FIXTURE = $previousFixture + } + } + } + It 'processes full history when MaxPairs is omitted' { if (-not $_pairs) { Set-ItResult -Skipped -Because 'Missing commit data'; return } @@ -657,7 +719,7 @@ exit 0 $modeManifest = Get-Content -LiteralPath $modeEntry.manifestPath -Raw | ConvertFrom-Json $modeManifest.maxPairs | Should -BeNullOrEmpty - $modeManifest.stats.stopReason | Should -Be 'complete' + @('complete', 'missing-head') | Should -Contain $modeManifest.stats.stopReason $modeManifest.stats.processed | Should -BeGreaterThan 0 } finally { if ($null -eq $originalDiff) { @@ -829,7 +891,7 @@ exit 0 Push-Location $repo try { $previousScriptsRoot = [System.Environment]::GetEnvironmentVariable('COMPAREVI_SCRIPTS_ROOT', 'Process') - [System.Environment]::SetEnvironmentVariable('COMPAREVI_SCRIPTS_ROOT', (Join-Path $_repoRoot 'tools'), 'Process') + [System.Environment]::SetEnvironmentVariable('COMPAREVI_SCRIPTS_ROOT', $_repoRoot, 'Process') & pwsh -NoLogo -NoProfile -File (Join-Path $_repoRoot 'tools/Compare-VIHistory.ps1') ` -TargetPath 'VI1.vi' ` -StartRef $mergeCommit ` @@ -858,6 +920,81 @@ exit 0 $manifest.comparisons[0].head.ref | Should -Be $mergeCommit } + It 'builds comparison pairs from VI touch history instead of first-parent lineage' { + $repo = Join-Path $TestDrive 'history-touch-sequence' + New-Item -ItemType Directory -Path $repo -Force | Out-Null + & git -C $repo init -b main | Out-Null + & git -C $repo config user.name 'CompareVI Test' | Out-Null + & git -C $repo config user.email 'comparevi@example.test' | Out-Null + + 'base' | Set-Content -LiteralPath (Join-Path $repo 'VI1.vi') -Encoding utf8 + & git -C $repo add VI1.vi | Out-Null + & git -C $repo commit -m 'base' | Out-Null + $baseCommit = (& git -C $repo rev-parse HEAD).Trim() + + & git -C $repo checkout -b feature/history-pairs | Out-Null + 'feature change 1' | Set-Content -LiteralPath (Join-Path $repo 'VI1.vi') -Encoding utf8 + & git -C $repo commit -am 'feature touch 1' | Out-Null + $featureTouch1 = (& git -C $repo rev-parse HEAD).Trim() + + 'feature change 2' | Set-Content -LiteralPath (Join-Path $repo 'VI1.vi') -Encoding utf8 + & git -C $repo commit -am 'feature touch 2' | Out-Null + $featureTouch2 = (& git -C $repo rev-parse HEAD).Trim() + + & git -C $repo checkout main | Out-Null + 'mainline context' | Set-Content -LiteralPath (Join-Path $repo 'README.md') -Encoding utf8 + & git -C $repo add README.md | Out-Null + & git -C $repo commit -m 'mainline context' | Out-Null + & git -C $repo merge --no-ff feature/history-pairs -m 'merge feature history' | Out-Null + + 'post merge context' | Add-Content -LiteralPath (Join-Path $repo 'README.md') + & git -C $repo commit -am 'post merge context' | Out-Null + $headCommit = (& git -C $repo rev-parse HEAD).Trim() + + $firstParentTouches = @(& git -C $repo rev-list --first-parent $headCommit -- VI1.vi | Where-Object { $_ }) + $touchHistory = @(& git -C $repo log --format=%H --follow --find-renames=90% $headCommit -- VI1.vi | Where-Object { $_ }) + $touchHistory.Count | Should -BeGreaterThan $firstParentTouches.Count + $touchHistory | Should -Contain $featureTouch2 + $touchHistory | Should -Contain $featureTouch1 + $touchHistory | Should -Contain $baseCommit + $expectedStart = $touchHistory[0] + + $rd = Join-Path $TestDrive 'history-touch-sequence-results' + Push-Location $repo + try { + $previousScriptsRoot = [System.Environment]::GetEnvironmentVariable('COMPAREVI_SCRIPTS_ROOT', 'Process') + [System.Environment]::SetEnvironmentVariable('COMPAREVI_SCRIPTS_ROOT', $_repoRoot, 'Process') + & pwsh -NoLogo -NoProfile -File (Join-Path $_repoRoot 'tools/Compare-VIHistory.ps1') ` + -TargetPath 'VI1.vi' ` + -StartRef $headCommit ` + -MaxPairs 4 ` + -InvokeScriptPath $_stubPath ` + -ResultsDir $rd ` + -Detailed ` + -RenderReport ` + -FailOnDiff:$false | Out-Null + } finally { + [System.Environment]::SetEnvironmentVariable('COMPAREVI_SCRIPTS_ROOT', $previousScriptsRoot, 'Process') + Pop-Location + } + + $suitePath = Join-Path $rd 'manifest.json' + Test-Path -LiteralPath $suitePath | Should -BeTrue + $aggregate = Get-Content -LiteralPath $suitePath -Raw | ConvertFrom-Json + $modeEntry = $aggregate.modes | Where-Object { $_.slug -eq 'default' } + $modeEntry | Should -Not -BeNullOrEmpty + Test-Path -LiteralPath $modeEntry.manifestPath | Should -BeTrue + $manifest = Get-Content -LiteralPath $modeEntry.manifestPath -Raw | ConvertFrom-Json + + $manifest.requestedStartRef | Should -Be $headCommit + $manifest.startRef | Should -Be $expectedStart + $manifest.stats.processed | Should -BeGreaterThan 0 + $manifest.comparisons[0].lineage.type | Should -Be 'touch-history' + $manifest.comparisons[0].head.ref | Should -Be $touchHistory[0] + $manifest.comparisons[0].base.ref | Should -Be $touchHistory[1] + @($manifest.comparisons | ForEach-Object { $_.head.ref }) | Should -Contain $featureTouch2 + } + It 'exposes attribute-focused mode when requested' { if (-not $_pairs) { Set-ItResult -Skipped -Because 'Missing commit data'; return } $env:STUB_COMPARE_DIFF = '0' @@ -1657,6 +1794,7 @@ Describe 'Compare-VIHistory source control handling' -Tag 'Integration' { } It 'detects when SCC is enabled in LabVIEW.ini' { + if (-not $IsWindows) { Set-ItResult -Skipped -Because 'LabVIEW.ini lookup is only supported on Windows'; return } $tempRoot = Join-Path $TestDrive 'lv-scc-enabled' New-Item -ItemType Directory -Path $tempRoot | Out-Null $fakeExe = Join-Path $tempRoot 'LabVIEW.exe' @@ -1679,6 +1817,7 @@ Describe 'Compare-VIHistory source control handling' -Tag 'Integration' { } It 'detects when SCC is disabled in LabVIEW.ini' { + if (-not $IsWindows) { Set-ItResult -Skipped -Because 'LabVIEW.ini lookup is only supported on Windows'; return } $tempRoot = Join-Path $TestDrive 'lv-scc-disabled' New-Item -ItemType Directory -Path $tempRoot | Out-Null $fakeExe = Join-Path $tempRoot 'LabVIEW.exe' diff --git a/tests/Render-VIHistoryReport.Tests.ps1 b/tests/Render-VIHistoryReport.Tests.ps1 index c90a3271a..d1273d82a 100644 --- a/tests/Render-VIHistoryReport.Tests.ps1 +++ b/tests/Render-VIHistoryReport.Tests.ps1 @@ -115,14 +115,14 @@ Describe 'Render-VIHistoryReport.ps1' -Tag 'Unit' { subject= 'Clean head commit' } lineage = [ordered]@{ - type = 'mainline' + type = 'touch-history' parentIndex = 1 parentCount = 1 mergeCommit = $null branchHead = $null depth = 0 } - lineageLabel = 'Mainline' + lineageLabel = 'Touch history' result = [ordered]@{ diff = $false duration_s = 0.45 @@ -218,6 +218,7 @@ Describe 'Render-VIHistoryReport.ps1' -Tag 'Unit' { $markdown | Should -Match '\| Outcome Labels \| `clean`, `signal-diff` \|' $markdown | Should -Match '\| Mode \| Processed \| Diffs \| Signal \| Collapsed Noise \| Missing \| Categories \| Buckets \| Flags \|' $markdown | Should -Match '\| Mode \| Pair \| Lineage \| Base \| Head \| Diff \| Duration \(s\) \| Categories \| Buckets \| Report \| Highlights \|' + $markdown | Should -Match 'Touch history' $html = Get-Content -LiteralPath $htmlPath -Raw $html | Should -Match 'Source branch' @@ -238,6 +239,7 @@ Describe 'Render-VIHistoryReport.ps1' -Tag 'Unit' { $html | Should -Match 'Lineage' $html | Should -Match 'Categories' $html | Should -Match 'Buckets' + $html | Should -Match 'Touch history' $html | Should -Match 'data-buckets=' $html | Should -Match 'Functional behavior \(1\)' diff --git a/tools/Compare-RefsToTemp.ps1 b/tools/Compare-RefsToTemp.ps1 index f22ad8dd9..68952004f 100644 --- a/tools/Compare-RefsToTemp.ps1 +++ b/tools/Compare-RefsToTemp.ps1 @@ -164,9 +164,10 @@ function Parse-DiffHeadings { $raw = $match.Groups['text'].Value if ([string]::IsNullOrWhiteSpace($raw)) { continue } - $decoded = [System.Net.WebUtility]::HtmlDecode($raw.Trim()) + $decoded = Normalize-ComparisonReportText -Value $raw $decoded = ($decoded -replace '^\s*\d+[\.\)]\s*', '') if ([string]::IsNullOrWhiteSpace($decoded)) { continue } + if (Test-IsComparisonIdentityLabel -Value $decoded) { continue } if (-not $headings.Contains($decoded)) { $headings.Add($decoded) | Out-Null } @@ -176,6 +177,25 @@ function Parse-DiffHeadings { return @($headings.ToArray()) } +function Normalize-ComparisonReportText { + param([string]$Value) + + if ([string]::IsNullOrWhiteSpace($Value)) { return '' } + + $decoded = [System.Net.WebUtility]::HtmlDecode($Value) + $withoutTags = [regex]::Replace($decoded, '<[^>]+>', ' ') + return ([regex]::Replace($withoutTags, '\s+', ' ')).Trim() +} + +function Test-IsComparisonIdentityLabel { + param([string]$Value) + + $normalized = Normalize-ComparisonReportText -Value $Value + if ([string]::IsNullOrWhiteSpace($normalized)) { return $false } + + return ($normalized -match '^\s*First\s+VI:\s*.+?\s+Second\s+VI:\s*.+?\s*$') +} + function Parse-DiffDetails { param([string]$Html) @@ -186,7 +206,8 @@ function Parse-DiffDetails { foreach ($match in [System.Text.RegularExpressions.Regex]::Matches($Html, $pattern, 'IgnoreCase')) { $raw = $match.Groups['text'].Value if ([string]::IsNullOrWhiteSpace($raw)) { continue } - $decoded = [System.Net.WebUtility]::HtmlDecode($raw.Trim()) + $decoded = Normalize-ComparisonReportText -Value $raw + if (Test-IsComparisonIdentityLabel -Value $decoded) { continue } if ($decoded) { $details.Add($decoded) | Out-Null } diff --git a/tools/Compare-VIHistory.ps1 b/tools/Compare-VIHistory.ps1 index d669c5caf..c49521807 100644 --- a/tools/Compare-VIHistory.ps1 +++ b/tools/Compare-VIHistory.ps1 @@ -237,6 +237,29 @@ function Get-ComparisonCategories { return @($categories.ToArray()) } +function Normalize-ComparisonReportText { + param([string]$Value) + + if ([string]::IsNullOrWhiteSpace($Value)) { + return '' + } + + $decoded = [System.Net.WebUtility]::HtmlDecode($Value) + $withoutTags = [regex]::Replace($decoded, '<[^>]+>', ' ') + return ([regex]::Replace($withoutTags, '\s+', ' ')).Trim() +} + +function Test-IsComparisonIdentityLabel { + param([string]$Value) + + $normalized = Normalize-ComparisonReportText -Value $Value + if ([string]::IsNullOrWhiteSpace($normalized)) { + return $false + } + + return ($normalized -match '^\s*First\s+VI:\s*.+?\s+Second\s+VI:\s*.+?\s*$') +} + function Parse-ReportDiffHeadings { param([string]$Html) @@ -258,9 +281,10 @@ function Parse-ReportDiffHeadings { foreach ($match in [System.Text.RegularExpressions.Regex]::Matches($Html, $pattern, $regexOptions)) { $raw = $match.Groups['text'].Value if ([string]::IsNullOrWhiteSpace($raw)) { continue } - $decoded = [System.Net.WebUtility]::HtmlDecode($raw.Trim()) + $decoded = Normalize-ComparisonReportText -Value $raw $decoded = ($decoded -replace '^\s*\d+[\.\)]\s*', '') if ([string]::IsNullOrWhiteSpace($decoded)) { continue } + if (Test-IsComparisonIdentityLabel -Value $decoded) { continue } if (-not $headings.Contains($decoded)) { $headings.Add($decoded) | Out-Null } @@ -280,7 +304,8 @@ function Parse-ReportDiffDetails { foreach ($match in [System.Text.RegularExpressions.Regex]::Matches($Html, $pattern, 'IgnoreCase')) { $raw = $match.Groups['text'].Value if ([string]::IsNullOrWhiteSpace($raw)) { continue } - $decoded = [System.Net.WebUtility]::HtmlDecode($raw.Trim()) + $decoded = Normalize-ComparisonReportText -Value $raw + if (Test-IsComparisonIdentityLabel -Value $decoded) { continue } if ($decoded) { $details.Add($decoded) | Out-Null } @@ -739,6 +764,30 @@ function Test-CommitTouchesPath { return -not [string]::IsNullOrWhiteSpace($result) } +function Get-TouchHistoryCommits { + param( + [Parameter(Mandatory = $true)][string]$Ref, + [Parameter(Mandatory = $true)][string]$Path + ) + + if ([string]::IsNullOrWhiteSpace($Ref) -or [string]::IsNullOrWhiteSpace($Path)) { + return @() + } + + $raw = Invoke-Git -Arguments @('log','--format=%H','--follow','--find-renames=90%',$Ref,'--',$Path) -Quiet + $commits = New-Object System.Collections.Generic.List[string] + $seen = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($line in ($raw -split "`n")) { + $commit = $line.Trim() + if ([string]::IsNullOrWhiteSpace($commit)) { continue } + if ($seen.Add($commit)) { + $commits.Add($commit) | Out-Null + } + } + + return $commits.ToArray() +} + function Get-CommitParents { param( [Parameter(Mandatory = $true)][string]$Commit @@ -930,34 +979,20 @@ function Get-MergeParentPlan { function Build-ComparisonPlan { param( - [Parameter(Mandatory = $true)][string[]]$MainlineCommits, + [Parameter(Mandatory = $true)][string[]]$TouchCommits, [Parameter(Mandatory = $true)][string]$TargetRel, [string]$EndRef, [switch]$IncludeMergeParents ) - $mainlineSet = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase) - $mainlineList = New-Object System.Collections.Generic.List[string] - foreach ($entry in $MainlineCommits) { + $touchSet = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase) + $touchList = New-Object System.Collections.Generic.List[string] + foreach ($entry in $TouchCommits) { if ([string]::IsNullOrWhiteSpace($entry)) { continue } $trimmed = $entry.Trim() if ([string]::IsNullOrWhiteSpace($trimmed)) { continue } - if ($mainlineSet.Add($trimmed)) { - $mainlineList.Add($trimmed) - } - } - - for ($index = 0; $index -lt $mainlineList.Count; $index++) { - $currentCommit = $mainlineList[$index] - $parentsForCurrent = @(Get-CommitParents -Commit $currentCommit) - if (-not $parentsForCurrent -or $parentsForCurrent.Count -eq 0) { continue } - $firstParentForCurrent = $parentsForCurrent[0] - if ([string]::IsNullOrWhiteSpace($firstParentForCurrent)) { continue } - if ($mainlineSet.Add($firstParentForCurrent)) { - $mainlineList.Add($firstParentForCurrent) - } - if ($EndRef -and [string]::Equals($firstParentForCurrent, $EndRef, [System.StringComparison]::OrdinalIgnoreCase)) { - break + if ($touchSet.Add($trimmed)) { + $touchList.Add($trimmed) } } @@ -970,18 +1005,33 @@ function Build-ComparisonPlan { function Add-Spec { param([object]$Spec, [System.Collections.Generic.HashSet[string]]$KeySet, [System.Collections.Generic.List[object]]$PlanList) if (-not $Spec) { return } - if ([string]::IsNullOrWhiteSpace($Spec.Head) -or [string]::IsNullOrWhiteSpace($Spec.Base)) { return } - - $lineage = $Spec.Lineage - $parentIndexKey = if ($lineage -and $lineage.PSObject.Properties['parentIndex']) { [int]$lineage.parentIndex } else { 0 } - $mergeCommitKey = if ($lineage -and $lineage.PSObject.Properties['mergeCommit']) { [string]$lineage.mergeCommit } else { '' } - $depthKey = if ($lineage -and $lineage.PSObject.Properties['depth']) { [int]$lineage.depth } else { 0 } - $key = "{0}|{1}|{2}|{3}|{4}" -f $Spec.Head, $Spec.Base, $parentIndexKey, $mergeCommitKey, $depthKey + $headValue = if ($Spec -is [System.Collections.IDictionary]) { [string]$Spec['Head'] } else { [string]$Spec.Head } + $baseValue = if ($Spec -is [System.Collections.IDictionary]) { [string]$Spec['Base'] } else { [string]$Spec.Base } + if ([string]::IsNullOrWhiteSpace($headValue) -or [string]::IsNullOrWhiteSpace($baseValue)) { return } + + $lineage = if ($Spec -is [System.Collections.IDictionary]) { $Spec['Lineage'] } else { $Spec.Lineage } + $parentIndexKey = if ($lineage -is [System.Collections.IDictionary]) { + if ($lineage.Contains('parentIndex')) { [int]$lineage['parentIndex'] } else { 0 } + } elseif ($lineage -and $lineage.PSObject.Properties['parentIndex']) { + [int]$lineage.parentIndex + } else { 0 } + $mergeCommitKey = if ($lineage -is [System.Collections.IDictionary]) { + if ($lineage.Contains('mergeCommit')) { [string]$lineage['mergeCommit'] } else { '' } + } elseif ($lineage -and $lineage.PSObject.Properties['mergeCommit']) { + [string]$lineage.mergeCommit + } else { '' } + $depthKey = if ($lineage -is [System.Collections.IDictionary]) { + if ($lineage.Contains('depth')) { [int]$lineage['depth'] } else { 0 } + } elseif ($lineage -and $lineage.PSObject.Properties['depth']) { + [int]$lineage.depth + } else { 0 } + $key = "{0}|{1}|{2}|{3}|{4}" -f $headValue, $baseValue, $parentIndexKey, $mergeCommitKey, $depthKey if (-not $KeySet.Add($key)) { return } $PlanList.Add([pscustomobject]$Spec) | Out-Null } - foreach ($rawHead in $mainlineList) { + for ($index = 0; $index -lt $touchList.Count; $index++) { + $rawHead = $touchList[$index] $head = $rawHead.Trim() if (-not $head) { continue } @@ -998,13 +1048,49 @@ function Build-ComparisonPlan { $parentCount = $parents.Count $firstParent = $parents[0] + $nextTouch = $null + if (($index + 1) -lt $touchList.Count) { + $nextTouch = $touchList[$index + 1] + } + + $baseCommit = $null + $stopAfter = $false + $lineageType = 'touch-history' + + if ($EndRef -and $nextTouch -and [string]::Equals($nextTouch, $EndRef, [System.StringComparison]::OrdinalIgnoreCase)) { + $baseCommit = $nextTouch + $stopAfter = $true + } elseif ($EndRef -and (Test-IsAncestorSafe -Ancestor $EndRef -Descendant $head)) { + if ($nextTouch) { + if (Test-IsAncestorSafe -Ancestor $nextTouch -Descendant $EndRef) { + $baseCommit = $EndRef + $stopAfter = $true + } + } else { + $baseCommit = $EndRef + $stopAfter = $true + } + } + + if (-not $baseCommit) { + if ($nextTouch) { + $baseCommit = $nextTouch + } else { + $baseCommit = $firstParent + $lineageType = 'mainline' + } + } + + if ([string]::IsNullOrWhiteSpace($baseCommit)) { + $terminalHint = 'reached-root' + break + } - $stopAfter = $EndRef -and [string]::Equals($firstParent, $EndRef, [System.StringComparison]::OrdinalIgnoreCase) $spec = [ordered]@{ Head = $head - Base = $firstParent + Base = $baseCommit Lineage = [ordered]@{ - type = 'mainline' + type = $lineageType parentIndex = 1 parentCount = $parentCount mergeCommit = $head @@ -1030,7 +1116,7 @@ function Build-ComparisonPlan { if ($stopAfter) { break } } - return [ordered]@{ + return [pscustomobject]@{ Plan = $plan.ToArray() TerminalHint = $terminalHint } @@ -1054,6 +1140,45 @@ function Test-IsAncestor { throw ("git merge-base --is-ancestor failed: {0}" -f $result.StdErr) } +function Test-IsAncestorSafe { + param( + [Parameter(Mandatory = $true)][string]$Ancestor, + [Parameter(Mandatory = $true)][string]$Descendant + ) + + try { + return Test-IsAncestor -Ancestor $Ancestor -Descendant $Descendant + } catch { + return $false + } +} + +function Select-TouchHistoryWindow { + param( + [Parameter(Mandatory = $true)][string[]]$TouchCommits, + [Parameter(Mandatory = $true)][string]$StartCommit + ) + + $selected = New-Object System.Collections.Generic.List[string] + $startFound = $false + foreach ($rawCommit in $TouchCommits) { + $commit = $rawCommit.Trim() + if ([string]::IsNullOrWhiteSpace($commit)) { continue } + if (-not $startFound) { + if (-not [string]::Equals($commit, $StartCommit, [System.StringComparison]::OrdinalIgnoreCase)) { + continue + } + $startFound = $true + } + $selected.Add($commit) | Out-Null + } + + return [pscustomobject]@{ + Commits = $selected.ToArray() + StartFound = $startFound + } +} + function Resolve-CommitWithChange { param( [Parameter(Mandatory = $true)][string]$StartRef, @@ -1065,27 +1190,30 @@ function Resolve-CommitWithChange { return $StartRef } - $upRaw = Invoke-Git -Arguments @('rev-list','--first-parent',"$StartRef..$HeadRef",'--',$Path) -Quiet - $upList = @($upRaw -split "`n" | Where-Object { $_ }) - if ($upList.Count -gt 0) { - for ($i = $upList.Count - 1; $i -ge 0; $i--) { - $commit = $upList[$i] - if (Test-IsAncestor -Ancestor $StartRef -Descendant $commit) { - return $commit + $headTouchHistory = @(Get-TouchHistoryCommits -Ref $HeadRef -Path $Path) + if ($headTouchHistory.Count -gt 0) { + $nearestNewer = $null + foreach ($commit in $headTouchHistory) { + if (Test-IsAncestorSafe -Ancestor $StartRef -Descendant $commit) { + $nearestNewer = $commit } } - } + if ($nearestNewer) { + return $nearestNewer + } - $downRaw = Invoke-Git -Arguments @('rev-list','--first-parent',$StartRef,'--',$Path) -Quiet - $downList = @($downRaw -split "`n" | Where-Object { $_ }) - if ($downList.Count -gt 0) { - foreach ($commit in $downList) { - if (Test-CommitTouchesPath -Commit $commit -Path $Path) { + foreach ($commit in $headTouchHistory) { + if (Test-IsAncestorSafe -Ancestor $commit -Descendant $StartRef) { return $commit } } } + $downList = @(Get-TouchHistoryCommits -Ref $StartRef -Path $Path) + if ($downList.Count -gt 0) { + return $downList[0] + } + return $null } @@ -1348,16 +1476,30 @@ Write-Verbose ("StartRef before ensure: {0}; Target: {1}" -f $startRef, $targetR Ensure-FileExistsAtRef -Ref $startRef -Path $targetRel if ($endRef) { Ensure-FileExistsAtRef -Ref $endRef -Path $targetRel } -$revArgs = @('rev-list','--first-parent',$startRef) -if ($maxPairsRequested) { - $revArgs += ("--max-count={0}" -f ([int]($MaxPairs + 5))) +$allTouchCommits = @(Get-TouchHistoryCommits -Ref 'HEAD' -Path $targetRel) +$touchWindow = Select-TouchHistoryWindow -TouchCommits $allTouchCommits -StartCommit $startRef +$commitList = @() +if ($touchWindow.StartFound) { + $commitList = @($touchWindow.Commits) +} elseif (Test-CommitTouchesPath -Commit $startRef -Path $targetRel) { + $olderTouches = New-Object System.Collections.Generic.List[string] + foreach ($commit in $allTouchCommits) { + if (Test-IsAncestorSafe -Ancestor $commit -Descendant $startRef) { + $olderTouches.Add($commit) | Out-Null + } + } + $commitList = @($startRef) + if ($olderTouches.Count -gt 0) { + $commitList += @($olderTouches.ToArray()) + } +} else { + $commitList = @(Get-TouchHistoryCommits -Ref $startRef -Path $targetRel) +} +if ($maxPairsRequested -and $commitList.Count -gt ($MaxPairs + 5)) { + $commitList = @($commitList | Select-Object -First ([int]($MaxPairs + 5))) } -$revArgs += '--' -$revArgs += $targetRel -$revListRaw = Invoke-Git -Arguments $revArgs -Quiet -$commitList = @($revListRaw -split "`n" | Where-Object { $_ }) Write-Verbose ("Commit list count: {0}" -f $commitList.Count) -$planResult = Build-ComparisonPlan -MainlineCommits $commitList -TargetRel $targetRel -EndRef $endRef -IncludeMergeParents:$IncludeMergeParents.IsPresent +$planResult = Build-ComparisonPlan -TouchCommits $commitList -TargetRel $targetRel -EndRef $endRef -IncludeMergeParents:$IncludeMergeParents.IsPresent $comparisonPlan = @() $planTerminalHint = $null if ($planResult) { @@ -1916,10 +2058,18 @@ foreach ($modeSpec in $modeSpecs) { $cliCategoryBuckets = @() $cliCategoryBucketDetails = @() if ($summaryJson.cli -and $summaryJson.cli.PSObject.Properties['highlights'] -and $summaryJson.cli.highlights) { - $highlights += @($summaryJson.cli.highlights | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) + $highlights += @( + $summaryJson.cli.highlights | + Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | + Where-Object { -not (Test-IsComparisonIdentityLabel -Value ([string]$_)) } + ) } if ($summaryJson.cli -and $summaryJson.cli.PSObject.Properties['categories'] -and $summaryJson.cli.categories) { - $cliCategories += @($summaryJson.cli.categories | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) + $cliCategories += @( + $summaryJson.cli.categories | + Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | + Where-Object { -not (Test-IsComparisonIdentityLabel -Value ([string]$_)) } + ) } if ($summaryJson.cli -and $summaryJson.cli.PSObject.Properties['categoryDetails'] -and $summaryJson.cli.categoryDetails) { $cliCategoryDetails += @($summaryJson.cli.categoryDetails) diff --git a/tools/Render-VIHistoryReport.ps1 b/tools/Render-VIHistoryReport.ps1 index c88a448c8..143bf7a98 100644 --- a/tools/Render-VIHistoryReport.ps1 +++ b/tools/Render-VIHistoryReport.ps1 @@ -126,6 +126,9 @@ function Get-LineageLabel { $rootMerge = if ($Lineage.PSObject.Properties['rootMerge']) { [string]$Lineage.rootMerge } else { $mergeCommit } switch ($type.ToLowerInvariant()) { + 'touch-history' { + return 'Touch history' + } 'merge-parent' { $label = if ($parentIndex -and $parentIndex -gt 0) { "Merge parent #$parentIndex" } else { 'Merge parent' } if ($depth -and $depth -gt 0) { From 809ba89153b5fa61bf387093461a87843a7ccad6 Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Mon, 30 Mar 2026 15:48:40 -0700 Subject: [PATCH 04/44] Preserve VI history category specificity (#2057) Co-authored-by: svelderrainruiz --- tests/CompareVI.History.Tests.ps1 | 13 +++- tests/VICategoryBuckets.Tests.ps1 | 8 +++ tools/Compare-RefsToTemp.ps1 | 82 ++++++++++++++++++++++-- tools/Compare-VIHistory.ps1 | 101 +++++++++++++++++++++++++++--- tools/VICategoryBuckets.psm1 | 28 +++++---- 5 files changed, 203 insertions(+), 29 deletions(-) diff --git a/tests/CompareVI.History.Tests.ps1 b/tests/CompareVI.History.Tests.ps1 index a18f58342..2808afe73 100644 --- a/tests/CompareVI.History.Tests.ps1 +++ b/tests/CompareVI.History.Tests.ps1 @@ -512,6 +512,7 @@ exit 0 $aggregate.stats.signalDiffs | Should -Be 0 $aggregate.stats.noiseCollapsed | Should -BeGreaterThan 0 $aggregate.stats.categoryCounts.PSObject.Properties.Name | Should -Contain 'VI Attribute' + $aggregate.stats.categoryCounts.PSObject.Properties.Name | Should -Not -Contain 'Cosmetic' $aggregate.stats.categoryCounts.PSObject.Properties.Name | Should -Not -Contain 'unspecified' [int]$aggregate.stats.bucketCounts.metadata | Should -BeGreaterThan 0 $outputText | Should -Match 'LVCompare detected differences' @@ -766,6 +767,7 @@ exit 0 $manifest.flags | Should -Contain '-nobdcosm' $manifest.stats.stopReason | Should -Be 'max-pairs' $manifest.comparisons.Count | Should -Be 0 + $manifest.stats.categoryCounts.PSObject.Properties.Name | Should -Not -Contain 'Cosmetic' } It 'captures xml report when alternate format requested' { @@ -1513,11 +1515,12 @@ exit 0 FixtureRel = Join-Path 'fixtures' 'vi-report' 'block-diagram' ExpectPattern = 'Block Diagram Cosmetic' ExpectedCategories = @('cosmetic') + ExpectedDetailSlug = 'block-diagram-cosmetic' } ) It "surfaces highlights when suppression is removed ()" -TestCases $fixtureCases { - param($Name, $Param, $FixtureRel, $ExpectPattern, $ExpectedCategories) + param($Name, $Param, $FixtureRel, $ExpectPattern, $ExpectedCategories, $ExpectedDetailSlug) if (-not $_pairs) { Set-ItResult -Skipped -Because 'Missing commit data'; return } $pair = $_pairs[0] @@ -1581,6 +1584,14 @@ exit 0 if ($targetFlag) { ($variantManifest.flags -contains $targetFlag) | Should -BeFalse } + if (-not [string]::IsNullOrWhiteSpace($ExpectedDetailSlug)) { + $variantComparison = @($variantManifest.comparisons)[0] + $variantComparison | Should -Not -BeNullOrEmpty + $variantComparison.result.categories | Should -Contain $ExpectPattern + $detailSlugs = @($variantComparison.result.categoryDetails | ForEach-Object { [string]$_.slug }) + $detailSlugs | Should -Contain $ExpectedDetailSlug + $detailSlugs | Should -Not -Contain 'cosmetic' + } $historyReport = Get-Content -LiteralPath (Join-Path $variantDir 'history-report.md') -Raw $historyReport | Should -Match '\| Metric \| Value \|' diff --git a/tests/VICategoryBuckets.Tests.ps1 b/tests/VICategoryBuckets.Tests.ps1 index 56db0183f..ab7693b4f 100644 --- a/tests/VICategoryBuckets.Tests.ps1 +++ b/tests/VICategoryBuckets.Tests.ps1 @@ -14,6 +14,14 @@ Describe 'VICategoryBuckets module' -Tag 'Unit' { $meta.bucketClassification | Should -Be 'neutral' } + It 'preserves known slug inputs without degrading category specificity' { + $meta = Get-VICategoryMetadata -Name 'block-diagram-cosmetic' + $meta | Should -Not -BeNullOrEmpty + $meta.slug | Should -Be 'block-diagram-cosmetic' + $meta.label | Should -Be 'Block diagram (cosmetic)' + $meta.bucketSlug | Should -Be 'ui-visual' + } + It 'collects bucket details for multiple categories' { $inputCategories = @( 'Block Diagram Functional', diff --git a/tools/Compare-RefsToTemp.ps1 b/tools/Compare-RefsToTemp.ps1 index 68952004f..09e6cd416 100644 --- a/tools/Compare-RefsToTemp.ps1 +++ b/tools/Compare-RefsToTemp.ps1 @@ -28,12 +28,6 @@ $ErrorActionPreference = 'Stop' try { git --version | Out-Null } catch { throw 'git is required on PATH to fetch file content at refs.' } $repoRoot = (Get-Location).Path -try { - $categoryModule = Join-Path $repoRoot 'tools' 'VICategoryBuckets.psm1' - if (Test-Path -LiteralPath $categoryModule -PathType Leaf) { - Import-Module $categoryModule -Force - } -} catch {} function Resolve-CompareVIScriptsRoot { param([string]$PrimaryRoot) @@ -55,6 +49,21 @@ function Resolve-CompareVIScriptsRoot { return $PrimaryRoot } +try { + $categoryModuleCandidates = New-Object System.Collections.Generic.List[string] + $categoryModuleCandidates.Add((Join-Path $repoRoot 'tools' 'VICategoryBuckets.psm1')) | Out-Null + $resolvedScriptsRoot = Resolve-CompareVIScriptsRoot -PrimaryRoot $repoRoot + if (-not [string]::IsNullOrWhiteSpace($resolvedScriptsRoot)) { + $categoryModuleCandidates.Add((Join-Path $resolvedScriptsRoot 'tools' 'VICategoryBuckets.psm1')) | Out-Null + } + foreach ($candidate in @($categoryModuleCandidates | Select-Object -Unique)) { + if (Test-Path -LiteralPath $candidate -PathType Leaf) { + Import-Module $candidate -Force + break + } + } +} catch {} + function Split-ArgString { param([string]$Value) if ([string]::IsNullOrWhiteSpace($Value)) { return @() } @@ -275,6 +284,61 @@ function Infer-DiffCategoriesFromDetails { return @($inferred.ToArray()) } +function Normalize-ReportCategories { + param([System.Collections.IEnumerable]$Categories) + + if (-not $Categories -or -not (Get-Command -Name Get-VICategoryBuckets -ErrorAction SilentlyContinue)) { + return @( + $Categories | + Where-Object { -not [string]::IsNullOrWhiteSpace([string]$_) } | + Select-Object -Unique + ) + } + + $categoryInfo = Get-VICategoryBuckets -Names @($Categories) + if ($null -eq $categoryInfo -or -not $categoryInfo.Details) { + return @( + $Categories | + Where-Object { -not [string]::IsNullOrWhiteSpace([string]$_) } | + Select-Object -Unique + ) + } + + $details = @($categoryInfo.Details) + if ($details.Count -gt 1) { + $specificDetails = @($details | Where-Object { [string]$_.slug -ne 'cosmetic' }) + if ($specificDetails.Count -gt 0) { + $details = $specificDetails + } + } + + return @( + $details | + ForEach-Object { + switch ([string]$_.slug) { + 'block-diagram' { 'Block Diagram' } + 'block-diagram-functional' { 'Block Diagram Functional' } + 'block-diagram-cosmetic' { 'Block Diagram Cosmetic' } + 'connector-pane' { 'Connector Pane' } + 'front-panel' { 'Front Panel' } + 'front-panel-position-size' { 'Front Panel Position/Size' } + 'control-changes' { 'Front Panel Controls' } + 'window' { 'Window Properties' } + 'attributes' { 'Attributes' } + 'vi-attribute' { 'VI Attribute' } + 'documentation' { 'Documentation' } + 'execution' { 'Execution Settings' } + 'icon' { 'Icon' } + 'unspecified' { 'Unspecified' } + 'cosmetic' { 'Cosmetic' } + default { if ([string]::IsNullOrWhiteSpace([string]$_.label)) { [string]$_.slug } else { [string]$_.label } } + } + } | + Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | + Select-Object -Unique + ) +} + function Get-ReportCategoryMetadata { param([string]$ReportPath) @@ -337,6 +401,12 @@ function Get-ReportCategoryMetadata { } } + $normalizedCategories = @(Normalize-ReportCategories -Categories @($categories.ToArray())) + $categories = New-Object System.Collections.Generic.List[string] + foreach ($name in $normalizedCategories) { + $categories.Add([string]$name) | Out-Null + } + $categoryDetails = @() $categoryBuckets = @() $categoryBucketDetails = @() diff --git a/tools/Compare-VIHistory.ps1 b/tools/Compare-VIHistory.ps1 index c49521807..2d6a7b6fa 100644 --- a/tools/Compare-VIHistory.ps1 +++ b/tools/Compare-VIHistory.ps1 @@ -435,6 +435,12 @@ function Get-ComparisonCategoriesFromReport { } } + $normalizedCategories = @(Normalize-ComparisonCategoryList -Categories @($categories.ToArray())) + $categories = New-Object System.Collections.Generic.List[string] + foreach ($name in $normalizedCategories) { + $categories.Add([string]$name) | Out-Null + } + $categoryDetails = @() $categoryBuckets = @() $categoryBucketDetails = @() @@ -505,6 +511,81 @@ function Get-ComparisonCategoryDisplayName { return $Name } +function Get-CanonicalComparisonCategoryName { + param([string]$Name) + + if ([string]::IsNullOrWhiteSpace($Name)) { return $null } + + $meta = $null + try { + $meta = Get-VICategoryMetadata -Name $Name + } catch { + $meta = $null + } + + if ($meta) { + switch ([string]$meta.slug) { + 'block-diagram' { return 'Block Diagram' } + 'block-diagram-functional' { return 'Block Diagram Functional' } + 'block-diagram-cosmetic' { return 'Block Diagram Cosmetic' } + 'connector-pane' { return 'Connector Pane' } + 'front-panel' { return 'Front Panel' } + 'front-panel-position-size' { return 'Front Panel Position/Size' } + 'control-changes' { return 'Front Panel Controls' } + 'window' { return 'Window Properties' } + 'attributes' { return 'Attributes' } + 'vi-attribute' { return 'VI Attribute' } + 'documentation' { return 'Documentation' } + 'execution' { return 'Execution Settings' } + 'icon' { return 'Icon' } + 'unspecified' { return 'Unspecified' } + 'cosmetic' { return 'Cosmetic' } + } + + if (-not [string]::IsNullOrWhiteSpace([string]$meta.label)) { + return [string]$meta.label + } + } + + return [string]$Name +} + +function Normalize-ComparisonCategoryList { + param([System.Collections.IEnumerable]$Categories) + + if (-not $Categories) { return @() } + + $categoryInfo = $null + try { + $categoryInfo = Get-VICategoryBuckets -Names @($Categories) + } catch { + $categoryInfo = $null + } + + if ($null -eq $categoryInfo -or -not $categoryInfo.Details) { + return @( + $Categories | + Where-Object { -not [string]::IsNullOrWhiteSpace([string]$_) } | + Select-Object -Unique + ) + } + + $details = @($categoryInfo.Details) + if ($details.Count -gt 1) { + $specificDetails = @($details | Where-Object { [string]$_.slug -ne 'cosmetic' }) + if ($specificDetails.Count -gt 0) { + $details = $specificDetails + } + } + + return @( + $details | + ForEach-Object { Get-CanonicalComparisonCategoryName -Name ([string]$_.slug) } | + Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | + Select-Object -Unique + ) +} + function Update-TallyFromDetails { param( [System.Collections.IDictionary]$Target, @@ -2102,6 +2183,7 @@ foreach ($modeSpec in $modeSpecs) { $outNode = $summaryJson.out if ((@($cliCategories).Count -eq 0 -or @($cliCategoryDetails).Count -eq 0) -and $outNode) { $reportCandidate = $null + $reportCategoryMetadata = $null if ($outNode.PSObject.Properties['reportHtml'] -and $outNode.reportHtml) { $reportCandidate = [string]$outNode.reportHtml } elseif ($outNode.PSObject.Properties['reportPath'] -and $outNode.reportPath) { @@ -2145,29 +2227,30 @@ foreach ($modeSpec in $modeSpecs) { Select-Object -Unique ) } + $categories = @(Normalize-ComparisonCategoryList -Categories $categories) $categoryInfo = $null if (@($categories).Count -gt 0) { $categoryInfo = Get-VICategoryBuckets -Names $categories } - $resolvedCategoryDetails = if (@($cliCategoryDetails).Count -gt 0) { - @($cliCategoryDetails) - } elseif ($categoryInfo -and $categoryInfo.Details) { + $resolvedCategoryDetails = if ($categoryInfo -and $categoryInfo.Details) { @($categoryInfo.Details) + } elseif (@($cliCategoryDetails).Count -gt 0) { + @($cliCategoryDetails) } else { @() } - $resolvedCategoryBuckets = if (@($cliCategoryBuckets).Count -gt 0) { - @($cliCategoryBuckets | Select-Object -Unique) - } elseif ($categoryInfo -and $categoryInfo.BucketSlugs) { + $resolvedCategoryBuckets = if ($categoryInfo -and $categoryInfo.BucketSlugs) { @($categoryInfo.BucketSlugs) + } elseif (@($cliCategoryBuckets).Count -gt 0) { + @($cliCategoryBuckets | Select-Object -Unique) } else { @() } - $resolvedCategoryBucketDetails = if (@($cliCategoryBucketDetails).Count -gt 0) { - @($cliCategoryBucketDetails) - } elseif ($categoryInfo -and $categoryInfo.BucketDetails) { + $resolvedCategoryBucketDetails = if ($categoryInfo -and $categoryInfo.BucketDetails) { @($categoryInfo.BucketDetails) + } elseif (@($cliCategoryBucketDetails).Count -gt 0) { + @($cliCategoryBucketDetails) } else { @() } diff --git a/tools/VICategoryBuckets.psm1 b/tools/VICategoryBuckets.psm1 index e839735b7..9284566c2 100644 --- a/tools/VICategoryBuckets.psm1 +++ b/tools/VICategoryBuckets.psm1 @@ -105,19 +105,21 @@ function Resolve-VICategorySlug { if ([string]::IsNullOrWhiteSpace($Name)) { return $null } $token = $Name.Trim().ToLowerInvariant() - - if ($token -match 'block diagram' -and $token -match 'cosmetic') { return 'block-diagram-cosmetic' } - if ($token -match 'block diagram' -and $token -match 'functional') { return 'block-diagram-functional' } - if ($token -match 'block diagram') { return 'block-diagram' } - if ($token -match 'connector') { return 'connector-pane' } - if ($token -match 'vi attribute' -or $token -match 'attributes') { return 'vi-attribute' } - if ($token -match 'front panel position') { return 'front-panel-position-size' } - if ($token -match 'front panel' -or $token -match 'control changes') { return 'front-panel' } - if ($token -match 'cosmetic') { return 'cosmetic' } - if ($token -match 'window') { return 'window' } - if ($token -match 'icon') { return 'icon' } - if ($token -match 'documentation') { return 'documentation' } - if ($token -match 'execution') { return 'execution' } + if ($script:CategoryDefinitions.ContainsKey($token)) { return $token } + $normalizedToken = $token -replace '[-_]+', ' ' + + if ($normalizedToken -match 'block diagram' -and $normalizedToken -match 'cosmetic') { return 'block-diagram-cosmetic' } + if ($normalizedToken -match 'block diagram' -and $normalizedToken -match 'functional') { return 'block-diagram-functional' } + if ($normalizedToken -match 'block diagram') { return 'block-diagram' } + if ($normalizedToken -match 'connector') { return 'connector-pane' } + if ($normalizedToken -match 'vi attribute' -or $normalizedToken -match 'attributes') { return 'vi-attribute' } + if ($normalizedToken -match 'front panel position') { return 'front-panel-position-size' } + if ($normalizedToken -match 'front panel' -or $normalizedToken -match 'control changes') { return 'front-panel' } + if ($normalizedToken -match 'cosmetic') { return 'cosmetic' } + if ($normalizedToken -match 'window') { return 'window' } + if ($normalizedToken -match 'icon') { return 'icon' } + if ($normalizedToken -match 'documentation') { return 'documentation' } + if ($normalizedToken -match 'execution') { return 'execution' } return ($token -replace '[^a-z0-9]+', '-').Trim('-') } From 520068df6c41a1eeb0813d4cbf45f34e222d6902 Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Mon, 30 Mar 2026 18:44:53 -0700 Subject: [PATCH 05/44] Reveal collapsed VI history pairs (#2058) Co-authored-by: svelderrainruiz --- tests/CompareVI.History.Tests.ps1 | 16 ++++++++++--- tests/VICategoryBuckets.Tests.ps1 | 8 +++++++ tools/Compare-RefsToTemp.ps1 | 2 +- tools/Compare-VIHistory.ps1 | 11 ++++++--- tools/Publish-VICompareSummary.ps1 | 4 ++-- tools/Render-VIHistoryReport.ps1 | 38 +++++++++++++++++++++++++----- tools/VICategoryBuckets.psm1 | 1 + 7 files changed, 65 insertions(+), 15 deletions(-) diff --git a/tests/CompareVI.History.Tests.ps1 b/tests/CompareVI.History.Tests.ps1 index 2808afe73..dd6d854a5 100644 --- a/tests/CompareVI.History.Tests.ps1 +++ b/tests/CompareVI.History.Tests.ps1 @@ -502,6 +502,7 @@ exit 0 $modeManifest.stats.signalDiffs | Should -Be 0 $modeManifest.stats.noiseCollapsed | Should -BeGreaterThan 0 @($modeManifest.comparisons).Count | Should -Be 0 + @($modeManifest.collapsedComparisons).Count | Should -BeGreaterThan 0 $modeManifest.stats.collapsedNoise.count | Should -Be $modeManifest.stats.noiseCollapsed $modeManifest.stats.collapsedNoise.categoryCounts.PSObject.Properties.Name | Should -Contain 'vi-attribute' $modeManifest.stats.collapsedNoise.categoryCounts.PSObject.Properties.Name | Should -Not -Contain 'unspecified' @@ -512,12 +513,17 @@ exit 0 $aggregate.stats.signalDiffs | Should -Be 0 $aggregate.stats.noiseCollapsed | Should -BeGreaterThan 0 $aggregate.stats.categoryCounts.PSObject.Properties.Name | Should -Contain 'VI Attribute' + $aggregate.stats.categoryCounts.PSObject.Properties.Name | Should -Not -Contain 'Attributes' $aggregate.stats.categoryCounts.PSObject.Properties.Name | Should -Not -Contain 'Cosmetic' $aggregate.stats.categoryCounts.PSObject.Properties.Name | Should -Not -Contain 'unspecified' [int]$aggregate.stats.bucketCounts.metadata | Should -BeGreaterThan 0 $outputText | Should -Match 'LVCompare detected differences' $outputText | Should -Match 'VI attribute \(\d+\)' $outputText | Should -Not -Match 'unspecified' + + $historyMd = Get-Content -LiteralPath (Join-Path $rd 'history-report.md') -Raw + $historyMd | Should -Match 'collapsed noise' + $historyMd | Should -Match 'Touch history' } finally { if ($null -eq $previousDiff) { Remove-Item Env:STUB_COMPARE_DIFF -ErrorAction SilentlyContinue @@ -1449,7 +1455,10 @@ exit 0 $historyMd | Should -Match '\| Coverage Class \| `catalog-aligned` \|' $historyMd | Should -Match '## Mode overview' $historyMd | Should -Match '\| Mode \| Processed \| Diffs \| Signal \| Collapsed Noise \| Missing \| Categories \| Buckets \| Flags \|' - $historyMd | Should -Match '## Attribute coverage' + $historyMd | Should -Match '## Commit pairs' + $historyMd | Should -Match 'collapsed noise' + $historyMd | Should -Match 'Touch history' + $historyMd | Should -Match '## Mode filter coverage' $historyMd | Should -Match 'History manifest:' $historyHtml = Get-Content -LiteralPath (Join-Path $rd 'history-report.html') -Raw @@ -1463,8 +1472,9 @@ exit 0 $historyHtml | Should -Match 'Signal' $historyHtml | Should -Match 'Collapsed Noise' $historyHtml | Should -Match '

Commit pairs

' - $historyHtml | Should -Match 'No commit pairs were captured' - $historyHtml | Should -Match '

Attribute coverage

' + $historyHtml | Should -Match 'Collapsed noise' + $historyHtml | Should -Match 'Touch history' + $historyHtml | Should -Match '

Mode filter coverage

' } finally { if ($null -eq $previousDiff) { Remove-Item Env:STUB_COMPARE_DIFF -ErrorAction SilentlyContinue diff --git a/tests/VICategoryBuckets.Tests.ps1 b/tests/VICategoryBuckets.Tests.ps1 index ab7693b4f..51267a89b 100644 --- a/tests/VICategoryBuckets.Tests.ps1 +++ b/tests/VICategoryBuckets.Tests.ps1 @@ -14,6 +14,14 @@ Describe 'VICategoryBuckets module' -Tag 'Unit' { $meta.bucketClassification | Should -Be 'neutral' } + It 'treats generic Attributes as an alias of VI Attribute' { + $meta = Get-VICategoryMetadata -Name 'Attributes' + $meta | Should -Not -BeNullOrEmpty + $meta.slug | Should -Be 'vi-attribute' + $meta.label | Should -Be 'VI attribute' + $meta.bucketSlug | Should -Be 'metadata' + } + It 'preserves known slug inputs without degrading category specificity' { $meta = Get-VICategoryMetadata -Name 'block-diagram-cosmetic' $meta | Should -Not -BeNullOrEmpty diff --git a/tools/Compare-RefsToTemp.ps1 b/tools/Compare-RefsToTemp.ps1 index 09e6cd416..8fa85531c 100644 --- a/tools/Compare-RefsToTemp.ps1 +++ b/tools/Compare-RefsToTemp.ps1 @@ -324,7 +324,7 @@ function Normalize-ReportCategories { 'front-panel-position-size' { 'Front Panel Position/Size' } 'control-changes' { 'Front Panel Controls' } 'window' { 'Window Properties' } - 'attributes' { 'Attributes' } + 'attributes' { 'VI Attribute' } 'vi-attribute' { 'VI Attribute' } 'documentation' { 'Documentation' } 'execution' { 'Execution Settings' } diff --git a/tools/Compare-VIHistory.ps1 b/tools/Compare-VIHistory.ps1 index 2d6a7b6fa..694f9fe89 100644 --- a/tools/Compare-VIHistory.ps1 +++ b/tools/Compare-VIHistory.ps1 @@ -533,7 +533,7 @@ function Get-CanonicalComparisonCategoryName { 'front-panel-position-size' { return 'Front Panel Position/Size' } 'control-changes' { return 'Front Panel Controls' } 'window' { return 'Window Properties' } - 'attributes' { return 'Attributes' } + 'attributes' { return 'VI Attribute' } 'vi-attribute' { return 'VI Attribute' } 'documentation' { return 'Documentation' } 'execution' { return 'Execution Settings' } @@ -1837,6 +1837,7 @@ foreach ($modeSpec in $modeSpecs) { flags = $modeFlags resultsDir = $modeResultsResolved comparisons = @() + collapsedComparisons = @() stats = [ordered]@{ processed = 0 diffs = 0 @@ -2361,6 +2362,10 @@ foreach ($modeSpec in $modeSpecs) { } } + if ($collapsedThis) { + $modeManifest.collapsedComparisons += $comparisonRecordObject + } + if ($appendComparison) { $modeManifest.comparisons += $comparisonRecordObject } @@ -2836,7 +2841,7 @@ if (-not $renderSucceeded) { '| --- | --- | --- |' '| n/a | n/a | [report](./) |' '' - '## Attribute coverage' + '## Mode filter coverage' '' '_History renderer unavailable; see manifest for details._' ) @@ -2866,7 +2871,7 @@ if (-not $renderSucceeded) {
-

Attribute coverage

+

Mode filter coverage

History renderer unavailable.

diff --git a/tools/Publish-VICompareSummary.ps1 b/tools/Publish-VICompareSummary.ps1 index 56cd31d7e..1fb61cc9a 100644 --- a/tools/Publish-VICompareSummary.ps1 +++ b/tools/Publish-VICompareSummary.ps1 @@ -232,7 +232,7 @@ if ($totalDiffs -gt 0) { } $lines.Add("") -$lines.Add("#### Attribute coverage") +$lines.Add("#### Mode filter coverage") $attributeCoverageAdded = $false foreach ($mode in $modeSummaries) { $modeName = Get-ObjectPropertyValue -InputObject $mode -PropertyName 'mode' @@ -296,7 +296,7 @@ foreach ($mode in $modeSummaries) { } if (-not $attributeCoverageAdded) { - $lines.Add("- *(Attribute coverage unavailable in manifest)*") + $lines.Add("- *(Mode filter coverage unavailable in manifest)*") } $body = $lines -join "`n" diff --git a/tools/Render-VIHistoryReport.ps1 b/tools/Render-VIHistoryReport.ps1 index 143bf7a98..f8de616a3 100644 --- a/tools/Render-VIHistoryReport.ps1 +++ b/tools/Render-VIHistoryReport.ps1 @@ -823,7 +823,19 @@ function Build-FallbackHistoryContext { continue } - foreach ($comparison in @($modeManifest.comparisons)) { + $modeComparisons = New-Object System.Collections.Generic.List[object] + foreach ($comparisonEntry in @($modeManifest.comparisons)) { + if ($comparisonEntry) { + $modeComparisons.Add($comparisonEntry) | Out-Null + } + } + foreach ($comparisonEntry in @($modeManifest.collapsedComparisons)) { + if ($comparisonEntry) { + $modeComparisons.Add($comparisonEntry) | Out-Null + } + } + + foreach ($comparison in @($modeComparisons | Sort-Object { [int](Coalesce $_.index 0) })) { if (-not $comparison) { continue } $baseNode = $comparison.base $headNode = $comparison.head @@ -864,6 +876,9 @@ function Build-FallbackHistoryContext { if ($resultNode.PSObject.Properties['classification'] -and $resultNode.classification) { $resultPayload.classification = $resultNode.classification } + if ($resultNode.PSObject.Properties['collapsed']) { + $resultPayload.collapsed = [bool]$resultNode.collapsed + } if ($resultNode.PSObject.Properties['artifactDir'] -and $resultNode.artifactDir) { $resultPayload.artifactDir = $resultNode.artifactDir } @@ -1217,7 +1232,13 @@ if ($comparisons.Count -gt 0) { $diffValue = $hasDiffValue -and ($resultNode.diff -eq $true) $statusValue = if ($resultNode -and $resultNode.PSObject.Properties['status']) { [string]$resultNode.status } else { $null } $diffCell = if ($hasDiffValue) { - if ($diffValue) { '**diff**' } else { 'clean' } + if ($diffValue) { + if ($resultNode -and $resultNode.PSObject.Properties['collapsed'] -and [bool]$resultNode.collapsed) { + '_collapsed noise_' + } else { + '**diff**' + } + } else { 'clean' } } elseif ($statusValue) { ('_{0}_' -f $statusValue) } else { @@ -1310,6 +1331,7 @@ if ($comparisons.Count -gt 0) { LineageLabel = $lineageLabel LineageType = if ($lineageNode -and $lineageNode.PSObject.Properties['type']) { [string]$lineageNode.type } else { 'mainline' } Diff = [bool]$diffValue + Collapsed = if ($resultNode -and $resultNode.PSObject.Properties['collapsed']) { [bool]$resultNode.collapsed } else { $false } HasDiff = $hasDiffValue Status = $statusValue Duration = $durationValue @@ -1336,7 +1358,7 @@ if ($comparisons.Count -gt 0) { } $summaryLines.Add('') -$summaryLines.Add('## Attribute coverage') +$summaryLines.Add('## Mode filter coverage') $summaryLines.Add('') if ($modeEntries.Count -gt 0) { foreach ($mode in $modeEntries) { @@ -1518,8 +1540,12 @@ if ($emitHtml -and $HtmlPath) { [void]$htmlBuilder.AppendLine(' ModePairLineageBaseHeadDiffDuration (s)CategoriesBucketsReportHighlights') [void]$htmlBuilder.AppendLine(' ') foreach ($row in $comparisonHtmlRows) { - $diffClass = if ($row.Diff) { 'diff-yes' } elseif ($row.Status) { 'diff-status' } else { 'diff-no' } - $diffLabel = if ($row.Diff) { 'Diff' } elseif ($row.Status) { ConvertTo-HtmlSafe $row.Status } else { 'No' } + $diffClass = if ($row.Diff) { + if ($row.Collapsed) { 'diff-collapsed' } else { 'diff-yes' } + } elseif ($row.Status) { 'diff-status' } else { 'diff-no' } + $diffLabel = if ($row.Diff) { + if ($row.Collapsed) { 'Collapsed noise' } else { 'Diff' } + } elseif ($row.Status) { ConvertTo-HtmlSafe $row.Status } else { 'No' } $durationDisplay = 'n/a' if ($row.DurationDisplay -and $row.DurationDisplay -ne 'n/a') { $durationDisplay = ConvertTo-HtmlSafe $row.DurationDisplay @@ -1688,7 +1714,7 @@ if ($emitHtml -and $HtmlPath) { [void]$htmlBuilder.AppendLine('

No commit pairs were captured for the requested history window.

') } - [void]$htmlBuilder.AppendLine('

Attribute coverage

') + [void]$htmlBuilder.AppendLine('

Mode filter coverage

') if ($modeEntries.Count -gt 0) { [void]$htmlBuilder.AppendLine('
    ') foreach ($mode in $modeEntries) { diff --git a/tools/VICategoryBuckets.psm1 b/tools/VICategoryBuckets.psm1 index 9284566c2..a13d8dbff 100644 --- a/tools/VICategoryBuckets.psm1 +++ b/tools/VICategoryBuckets.psm1 @@ -105,6 +105,7 @@ function Resolve-VICategorySlug { if ([string]::IsNullOrWhiteSpace($Name)) { return $null } $token = $Name.Trim().ToLowerInvariant() + if ($token -eq 'attributes') { return 'vi-attribute' } if ($script:CategoryDefinitions.ContainsKey($token)) { return $token } $normalizedToken = $token -replace '[-_]+', ' ' From e17aa99160686b294a4264601e0bdb926d55f7e5 Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Mon, 30 Mar 2026 19:16:04 -0700 Subject: [PATCH 06/44] Clarify VI history decision guidance (#2059) * Add VI history decision guidance * Prioritize VI history signal buckets * Separate VI history focus from context --------- Co-authored-by: svelderrainruiz --- ...parevi-tools-history-facade-v1.schema.json | 61 +++++++ tests/CompareVI.History.Tests.ps1 | 12 ++ tools/Render-VIHistoryReport.ps1 | 161 +++++++++++++++++- 3 files changed, 233 insertions(+), 1 deletion(-) diff --git a/docs/schemas/comparevi-tools-history-facade-v1.schema.json b/docs/schemas/comparevi-tools-history-facade-v1.schema.json index 0bd39b4ea..249ecd86d 100644 --- a/docs/schemas/comparevi-tools-history-facade-v1.schema.json +++ b/docs/schemas/comparevi-tools-history-facade-v1.schema.json @@ -10,6 +10,7 @@ "execution", "observedInterpretation", "summary", + "decisionGuidance", "reports", "modes" ], @@ -149,6 +150,66 @@ }, "additionalProperties": false }, + "decisionGuidance": { + "type": "object", + "required": [ + "reviewPriority", + "latestPair", + "signalPairs", + "collapsedPairs", + "focusBuckets", + "contextBuckets" + ], + "properties": { + "reviewPriority": { + "type": "string" + }, + "latestPair": { + "type": "object", + "required": [ + "index", + "status" + ], + "properties": { + "index": { + "type": "integer", + "minimum": 0 + }, + "status": { + "type": "string" + } + }, + "additionalProperties": false + }, + "signalPairs": { + "type": "array", + "items": { + "type": "integer", + "minimum": 0 + } + }, + "collapsedPairs": { + "type": "array", + "items": { + "type": "integer", + "minimum": 0 + } + }, + "focusBuckets": { + "type": "array", + "items": { + "type": "string" + } + }, + "contextBuckets": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false + }, "summary": { "type": "object", "required": [ diff --git a/tests/CompareVI.History.Tests.ps1 b/tests/CompareVI.History.Tests.ps1 index dd6d854a5..6d7e6946c 100644 --- a/tests/CompareVI.History.Tests.ps1 +++ b/tests/CompareVI.History.Tests.ps1 @@ -1445,6 +1445,12 @@ exit 0 $historySummary.observedInterpretation.coverageClass | Should -Be 'catalog-aligned' @($historySummary.execution.requestedModes) | Should -Be @('default') @($historySummary.execution.executedModes) | Should -Be @('default') + $historySummary.decisionGuidance.reviewPriority | Should -Be 'metadata-only-history' + @($historySummary.decisionGuidance.signalPairs) | Should -Be @() + @($historySummary.decisionGuidance.collapsedPairs) | Should -Be @(1) + $historySummary.decisionGuidance.latestPair.index | Should -Be 1 + $historySummary.decisionGuidance.latestPair.status | Should -Be 'collapsed-noise' + @($historySummary.decisionGuidance.PSObject.Properties.Name) | Should -Contain 'contextBuckets' $historyMd = Get-Content -LiteralPath (Join-Path $rd 'history-report.md') -Raw $historyMd | Should -Match 'Requested Modes: `default`' @@ -1455,6 +1461,9 @@ exit 0 $historyMd | Should -Match '\| Coverage Class \| `catalog-aligned` \|' $historyMd | Should -Match '## Mode overview' $historyMd | Should -Match '\| Mode \| Processed \| Diffs \| Signal \| Collapsed Noise \| Missing \| Categories \| Buckets \| Flags \|' + $historyMd | Should -Match '## Decision guidance' + $historyMd | Should -Match 'Review priority' + $historyMd | Should -Match 'Latest pair' $historyMd | Should -Match '## Commit pairs' $historyMd | Should -Match 'collapsed noise' $historyMd | Should -Match 'Touch history' @@ -1471,6 +1480,9 @@ exit 0 $historyHtml | Should -Match '

    Summary

    ' $historyHtml | Should -Match 'Signal' $historyHtml | Should -Match 'Collapsed Noise' + $historyHtml | Should -Match '

    Decision guidance

    ' + $historyHtml | Should -Match 'Review priority' + $historyHtml | Should -Match 'Latest pair' $historyHtml | Should -Match '

    Commit pairs

    ' $historyHtml | Should -Match 'Collapsed noise' $historyHtml | Should -Match 'Touch history' diff --git a/tools/Render-VIHistoryReport.ps1 b/tools/Render-VIHistoryReport.ps1 index f8de616a3..e1a7bc06d 100644 --- a/tools/Render-VIHistoryReport.ps1 +++ b/tools/Render-VIHistoryReport.ps1 @@ -664,12 +664,16 @@ function Get-BucketLabelEntries { foreach ($item in $items) { if ($null -eq $item) { continue } $slug = $null + $classification = $null if ($item -is [pscustomobject]) { if ($item.PSObject.Properties['bucketSlug']) { $slug = [string]$item.bucketSlug } elseif ($item.PSObject.Properties['slug']) { $slug = [string]$item.slug } + if ($item.PSObject.Properties['classification']) { + $classification = [string]$item.classification + } } else { $slug = [string]$item } @@ -680,7 +684,7 @@ function Get-BucketLabelEntries { $entries.Add([pscustomobject]@{ slug = $meta.slug label = $meta.label - classification = $meta.classification + classification = if ([string]::IsNullOrWhiteSpace($classification)) { $meta.classification } else { $classification } }) | Out-Null } } @@ -1203,6 +1207,126 @@ if ($modeEntries.Count -gt 0) { $stepSummaryLines = @($summaryLines) +$sortedComparisons = @( + $comparisons | + Sort-Object { + try { [int](Coalesce $_.index 0) } catch { 0 } + } +) +$latestComparisonEntry = if ($sortedComparisons.Count -gt 0) { $sortedComparisons[0] } else { $null } +$collapsedComparisonEntries = @( + $sortedComparisons | + Where-Object { + $_.result -and + $_.result.PSObject.Properties['collapsed'] -and + [bool]$_.result.collapsed + } +) +$signalComparisonEntries = @( + $sortedComparisons | + Where-Object { + $resultNode = $_.result + if (-not $resultNode) { return $false } + $hasDiff = $resultNode.PSObject.Properties['diff'] -and ($resultNode.diff -eq $true) + if (-not $hasDiff) { return $false } + if ($resultNode.PSObject.Properties['collapsed'] -and [bool]$resultNode.collapsed) { return $false } + $detailNodes = @() + if ($resultNode.PSObject.Properties['categoryDetails'] -and $resultNode.categoryDetails) { + $detailNodes = @($resultNode.categoryDetails) + } + if ($detailNodes.Count -eq 0) { return $true } + @($detailNodes | Where-Object { [string]$_.classification -ne 'noise' }).Count -gt 0 + } +) +$decisionFocusBuckets = New-Object System.Collections.Generic.List[string] +$decisionFocusBucketSet = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) +$decisionContextBuckets = New-Object System.Collections.Generic.List[string] +$decisionContextBucketSet = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) +foreach ($decisionEntry in $signalComparisonEntries) { + $decisionResultNode = $decisionEntry.result + if (-not $decisionResultNode) { continue } + $decisionBucketEntries = @() + if ($decisionResultNode.PSObject.Properties['categoryBucketDetails'] -and $decisionResultNode.categoryBucketDetails) { + $decisionBucketEntries = @(Get-BucketLabelEntries -Buckets $decisionResultNode.categoryBucketDetails) + } elseif ($decisionResultNode.PSObject.Properties['categoryBuckets'] -and $decisionResultNode.categoryBuckets) { + $decisionBucketEntries = @(Get-BucketLabelEntries -Buckets $decisionResultNode.categoryBuckets) + } + foreach ($decisionBucketEntry in $decisionBucketEntries) { + if ($null -eq $decisionBucketEntry) { continue } + $bucketLabel = [string]$decisionBucketEntry.label + if ([string]::IsNullOrWhiteSpace($bucketLabel)) { continue } + $bucketDisplayLabel = $bucketLabel + $isContextBucket = $false + switch ([string]$decisionBucketEntry.classification) { + 'noise' { + $bucketDisplayLabel = '{0} (noise)' -f $bucketLabel + if ($decisionContextBucketSet.Add($bucketDisplayLabel)) { + $decisionContextBuckets.Add($bucketDisplayLabel) | Out-Null + } + $isContextBucket = $true + break + } + 'neutral' { + $bucketDisplayLabel = '{0} (neutral)' -f $bucketLabel + if ($decisionContextBucketSet.Add($bucketDisplayLabel)) { + $decisionContextBuckets.Add($bucketDisplayLabel) | Out-Null + } + $isContextBucket = $true + break + } + } + if ($isContextBucket) { continue } + if ($decisionFocusBucketSet.Add($bucketDisplayLabel)) { + $decisionFocusBuckets.Add($bucketDisplayLabel) | Out-Null + } + } +} +$decisionReviewPriority = if ($signalComparisonEntries.Count -gt 0) { + 'review-signal-pairs' +} elseif ($collapsedComparisonEntries.Count -gt 0) { + 'metadata-only-history' +} else { + 'no-diff' +} +$decisionLatestStatus = 'n/a' +if ($latestComparisonEntry) { + $latestResultNode = $latestComparisonEntry.result + if ($latestResultNode -and $latestResultNode.PSObject.Properties['diff'] -and ($latestResultNode.diff -eq $true)) { + if ($latestResultNode.PSObject.Properties['collapsed'] -and [bool]$latestResultNode.collapsed) { + $decisionLatestStatus = 'collapsed-noise' + } else { + $decisionLatestStatus = 'signal-review' + } + } elseif ($latestResultNode -and $latestResultNode.PSObject.Properties['status'] -and $latestResultNode.status) { + $decisionLatestStatus = [string]$latestResultNode.status + } else { + $decisionLatestStatus = 'clean' + } +} +if ($sortedComparisons.Count -gt 0) { + $summaryLines.Add('') + $summaryLines.Add('## Decision guidance') + $summaryLines.Add('') + $summaryLines.Add(('- Review priority: `{0}`' -f $decisionReviewPriority)) + if ($latestComparisonEntry) { + $summaryLines.Add(('- Latest pair: `pair {0}` is `{1}`' -f (Coalesce $latestComparisonEntry.index 'n/a'), $decisionLatestStatus)) + } + if ($signalComparisonEntries.Count -gt 0) { + $summaryLines.Add(('- Signal pairs: `{0}`' -f ([string]::Join(', ', @($signalComparisonEntries | ForEach-Object { [string](Coalesce $_.index 'n/a') }))))) + } else { + $summaryLines.Add('- Signal pairs: `none`') + } + if ($collapsedComparisonEntries.Count -gt 0) { + $summaryLines.Add(('- Collapsed pairs: `{0}`' -f ([string]::Join(', ', @($collapsedComparisonEntries | ForEach-Object { [string](Coalesce $_.index 'n/a') }))))) + } + if ($decisionFocusBuckets.Count -gt 0) { + $summaryLines.Add(('- Focus buckets: `{0}`' -f ([string]::Join(', ', @($decisionFocusBuckets.ToArray()))))) + } + if ($decisionContextBuckets.Count -gt 0) { + $summaryLines.Add(('- Context buckets: `{0}`' -f ([string]::Join(', ', @($decisionContextBuckets.ToArray()))))) + } +} + $comparisonHtmlRows = New-Object System.Collections.Generic.List[object] if ($comparisons.Count -gt 0) { $summaryLines.Add('') @@ -1534,6 +1658,30 @@ if ($emitHtml -and $HtmlPath) { [void]$htmlBuilder.AppendLine(' ') } + if ($sortedComparisons.Count -gt 0) { + [void]$htmlBuilder.AppendLine('

    Decision guidance

    ') + [void]$htmlBuilder.AppendLine('
      ') + [void]$htmlBuilder.AppendLine(('
    • Review priority{0}
      • ' -f (Format-HtmlCodeList -Values @($decisionReviewPriority)))) + if ($latestComparisonEntry) { + [void]$htmlBuilder.AppendLine(('
    • Latest pairpair {0} is {1}
      • ' -f (ConvertTo-HtmlSafe (Coalesce $latestComparisonEntry.index 'n/a')), (ConvertTo-HtmlSafe $decisionLatestStatus))) + } + if ($signalComparisonEntries.Count -gt 0) { + [void]$htmlBuilder.AppendLine(('
    • Signal pairs{0}
      • ' -f (ConvertTo-HtmlSafe ([string]::Join(', ', @($signalComparisonEntries | ForEach-Object { [string](Coalesce $_.index 'n/a') })))))) + } else { + [void]$htmlBuilder.AppendLine('
    • Signal pairsnone
      • ') + } + if ($collapsedComparisonEntries.Count -gt 0) { + [void]$htmlBuilder.AppendLine(('
    • Collapsed pairs{0}
      • ' -f (ConvertTo-HtmlSafe ([string]::Join(', ', @($collapsedComparisonEntries | ForEach-Object { [string](Coalesce $_.index 'n/a') })))))) + } + if ($decisionFocusBuckets.Count -gt 0) { + [void]$htmlBuilder.AppendLine(('
    • Focus buckets{0}
      • ' -f (ConvertTo-HtmlSafe ([string]::Join(', ', @($decisionFocusBuckets.ToArray())))))) + } + if ($decisionContextBuckets.Count -gt 0) { + [void]$htmlBuilder.AppendLine(('
    • Context buckets{0}
      • ' -f (ConvertTo-HtmlSafe ([string]::Join(', ', @($decisionContextBuckets.ToArray())))))) + } + [void]$htmlBuilder.AppendLine('
    ') + } + [void]$htmlBuilder.AppendLine('

    Commit pairs

    ') if ($comparisonHtmlRows.Count -gt 0) { [void]$htmlBuilder.AppendLine(' ') @@ -1810,6 +1958,17 @@ $historySummary = [ordered]@{ modeSensitivity = [string]$modeSensitivity outcomeLabels = @(Get-SortedUniqueStringArray -Value $outcomeLabels) } + decisionGuidance = [ordered]@{ + reviewPriority = [string]$decisionReviewPriority + latestPair = [ordered]@{ + index = if ($latestComparisonEntry) { [int](Coalesce $latestComparisonEntry.index 0) } else { 0 } + status = [string]$decisionLatestStatus + } + signalPairs = @($signalComparisonEntries | ForEach-Object { [int](Coalesce $_.index 0) }) + collapsedPairs = @($collapsedComparisonEntries | ForEach-Object { [int](Coalesce $_.index 0) }) + focusBuckets = @($decisionFocusBuckets.ToArray()) + contextBuckets = @($decisionContextBuckets.ToArray()) + } summary = [ordered]@{ modes = if ($stats -and $stats.PSObject.Properties['modes']) { [int]$stats.modes } else { [int]$modeEntries.Count } comparisons = Get-IntPropertyValue -InputObject $stats -Name 'processed' From 6bce62f4e07f4f2b43401fd1396377d86c143b7a Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Mon, 30 Mar 2026 19:18:17 -0700 Subject: [PATCH 07/44] Make LV32 shadow proof non-blocking (#2060) Co-authored-by: svelderrainruiz --- .github/workflows/validate.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 61a2efc24..5f784c99b 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -1825,6 +1825,7 @@ jobs: needs: [smoke-gate, lint, session-index, session-index-v2-contract, vi-history-scenarios-plan, vi-history-scenarios-windows-lv32-plan] if: needs.smoke-gate.outputs.skip != 'true' && needs.vi-history-scenarios-plan.outputs.execute_lanes == 'true' && needs.vi-history-scenarios-windows-lv32-plan.outputs.available == 'true' runs-on: [self-hosted, Windows, X64, comparevi, capability-ingress, labview-2026, lv32] + continue-on-error: true timeout-minutes: 60 permissions: actions: read From a85d77af72049056ea3f4c0ffbfb17f16986cd6b Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Mon, 30 Mar 2026 19:30:35 -0700 Subject: [PATCH 08/44] Identify latest VI history signal pair (#2061) Co-authored-by: svelderrainruiz --- ...parevi-tools-history-facade-v1.schema.json | 22 +++++++++++++++++++ tests/CompareVI.History.Tests.ps1 | 3 +++ tools/Render-VIHistoryReport.ps1 | 16 ++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/docs/schemas/comparevi-tools-history-facade-v1.schema.json b/docs/schemas/comparevi-tools-history-facade-v1.schema.json index 249ecd86d..3ad2c207d 100644 --- a/docs/schemas/comparevi-tools-history-facade-v1.schema.json +++ b/docs/schemas/comparevi-tools-history-facade-v1.schema.json @@ -155,6 +155,7 @@ "required": [ "reviewPriority", "latestPair", + "latestSignalPair", "signalPairs", "collapsedPairs", "focusBuckets", @@ -181,6 +182,27 @@ }, "additionalProperties": false }, + "latestSignalPair": { + "type": "object", + "required": [ + "index", + "baseRef", + "headRef" + ], + "properties": { + "index": { + "type": "integer", + "minimum": 0 + }, + "baseRef": { + "type": "string" + }, + "headRef": { + "type": "string" + } + }, + "additionalProperties": false + }, "signalPairs": { "type": "array", "items": { diff --git a/tests/CompareVI.History.Tests.ps1 b/tests/CompareVI.History.Tests.ps1 index 6d7e6946c..ae008ec4d 100644 --- a/tests/CompareVI.History.Tests.ps1 +++ b/tests/CompareVI.History.Tests.ps1 @@ -1450,6 +1450,7 @@ exit 0 @($historySummary.decisionGuidance.collapsedPairs) | Should -Be @(1) $historySummary.decisionGuidance.latestPair.index | Should -Be 1 $historySummary.decisionGuidance.latestPair.status | Should -Be 'collapsed-noise' + $historySummary.decisionGuidance.latestSignalPair.index | Should -Be 0 @($historySummary.decisionGuidance.PSObject.Properties.Name) | Should -Contain 'contextBuckets' $historyMd = Get-Content -LiteralPath (Join-Path $rd 'history-report.md') -Raw @@ -1464,6 +1465,7 @@ exit 0 $historyMd | Should -Match '## Decision guidance' $historyMd | Should -Match 'Review priority' $historyMd | Should -Match 'Latest pair' + $historyMd | Should -Not -Match 'Review first' $historyMd | Should -Match '## Commit pairs' $historyMd | Should -Match 'collapsed noise' $historyMd | Should -Match 'Touch history' @@ -1483,6 +1485,7 @@ exit 0 $historyHtml | Should -Match '

    Decision guidance

    ' $historyHtml | Should -Match 'Review priority' $historyHtml | Should -Match 'Latest pair' + $historyHtml | Should -Not -Match 'Review first' $historyHtml | Should -Match '

    Commit pairs

    ' $historyHtml | Should -Match 'Collapsed noise' $historyHtml | Should -Match 'Touch history' diff --git a/tools/Render-VIHistoryReport.ps1 b/tools/Render-VIHistoryReport.ps1 index e1a7bc06d..059c02dcb 100644 --- a/tools/Render-VIHistoryReport.ps1 +++ b/tools/Render-VIHistoryReport.ps1 @@ -1288,6 +1288,7 @@ $decisionReviewPriority = if ($signalComparisonEntries.Count -gt 0) { } else { 'no-diff' } +$latestSignalComparisonEntry = if ($signalComparisonEntries.Count -gt 0) { $signalComparisonEntries[0] } else { $null } $decisionLatestStatus = 'n/a' if ($latestComparisonEntry) { $latestResultNode = $latestComparisonEntry.result @@ -1311,6 +1312,11 @@ if ($sortedComparisons.Count -gt 0) { if ($latestComparisonEntry) { $summaryLines.Add(('- Latest pair: `pair {0}` is `{1}`' -f (Coalesce $latestComparisonEntry.index 'n/a'), $decisionLatestStatus)) } + if ($latestSignalComparisonEntry) { + $latestSignalBaseRef = [string](Coalesce (Coalesce $latestSignalComparisonEntry.base.short $latestSignalComparisonEntry.base.full) 'n/a') + $latestSignalHeadRef = [string](Coalesce (Coalesce $latestSignalComparisonEntry.head.short $latestSignalComparisonEntry.head.full) 'n/a') + $summaryLines.Add(('- Review first: `pair {0}` (`{1}` -> `{2}`)' -f (Coalesce $latestSignalComparisonEntry.index 'n/a'), $latestSignalBaseRef, $latestSignalHeadRef)) + } if ($signalComparisonEntries.Count -gt 0) { $summaryLines.Add(('- Signal pairs: `{0}`' -f ([string]::Join(', ', @($signalComparisonEntries | ForEach-Object { [string](Coalesce $_.index 'n/a') }))))) } else { @@ -1665,6 +1671,11 @@ if ($emitHtml -and $HtmlPath) { if ($latestComparisonEntry) { [void]$htmlBuilder.AppendLine(('
  • Latest pairpair {0} is {1}
    • ' -f (ConvertTo-HtmlSafe (Coalesce $latestComparisonEntry.index 'n/a')), (ConvertTo-HtmlSafe $decisionLatestStatus))) } + if ($latestSignalComparisonEntry) { + $latestSignalBaseRef = [string](Coalesce (Coalesce $latestSignalComparisonEntry.base.short $latestSignalComparisonEntry.base.full) 'n/a') + $latestSignalHeadRef = [string](Coalesce (Coalesce $latestSignalComparisonEntry.head.short $latestSignalComparisonEntry.head.full) 'n/a') + [void]$htmlBuilder.AppendLine(('
  • Review firstpair {0} ({1} -> {2})
    • ' -f (ConvertTo-HtmlSafe (Coalesce $latestSignalComparisonEntry.index 'n/a')), (ConvertTo-HtmlSafe $latestSignalBaseRef), (ConvertTo-HtmlSafe $latestSignalHeadRef))) + } if ($signalComparisonEntries.Count -gt 0) { [void]$htmlBuilder.AppendLine(('
  • Signal pairs{0}
    • ' -f (ConvertTo-HtmlSafe ([string]::Join(', ', @($signalComparisonEntries | ForEach-Object { [string](Coalesce $_.index 'n/a') })))))) } else { @@ -1964,6 +1975,11 @@ $historySummary = [ordered]@{ index = if ($latestComparisonEntry) { [int](Coalesce $latestComparisonEntry.index 0) } else { 0 } status = [string]$decisionLatestStatus } + latestSignalPair = [ordered]@{ + index = if ($latestSignalComparisonEntry) { [int](Coalesce $latestSignalComparisonEntry.index 0) } else { 0 } + baseRef = if ($latestSignalComparisonEntry) { [string](Coalesce (Coalesce $latestSignalComparisonEntry.base.short $latestSignalComparisonEntry.base.full) '') } else { '' } + headRef = if ($latestSignalComparisonEntry) { [string](Coalesce (Coalesce $latestSignalComparisonEntry.head.short $latestSignalComparisonEntry.head.full) '') } else { '' } + } signalPairs = @($signalComparisonEntries | ForEach-Object { [int](Coalesce $_.index 0) }) collapsedPairs = @($collapsedComparisonEntries | ForEach-Object { [int](Coalesce $_.index 0) }) focusBuckets = @($decisionFocusBuckets.ToArray()) From 6a91ca561a544e94d6cf0ea54e10a5698795f1fb Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Mon, 30 Mar 2026 19:43:43 -0700 Subject: [PATCH 09/44] Add VI history review sequence guidance (#2062) Co-authored-by: svelderrainruiz --- ...parevi-tools-history-facade-v1.schema.json | 42 ++++++++++++++++++- tests/Render-VIHistoryReport.Tests.ps1 | 14 +++++++ tools/Render-VIHistoryReport.ps1 | 42 ++++++++++++++++++- 3 files changed, 96 insertions(+), 2 deletions(-) diff --git a/docs/schemas/comparevi-tools-history-facade-v1.schema.json b/docs/schemas/comparevi-tools-history-facade-v1.schema.json index 3ad2c207d..817c66f81 100644 --- a/docs/schemas/comparevi-tools-history-facade-v1.schema.json +++ b/docs/schemas/comparevi-tools-history-facade-v1.schema.json @@ -156,6 +156,7 @@ "reviewPriority", "latestPair", "latestSignalPair", + "reviewSequence", "signalPairs", "collapsedPairs", "focusBuckets", @@ -187,7 +188,9 @@ "required": [ "index", "baseRef", - "headRef" + "headRef", + "baseSubject", + "headSubject" ], "properties": { "index": { @@ -199,10 +202,47 @@ }, "headRef": { "type": "string" + }, + "baseSubject": { + "type": "string" + }, + "headSubject": { + "type": "string" } }, "additionalProperties": false }, + "reviewSequence": { + "type": "array", + "items": { + "type": "object", + "required": [ + "index", + "baseRef", + "headRef", + "baseSubject", + "headSubject" + ], + "properties": { + "index": { + "type": "integer", + "minimum": 0 + }, + "baseRef": { + "type": "string" + }, + "headRef": { + "type": "string" + }, + "baseSubject": { + "type": "string" + }, + "headSubject": { + "type": "string" + } + } + } + }, "signalPairs": { "type": "array", "items": { diff --git a/tests/Render-VIHistoryReport.Tests.ps1 b/tests/Render-VIHistoryReport.Tests.ps1 index d1273d82a..7ceecbc0b 100644 --- a/tests/Render-VIHistoryReport.Tests.ps1 +++ b/tests/Render-VIHistoryReport.Tests.ps1 @@ -216,6 +216,10 @@ Describe 'Render-VIHistoryReport.ps1' -Tag 'Unit' { $markdown | Should -Match '\| Coverage Class \| `catalog-partial` \|' $markdown | Should -Match '\| Mode Sensitivity \| `single-mode-observed` \|' $markdown | Should -Match '\| Outcome Labels \| `clean`, `signal-diff` \|' + $markdown | Should -Match '## Decision guidance' + $markdown | Should -Match 'Review first' + $markdown | Should -Match 'Review sequence' + $markdown | Should -Match 'pair 2 \(Base commit -> Head commit\)' $markdown | Should -Match '\| Mode \| Processed \| Diffs \| Signal \| Collapsed Noise \| Missing \| Categories \| Buckets \| Flags \|' $markdown | Should -Match '\| Mode \| Pair \| Lineage \| Base \| Head \| Diff \| Duration \(s\) \| Categories \| Buckets \| Report \| Highlights \|' $markdown | Should -Match 'Touch history' @@ -234,6 +238,9 @@ Describe 'Render-VIHistoryReport.ps1' -Tag 'Unit' { $html | Should -Match 'single-mode-observed' $html | Should -Match 'Outcome Labels' $html | Should -Match 'clean, signal-diff' + $html | Should -Match 'Review first' + $html | Should -Match 'Review sequence' + $html | Should -Match 'pair 2 \(Base commit -> Head commit\)' $html | Should -Match '    ' $html | Should -Match '' $html | Should -Match '' @@ -264,6 +271,13 @@ Describe 'Render-VIHistoryReport.ps1' -Tag 'Unit' { $historySummary.target.sourceBranchRef | Should -Be 'feature/history-source' $historySummary.target.branchBudget.maxCommitCount | Should -Be 64 $historySummary.target.branchBudget.commitCount | Should -Be 3 + $historySummary.decisionGuidance.latestSignalPair.index | Should -Be 2 + $historySummary.decisionGuidance.latestSignalPair.baseSubject | Should -Be 'Base commit' + $historySummary.decisionGuidance.latestSignalPair.headSubject | Should -Be 'Head commit' + @($historySummary.decisionGuidance.reviewSequence).Count | Should -Be 1 + $historySummary.decisionGuidance.reviewSequence[0].index | Should -Be 2 + $historySummary.decisionGuidance.reviewSequence[0].baseSubject | Should -Be 'Base commit' + $historySummary.decisionGuidance.reviewSequence[0].headSubject | Should -Be 'Head commit' } It 'preserves branch budget numeric fields when the source object is a hashtable' { diff --git a/tools/Render-VIHistoryReport.ps1 b/tools/Render-VIHistoryReport.ps1 index 059c02dcb..719a3f5bd 100644 --- a/tools/Render-VIHistoryReport.ps1 +++ b/tools/Render-VIHistoryReport.ps1 @@ -1288,6 +1288,17 @@ $decisionReviewPriority = if ($signalComparisonEntries.Count -gt 0) { } else { 'no-diff' } +$decisionReviewSequence = @( + $signalComparisonEntries | ForEach-Object { + [ordered]@{ + index = [int](Coalesce $_.index 0) + baseRef = [string](Coalesce (Coalesce $_.base.short $_.base.full) '') + headRef = [string](Coalesce (Coalesce $_.head.short $_.head.full) '') + baseSubject = [string](Coalesce $_.base.subject '') + headSubject = [string](Coalesce $_.head.subject '') + } + } +) $latestSignalComparisonEntry = if ($signalComparisonEntries.Count -gt 0) { $signalComparisonEntries[0] } else { $null } $decisionLatestStatus = 'n/a' if ($latestComparisonEntry) { @@ -1317,6 +1328,14 @@ if ($sortedComparisons.Count -gt 0) { $latestSignalHeadRef = [string](Coalesce (Coalesce $latestSignalComparisonEntry.head.short $latestSignalComparisonEntry.head.full) 'n/a') $summaryLines.Add(('- Review first: `pair {0}` (`{1}` -> `{2}`)' -f (Coalesce $latestSignalComparisonEntry.index 'n/a'), $latestSignalBaseRef, $latestSignalHeadRef)) } + if ($decisionReviewSequence.Count -gt 0) { + $reviewSequenceLabels = @( + $decisionReviewSequence | ForEach-Object { + 'pair {0} ({1} -> {2})' -f $_.index, (Coalesce $_.baseSubject 'n/a'), (Coalesce $_.headSubject 'n/a') + } + ) + $summaryLines.Add(('- Review sequence: `{0}`' -f ([string]::Join('; ', $reviewSequenceLabels)))) + } if ($signalComparisonEntries.Count -gt 0) { $summaryLines.Add(('- Signal pairs: `{0}`' -f ([string]::Join(', ', @($signalComparisonEntries | ForEach-Object { [string](Coalesce $_.index 'n/a') }))))) } else { @@ -1676,6 +1695,14 @@ if ($emitHtml -and $HtmlPath) { $latestSignalHeadRef = [string](Coalesce (Coalesce $latestSignalComparisonEntry.head.short $latestSignalComparisonEntry.head.full) 'n/a') [void]$htmlBuilder.AppendLine(('
  • Review firstpair {0} ({1} -> {2})
    • ' -f (ConvertTo-HtmlSafe (Coalesce $latestSignalComparisonEntry.index 'n/a')), (ConvertTo-HtmlSafe $latestSignalBaseRef), (ConvertTo-HtmlSafe $latestSignalHeadRef))) } + if ($decisionReviewSequence.Count -gt 0) { + $reviewSequenceLabels = @( + $decisionReviewSequence | ForEach-Object { + 'pair {0} ({1} -> {2})' -f $_.index, (Coalesce $_.baseSubject 'n/a'), (Coalesce $_.headSubject 'n/a') + } + ) + [void]$htmlBuilder.AppendLine(('
  • Review sequence{0}
    • ' -f (ConvertTo-HtmlSafe ([string]::Join('; ', $reviewSequenceLabels))))) + } if ($signalComparisonEntries.Count -gt 0) { [void]$htmlBuilder.AppendLine(('
  • Signal pairs{0}
    • ' -f (ConvertTo-HtmlSafe ([string]::Join(', ', @($signalComparisonEntries | ForEach-Object { [string](Coalesce $_.index 'n/a') })))))) } else { @@ -1979,7 +2006,20 @@ $historySummary = [ordered]@{ index = if ($latestSignalComparisonEntry) { [int](Coalesce $latestSignalComparisonEntry.index 0) } else { 0 } baseRef = if ($latestSignalComparisonEntry) { [string](Coalesce (Coalesce $latestSignalComparisonEntry.base.short $latestSignalComparisonEntry.base.full) '') } else { '' } headRef = if ($latestSignalComparisonEntry) { [string](Coalesce (Coalesce $latestSignalComparisonEntry.head.short $latestSignalComparisonEntry.head.full) '') } else { '' } - } + baseSubject = if ($latestSignalComparisonEntry) { [string](Coalesce $latestSignalComparisonEntry.base.subject '') } else { '' } + headSubject = if ($latestSignalComparisonEntry) { [string](Coalesce $latestSignalComparisonEntry.head.subject '') } else { '' } + } + reviewSequence = @( + $decisionReviewSequence | ForEach-Object { + [ordered]@{ + index = [int](Coalesce $_.index 0) + baseRef = [string](Coalesce $_.baseRef '') + headRef = [string](Coalesce $_.headRef '') + baseSubject = [string](Coalesce $_.baseSubject '') + headSubject = [string](Coalesce $_.headSubject '') + } + } + ) signalPairs = @($signalComparisonEntries | ForEach-Object { [int](Coalesce $_.index 0) }) collapsedPairs = @($collapsedComparisonEntries | ForEach-Object { [int](Coalesce $_.index 0) }) focusBuckets = @($decisionFocusBuckets.ToArray()) From 0e99c52b0705982eff7ea483b09e11d8c18efcbb Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Mon, 30 Mar 2026 20:41:49 -0700 Subject: [PATCH 10/44] Expose VI history decision chronology (#2063) * Add VI history decision statement * Expose VI history decision chronology --------- Co-authored-by: svelderrainruiz --- README.md | 6 +- docs/knowledgebase/VICompare-Refs-Workflow.md | 3 + ...parevi-tools-history-facade-v1.schema.json | 50 ++++- tests/CompareVI.History.Tests.ps1 | 21 +++ tests/Render-VIHistoryReport.Tests.ps1 | 24 ++- tools/Render-VIHistoryReport.ps1 | 173 +++++++++++++++--- 6 files changed, 242 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index ef942cc3f..492e826b2 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,11 @@ renders, `history-report-html`). Downstream workflows and reusable snippets can consume those keys to surface the Markdown/HTML report or to dispatch follow-up automation without spelunking the artifacts. When the renderer is unavailable, `Compare-VIHistory.ps1` writes a lightweight fallback report so the Markdown -output key always resolves to a readable summary. +output key always resolves to a readable summary. The stable +`history-summary-json` facade and rendered history report now also surface the +decision statement, the newest meaningful pair, and pair chronology (refs, +subjects, and commit dates) so reviewers can answer what changed, when it +changed, and whether it matters without opening raw manifests first. Provide the optional `notify_issue` input when dispatching the workflow to post the same summary table to a GitHub issue for stakeholders. diff --git a/docs/knowledgebase/VICompare-Refs-Workflow.md b/docs/knowledgebase/VICompare-Refs-Workflow.md index e2a6bce2f..41a7f39a5 100644 --- a/docs/knowledgebase/VICompare-Refs-Workflow.md +++ b/docs/knowledgebase/VICompare-Refs-Workflow.md @@ -220,6 +220,9 @@ gh workflow run vi-compare-refs.yml ` `Invoke-CompareVIHistoryFacade`. When HTML rendering is skipped or fails, the Markdown path still points at the fallback report so consumers always have a summary to ingest. A compressed `category-counts-json` blob is also published so downstream automation can react to runs dominated by cosmetic noise without re-reading the manifests. + The facade's `decisionGuidance` block now carries a human-readable `decisionStatement`, the newest surfaced pair, + and the review sequence with refs, subjects, and commit dates so downstream consumers can answer what changed, when, + and whether it matters from one stabilized payload. - History summary JSON (`tests/results/pr-vi-history/vi-history-summary.json`) now adds: - `targets[].reportImages` for per-target extraction metadata. - `pairTimeline[]` with additive per-pair contract fields: diff --git a/docs/schemas/comparevi-tools-history-facade-v1.schema.json b/docs/schemas/comparevi-tools-history-facade-v1.schema.json index 817c66f81..657069432 100644 --- a/docs/schemas/comparevi-tools-history-facade-v1.schema.json +++ b/docs/schemas/comparevi-tools-history-facade-v1.schema.json @@ -153,6 +153,7 @@ "decisionGuidance": { "type": "object", "required": [ + "decisionStatement", "reviewPriority", "latestPair", "latestSignalPair", @@ -163,6 +164,9 @@ "contextBuckets" ], "properties": { + "decisionStatement": { + "type": "string" + }, "reviewPriority": { "type": "string" }, @@ -170,7 +174,13 @@ "type": "object", "required": [ "index", - "status" + "status", + "baseRef", + "headRef", + "baseSubject", + "headSubject", + "baseDate", + "headDate" ], "properties": { "index": { @@ -179,6 +189,24 @@ }, "status": { "type": "string" + }, + "baseRef": { + "type": "string" + }, + "headRef": { + "type": "string" + }, + "baseSubject": { + "type": "string" + }, + "headSubject": { + "type": "string" + }, + "baseDate": { + "type": "string" + }, + "headDate": { + "type": "string" } }, "additionalProperties": false @@ -190,7 +218,9 @@ "baseRef", "headRef", "baseSubject", - "headSubject" + "headSubject", + "baseDate", + "headDate" ], "properties": { "index": { @@ -208,6 +238,12 @@ }, "headSubject": { "type": "string" + }, + "baseDate": { + "type": "string" + }, + "headDate": { + "type": "string" } }, "additionalProperties": false @@ -221,7 +257,9 @@ "baseRef", "headRef", "baseSubject", - "headSubject" + "headSubject", + "baseDate", + "headDate" ], "properties": { "index": { @@ -239,6 +277,12 @@ }, "headSubject": { "type": "string" + }, + "baseDate": { + "type": "string" + }, + "headDate": { + "type": "string" } } } diff --git a/tests/CompareVI.History.Tests.ps1 b/tests/CompareVI.History.Tests.ps1 index ae008ec4d..b250fd35d 100644 --- a/tests/CompareVI.History.Tests.ps1 +++ b/tests/CompareVI.History.Tests.ps1 @@ -574,6 +574,27 @@ exit 0 $aggregate.stats.signalDiffs | Should -Be 1 $aggregate.stats.noiseCollapsed | Should -Be 0 + + $historySummaryPath = Join-Path $rd 'history-summary.json' + Test-Path -LiteralPath $historySummaryPath | Should -BeTrue + $historySummary = Get-Content -LiteralPath $historySummaryPath -Raw | ConvertFrom-Json -Depth 12 + $historySummary.decisionGuidance.decisionStatement | Should -Match '^Start at pair 1; it is the newest meaningful change\. It touches ' + $historySummary.decisionGuidance.latestPair.status | Should -Be 'signal-review' + $historySummary.decisionGuidance.latestPair.baseDate | Should -Not -BeNullOrEmpty + $historySummary.decisionGuidance.latestPair.headDate | Should -Not -BeNullOrEmpty + $historySummary.decisionGuidance.latestSignalPair.index | Should -Be 1 + $historySummary.decisionGuidance.latestSignalPair.baseDate | Should -Not -BeNullOrEmpty + $historySummary.decisionGuidance.latestSignalPair.headDate | Should -Not -BeNullOrEmpty + @($historySummary.decisionGuidance.reviewSequence).Count | Should -Be 1 + $historySummary.decisionGuidance.reviewSequence[0].baseDate | Should -Not -BeNullOrEmpty + $historySummary.decisionGuidance.reviewSequence[0].headDate | Should -Not -BeNullOrEmpty + + $historyMd = Get-Content -LiteralPath (Join-Path $rd 'history-report.md') -Raw + $historyMd | Should -Match 'Decision statement' + $historyMd | Should -Match 'Review first' + $historyMd | Should -Match 'Review sequence' + $historyMd | Should -Match 'pair 1 \(' + $historyMd | Should -Match 'T\d{2}:\d{2}:\d{2}' } finally { if ($null -eq $previousDiff) { Remove-Item Env:STUB_COMPARE_DIFF -ErrorAction SilentlyContinue diff --git a/tests/Render-VIHistoryReport.Tests.ps1 b/tests/Render-VIHistoryReport.Tests.ps1 index 7ceecbc0b..d05351cd2 100644 --- a/tests/Render-VIHistoryReport.Tests.ps1 +++ b/tests/Render-VIHistoryReport.Tests.ps1 @@ -108,11 +108,13 @@ Describe 'Render-VIHistoryReport.ps1' -Tag 'Unit' { full = 'aaa111111111' short = 'aaa1111' subject= 'Clean base commit' + date = '2026-03-09T09:00:00Z' } head = @{ full = 'bbb222222222' short = 'bbb2222' subject= 'Clean head commit' + date = '2026-03-10T10:00:00Z' } lineage = [ordered]@{ type = 'touch-history' @@ -142,11 +144,13 @@ Describe 'Render-VIHistoryReport.ps1' -Tag 'Unit' { full = 'abc123456789' short = 'abc1234' subject= 'Base commit' + date = '2026-03-11T11:00:00Z' } head = @{ full = 'def987654321' short = 'def9876' subject= 'Head commit' + date = '2026-03-12T12:00:00Z' } lineage = [ordered]@{ type = 'merge-parent' @@ -217,11 +221,15 @@ Describe 'Render-VIHistoryReport.ps1' -Tag 'Unit' { $markdown | Should -Match '\| Mode Sensitivity \| `single-mode-observed` \|' $markdown | Should -Match '\| Outcome Labels \| `clean`, `signal-diff` \|' $markdown | Should -Match '## Decision guidance' + $markdown | Should -Match 'Decision statement' + $markdown | Should -Match 'Start at pair 2; it is the newest meaningful change' $markdown | Should -Match 'Review first' $markdown | Should -Match 'Review sequence' - $markdown | Should -Match 'pair 2 \(Base commit -> Head commit\)' + $markdown | Should -Match 'pair 2 \(abc1234 \(Base commit; 2026-03-11T11:00:00\+00:00\) -> def9876 \(Head commit; 2026-03-12T12:00:00\+00:00\)\)' + $markdown | Should -Match 'pair 2 \(Base commit @ 2026-03-11T11:00:00\+00:00 -> Head commit @ 2026-03-12T12:00:00\+00:00\)' $markdown | Should -Match '\| Mode \| Processed \| Diffs \| Signal \| Collapsed Noise \| Missing \| Categories \| Buckets \| Flags \|' $markdown | Should -Match '\| Mode \| Pair \| Lineage \| Base \| Head \| Diff \| Duration \(s\) \| Categories \| Buckets \| Report \| Highlights \|' + $markdown | Should -Match 'aaa1111 \(Clean base commit; 2026-03-09T09:00:00\+00:00\)' $markdown | Should -Match 'Touch history' $html = Get-Content -LiteralPath $htmlPath -Raw @@ -239,8 +247,11 @@ Describe 'Render-VIHistoryReport.ps1' -Tag 'Unit' { $html | Should -Match 'Outcome Labels' $html | Should -Match 'clean, signal-diff' $html | Should -Match 'Review first' + $html | Should -Match 'Decision statement' + $html | Should -Match 'Start at pair 2; it is the newest meaningful change' $html | Should -Match 'Review sequence' - $html | Should -Match 'pair 2 \(Base commit -> Head commit\)' + $html | Should -Match 'pair 2 \(abc1234 \(Base commit; 2026-03-11T11:00:00\+00:00\) -> def9876 \(Head commit; 2026-03-12T12:00:00\+00:00\)\)' + $html | Should -Match 'pair 2 \(Base commit @ 2026-03-11T11:00:00\+00:00 -> Head commit @ 2026-03-12T12:00:00\+00:00\)' $html | Should -Match '' $html | Should -Match '' $html | Should -Match '' @@ -267,17 +278,24 @@ Describe 'Render-VIHistoryReport.ps1' -Tag 'Unit' { $historySummaryLine | Should -Not -BeNullOrEmpty $historySummaryPath = (($historySummaryLine -split '=', 2)[1]).Trim() Test-Path -LiteralPath $historySummaryPath | Should -BeTrue - $historySummary = Get-Content -LiteralPath $historySummaryPath -Raw | ConvertFrom-Json -Depth 12 + $historySummary = Get-Content -LiteralPath $historySummaryPath -Raw | ConvertFrom-Json -Depth 12 -DateKind String $historySummary.target.sourceBranchRef | Should -Be 'feature/history-source' $historySummary.target.branchBudget.maxCommitCount | Should -Be 64 $historySummary.target.branchBudget.commitCount | Should -Be 3 + $historySummary.decisionGuidance.latestPair.baseDate | Should -Be '2026-03-09T09:00:00+00:00' + $historySummary.decisionGuidance.latestPair.headDate | Should -Be '2026-03-10T10:00:00+00:00' + $historySummary.decisionGuidance.decisionStatement | Should -Be 'Start at pair 2; it is the newest meaningful change. It touches Functional behavior.' $historySummary.decisionGuidance.latestSignalPair.index | Should -Be 2 $historySummary.decisionGuidance.latestSignalPair.baseSubject | Should -Be 'Base commit' $historySummary.decisionGuidance.latestSignalPair.headSubject | Should -Be 'Head commit' + $historySummary.decisionGuidance.latestSignalPair.baseDate | Should -Be '2026-03-11T11:00:00+00:00' + $historySummary.decisionGuidance.latestSignalPair.headDate | Should -Be '2026-03-12T12:00:00+00:00' @($historySummary.decisionGuidance.reviewSequence).Count | Should -Be 1 $historySummary.decisionGuidance.reviewSequence[0].index | Should -Be 2 $historySummary.decisionGuidance.reviewSequence[0].baseSubject | Should -Be 'Base commit' $historySummary.decisionGuidance.reviewSequence[0].headSubject | Should -Be 'Head commit' + $historySummary.decisionGuidance.reviewSequence[0].baseDate | Should -Be '2026-03-11T11:00:00+00:00' + $historySummary.decisionGuidance.reviewSequence[0].headDate | Should -Be '2026-03-12T12:00:00+00:00' } It 'preserves branch budget numeric fields when the source object is a hashtable' { diff --git a/tools/Render-VIHistoryReport.ps1 b/tools/Render-VIHistoryReport.ps1 index 719a3f5bd..8dcacd2f9 100644 --- a/tools/Render-VIHistoryReport.ps1 +++ b/tools/Render-VIHistoryReport.ps1 @@ -106,6 +106,102 @@ function Get-ShortSha { return $Value.Substring(0, $Length) } +function Format-DecisionTimestamp { + param($Value) + + if ($null -eq $Value) { return '' } + + if ($Value -is [DateTimeOffset]) { + return $Value.ToString('yyyy-MM-ddTHH:mm:ssK') + } + + if ($Value -is [DateTime]) { + return ([DateTimeOffset]$Value).ToString('yyyy-MM-ddTHH:mm:ssK') + } + + $text = [string]$Value + if ([string]::IsNullOrWhiteSpace($text)) { return '' } + + try { + $parsed = [DateTimeOffset]::Parse($text, [System.Globalization.CultureInfo]::InvariantCulture, [System.Globalization.DateTimeStyles]::RoundtripKind) + return $parsed.ToString('yyyy-MM-ddTHH:mm:ssK') + } catch { + return $text + } +} + +function New-DecisionPairMetadata { + param([AllowNull()][object]$Entry) + + if (-not $Entry) { + return [ordered]@{ + index = 0 + baseRef = '' + headRef = '' + baseSubject = '' + headSubject = '' + baseDate = '' + headDate = '' + } + } + + return [ordered]@{ + index = [int](Coalesce $Entry.index 0) + baseRef = [string](Coalesce (Coalesce $Entry.base.short $Entry.base.full) '') + headRef = [string](Coalesce (Coalesce $Entry.head.short $Entry.head.full) '') + baseSubject = [string](Coalesce $Entry.base.subject '') + headSubject = [string](Coalesce $Entry.head.subject '') + baseDate = [string](Format-DecisionTimestamp -Value $Entry.base.date) + headDate = [string](Format-DecisionTimestamp -Value $Entry.head.date) + } +} + +function Format-DecisionCommitLabel { + param( + [string]$Ref, + [string]$Subject, + [string]$Date + ) + + $label = [string](Coalesce $Ref 'n/a') + $details = New-Object System.Collections.Generic.List[string] + if (-not [string]::IsNullOrWhiteSpace($Subject)) { + $details.Add($Subject) | Out-Null + } + if (-not [string]::IsNullOrWhiteSpace($Date)) { + $details.Add($Date) | Out-Null + } + if ($details.Count -gt 0) { + $label = '{0} ({1})' -f $label, ([string]::Join('; ', @($details.ToArray()))) + } + return $label +} + +function Format-DecisionPairTimelineLabel { + param( + [AllowNull()][object]$Pair, + [switch]$UseSubjectsOnly + ) + + if (-not $Pair) { return 'pair n/a' } + + if ($UseSubjectsOnly.IsPresent) { + $baseLabel = if (-not [string]::IsNullOrWhiteSpace([string]$Pair.baseSubject)) { [string]$Pair.baseSubject } else { [string](Coalesce $Pair.baseRef 'n/a') } + if (-not [string]::IsNullOrWhiteSpace([string]$Pair.baseDate)) { + $baseLabel = '{0} @ {1}' -f $baseLabel, [string]$Pair.baseDate + } + $headLabel = if (-not [string]::IsNullOrWhiteSpace([string]$Pair.headSubject)) { [string]$Pair.headSubject } else { [string](Coalesce $Pair.headRef 'n/a') } + if (-not [string]::IsNullOrWhiteSpace([string]$Pair.headDate)) { + $headLabel = '{0} @ {1}' -f $headLabel, [string]$Pair.headDate + } + } else { + $baseLabel = Format-DecisionCommitLabel -Ref ([string](Coalesce $Pair.baseRef 'n/a')) -Subject ([string](Coalesce $Pair.baseSubject '')) -Date ([string](Coalesce $Pair.baseDate '')) + $headLabel = Format-DecisionCommitLabel -Ref ([string](Coalesce $Pair.headRef 'n/a')) -Subject ([string](Coalesce $Pair.headSubject '')) -Date ([string](Coalesce $Pair.headDate '')) + } + + return 'pair {0} ({1} -> {2})' -f (Coalesce $Pair.index 'n/a'), $baseLabel, $headLabel +} + function Get-LineageLabel { param( [object]$Lineage, @@ -1288,18 +1384,14 @@ $decisionReviewPriority = if ($signalComparisonEntries.Count -gt 0) { } else { 'no-diff' } +$decisionLatestPair = New-DecisionPairMetadata -Entry $latestComparisonEntry +$latestSignalComparisonEntry = if ($signalComparisonEntries.Count -gt 0) { $signalComparisonEntries[0] } else { $null } +$decisionLatestSignalPair = New-DecisionPairMetadata -Entry $latestSignalComparisonEntry $decisionReviewSequence = @( $signalComparisonEntries | ForEach-Object { - [ordered]@{ - index = [int](Coalesce $_.index 0) - baseRef = [string](Coalesce (Coalesce $_.base.short $_.base.full) '') - headRef = [string](Coalesce (Coalesce $_.head.short $_.head.full) '') - baseSubject = [string](Coalesce $_.base.subject '') - headSubject = [string](Coalesce $_.head.subject '') - } + New-DecisionPairMetadata -Entry $_ } ) -$latestSignalComparisonEntry = if ($signalComparisonEntries.Count -gt 0) { $signalComparisonEntries[0] } else { $null } $decisionLatestStatus = 'n/a' if ($latestComparisonEntry) { $latestResultNode = $latestComparisonEntry.result @@ -1315,23 +1407,40 @@ if ($latestComparisonEntry) { $decisionLatestStatus = 'clean' } } +$decisionStatement = '' +if ($latestComparisonEntry -and $decisionLatestStatus -eq 'collapsed-noise' -and $latestSignalComparisonEntry) { + $decisionStatement = 'Newest VI touch is metadata-only.' + if ($decisionFocusBuckets.Count -gt 0) { + $decisionStatement = '{0} Start at pair {1}; newest meaningful change touches {2}.' -f $decisionStatement, (Coalesce $latestSignalComparisonEntry.index 'n/a'), ([string]::Join(', ', @($decisionFocusBuckets.ToArray()))) + } else { + $decisionStatement = '{0} Start at pair {1}; it is the newest meaningful change.' -f $decisionStatement, (Coalesce $latestSignalComparisonEntry.index 'n/a') + } +} elseif ($latestSignalComparisonEntry) { + $decisionStatement = 'Start at pair {0}; it is the newest meaningful change.' -f (Coalesce $latestSignalComparisonEntry.index 'n/a') + if ($decisionFocusBuckets.Count -gt 0) { + $decisionStatement = '{0} It touches {1}.' -f $decisionStatement, ([string]::Join(', ', @($decisionFocusBuckets.ToArray()))) + } +} elseif ($collapsedComparisonEntries.Count -gt 0) { + $decisionStatement = 'All observed pairs are metadata-only under the current noise policy.' +} else { + $decisionStatement = 'No meaningful differences were observed in the selected history window.' +} if ($sortedComparisons.Count -gt 0) { $summaryLines.Add('') $summaryLines.Add('## Decision guidance') $summaryLines.Add('') + $summaryLines.Add(('- Decision statement: {0}' -f $decisionStatement)) $summaryLines.Add(('- Review priority: `{0}`' -f $decisionReviewPriority)) if ($latestComparisonEntry) { - $summaryLines.Add(('- Latest pair: `pair {0}` is `{1}`' -f (Coalesce $latestComparisonEntry.index 'n/a'), $decisionLatestStatus)) + $summaryLines.Add(('- Latest pair: `{0}` is `{1}`' -f (Format-DecisionPairTimelineLabel -Pair $decisionLatestPair), $decisionLatestStatus)) } if ($latestSignalComparisonEntry) { - $latestSignalBaseRef = [string](Coalesce (Coalesce $latestSignalComparisonEntry.base.short $latestSignalComparisonEntry.base.full) 'n/a') - $latestSignalHeadRef = [string](Coalesce (Coalesce $latestSignalComparisonEntry.head.short $latestSignalComparisonEntry.head.full) 'n/a') - $summaryLines.Add(('- Review first: `pair {0}` (`{1}` -> `{2}`)' -f (Coalesce $latestSignalComparisonEntry.index 'n/a'), $latestSignalBaseRef, $latestSignalHeadRef)) + $summaryLines.Add(('- Review first: `{0}`' -f (Format-DecisionPairTimelineLabel -Pair $decisionLatestSignalPair))) } if ($decisionReviewSequence.Count -gt 0) { $reviewSequenceLabels = @( $decisionReviewSequence | ForEach-Object { - 'pair {0} ({1} -> {2})' -f $_.index, (Coalesce $_.baseSubject 'n/a'), (Coalesce $_.headSubject 'n/a') + Format-DecisionPairTimelineLabel -Pair $_ -UseSubjectsOnly } ) $summaryLines.Add(('- Review sequence: `{0}`' -f ([string]::Join('; ', $reviewSequenceLabels)))) @@ -1372,10 +1481,8 @@ if ($comparisons.Count -gt 0) { } if ([string]::IsNullOrWhiteSpace($lineageLabel)) { $lineageLabel = 'Mainline' } - $baseRef = Coalesce $entry.base.short $entry.base.full - if ($entry.base.subject) { $baseRef = '{0} ({1})' -f $baseRef, $entry.base.subject } - $headRef = Coalesce $entry.head.short $entry.head.full - if ($entry.head.subject) { $headRef = '{0} ({1})' -f $headRef, $entry.head.subject } + $baseRef = Format-DecisionCommitLabel -Ref ([string](Coalesce $entry.base.short $entry.base.full)) -Subject ([string](Coalesce $entry.base.subject '')) -Date (Format-DecisionTimestamp -Value $entry.base.date) + $headRef = Format-DecisionCommitLabel -Ref ([string](Coalesce $entry.head.short $entry.head.full)) -Subject ([string](Coalesce $entry.head.subject '')) -Date (Format-DecisionTimestamp -Value $entry.head.date) $resultNode = $entry.result $hasDiffValue = $resultNode -and $resultNode.PSObject.Properties['diff'] $diffValue = $hasDiffValue -and ($resultNode.diff -eq $true) @@ -1686,19 +1793,18 @@ if ($emitHtml -and $HtmlPath) { if ($sortedComparisons.Count -gt 0) { [void]$htmlBuilder.AppendLine('

    Decision guidance

    ') [void]$htmlBuilder.AppendLine('
      ') + [void]$htmlBuilder.AppendLine(('
    • Decision statement{0}
      • ' -f (ConvertTo-HtmlSafe $decisionStatement))) [void]$htmlBuilder.AppendLine(('
    • Review priority{0}
      • ' -f (Format-HtmlCodeList -Values @($decisionReviewPriority)))) if ($latestComparisonEntry) { - [void]$htmlBuilder.AppendLine(('
    • Latest pairpair {0} is {1}
      • ' -f (ConvertTo-HtmlSafe (Coalesce $latestComparisonEntry.index 'n/a')), (ConvertTo-HtmlSafe $decisionLatestStatus))) + [void]$htmlBuilder.AppendLine(('
    • Latest pair{0} is {1}
      • ' -f (ConvertTo-HtmlSafe (Format-DecisionPairTimelineLabel -Pair $decisionLatestPair)), (ConvertTo-HtmlSafe $decisionLatestStatus))) } if ($latestSignalComparisonEntry) { - $latestSignalBaseRef = [string](Coalesce (Coalesce $latestSignalComparisonEntry.base.short $latestSignalComparisonEntry.base.full) 'n/a') - $latestSignalHeadRef = [string](Coalesce (Coalesce $latestSignalComparisonEntry.head.short $latestSignalComparisonEntry.head.full) 'n/a') - [void]$htmlBuilder.AppendLine(('
    • Review firstpair {0} ({1} -> {2})
      • ' -f (ConvertTo-HtmlSafe (Coalesce $latestSignalComparisonEntry.index 'n/a')), (ConvertTo-HtmlSafe $latestSignalBaseRef), (ConvertTo-HtmlSafe $latestSignalHeadRef))) + [void]$htmlBuilder.AppendLine(('
    • Review first{0}
      • ' -f (ConvertTo-HtmlSafe (Format-DecisionPairTimelineLabel -Pair $decisionLatestSignalPair)))) } if ($decisionReviewSequence.Count -gt 0) { $reviewSequenceLabels = @( $decisionReviewSequence | ForEach-Object { - 'pair {0} ({1} -> {2})' -f $_.index, (Coalesce $_.baseSubject 'n/a'), (Coalesce $_.headSubject 'n/a') + Format-DecisionPairTimelineLabel -Pair $_ -UseSubjectsOnly } ) [void]$htmlBuilder.AppendLine(('
    • Review sequence{0}
      • ' -f (ConvertTo-HtmlSafe ([string]::Join('; ', $reviewSequenceLabels))))) @@ -1997,17 +2103,26 @@ $historySummary = [ordered]@{ outcomeLabels = @(Get-SortedUniqueStringArray -Value $outcomeLabels) } decisionGuidance = [ordered]@{ + decisionStatement = [string]$decisionStatement reviewPriority = [string]$decisionReviewPriority latestPair = [ordered]@{ - index = if ($latestComparisonEntry) { [int](Coalesce $latestComparisonEntry.index 0) } else { 0 } + index = [int](Coalesce $decisionLatestPair.index 0) status = [string]$decisionLatestStatus + baseRef = [string](Coalesce $decisionLatestPair.baseRef '') + headRef = [string](Coalesce $decisionLatestPair.headRef '') + baseSubject = [string](Coalesce $decisionLatestPair.baseSubject '') + headSubject = [string](Coalesce $decisionLatestPair.headSubject '') + baseDate = [string](Coalesce $decisionLatestPair.baseDate '') + headDate = [string](Coalesce $decisionLatestPair.headDate '') } latestSignalPair = [ordered]@{ - index = if ($latestSignalComparisonEntry) { [int](Coalesce $latestSignalComparisonEntry.index 0) } else { 0 } - baseRef = if ($latestSignalComparisonEntry) { [string](Coalesce (Coalesce $latestSignalComparisonEntry.base.short $latestSignalComparisonEntry.base.full) '') } else { '' } - headRef = if ($latestSignalComparisonEntry) { [string](Coalesce (Coalesce $latestSignalComparisonEntry.head.short $latestSignalComparisonEntry.head.full) '') } else { '' } - baseSubject = if ($latestSignalComparisonEntry) { [string](Coalesce $latestSignalComparisonEntry.base.subject '') } else { '' } - headSubject = if ($latestSignalComparisonEntry) { [string](Coalesce $latestSignalComparisonEntry.head.subject '') } else { '' } + index = [int](Coalesce $decisionLatestSignalPair.index 0) + baseRef = [string](Coalesce $decisionLatestSignalPair.baseRef '') + headRef = [string](Coalesce $decisionLatestSignalPair.headRef '') + baseSubject = [string](Coalesce $decisionLatestSignalPair.baseSubject '') + headSubject = [string](Coalesce $decisionLatestSignalPair.headSubject '') + baseDate = [string](Coalesce $decisionLatestSignalPair.baseDate '') + headDate = [string](Coalesce $decisionLatestSignalPair.headDate '') } reviewSequence = @( $decisionReviewSequence | ForEach-Object { @@ -2017,6 +2132,8 @@ $historySummary = [ordered]@{ headRef = [string](Coalesce $_.headRef '') baseSubject = [string](Coalesce $_.baseSubject '') headSubject = [string](Coalesce $_.headSubject '') + baseDate = [string](Coalesce $_.baseDate '') + headDate = [string](Coalesce $_.headDate '') } } ) From c648f4620f07c7334b8b51a3de03d99c921d36a3 Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Mon, 30 Mar 2026 21:07:15 -0700 Subject: [PATCH 11/44] Run Windows VI history proof on self-hosted ingress (#2065) * Route Windows VI history proof to self-hosted ingress * Export self-hosted Windows lane plan outputs --------- Co-authored-by: svelderrainruiz --- .github/workflows/validate.yml | 121 +++++++++++++++--- docs/DEVELOPER_GUIDE.md | 14 +- docs/ENVIRONMENT.md | 16 ++- docs/SELFHOSTED_CI_SETUP.md | 16 ++- docs/knowledgebase/FEATURE_BRANCH_POLICY.md | 16 ++- tools/Resolve-SelfHostedWindowsLanePlan.ps1 | 34 +++-- tools/Test-WindowsNI2026q1HostPreflight.ps1 | 9 +- tools/policy/runner-capability-routing.json | 7 + .../docker-labview-path-contract.test.mjs | 12 +- .../runner-capability-routing-policy.test.mjs | 4 + ...date-vi-history-dispatch-contract.test.mjs | 24 ++-- 11 files changed, 203 insertions(+), 70 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 5f784c99b..e7def6867 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -1496,6 +1496,9 @@ jobs: execution_model: ${{ steps.plan.outputs.execution_model }} runner_image: ${{ steps.plan.outputs.runner_image }} expected_context: ${{ steps.plan.outputs.expected_context }} + expected_os: ${{ steps.plan.outputs.expected_os }} + required_labels: ${{ steps.plan.outputs.required_labels }} + matching_runner_count: ${{ steps.plan.outputs.matching_runner_count }} steps: - uses: actions/checkout@v5 with: @@ -1513,18 +1516,37 @@ jobs: with: phase: J2 results-dir: tests/results - - name: Resolve portable hosted Windows lane + - name: Resolve self-hosted Windows Docker lane id: plan shell: pwsh + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN || secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GH_TOKEN || secrets.GITHUB_TOKEN }} run: | $resultsRoot = 'tests/results/_agent/vi-history-dispatch' New-Item -ItemType Directory -Path $resultsRoot -Force | Out-Null - $planPath = Join-Path $resultsRoot 'validate-vi-history-windows-hosted-plan.json' - pwsh -NoLogo -NoProfile -File tools/Resolve-HostedWindowsLanePlan.ps1 ` - -RunnerImage 'windows-2022' ` - -ContainerImage 'nationalinstruments/labview:2026q1-windows' ` - -ExpectedContext 'default' ` + $planPath = Join-Path $resultsRoot 'validate-vi-history-windows-docker-plan.json' + $requiredLabels = @( + 'self-hosted', + 'Windows', + 'X64', + 'comparevi', + 'capability-ingress', + 'docker-lane' + ) + pwsh -NoLogo -NoProfile -File tools/Resolve-SelfHostedWindowsLanePlan.ps1 ` + -Repository '${{ github.repository }}' ` + -RequiredLabels $requiredLabels ` + -ExecutionModel 'self-hosted-windows-docker-lane' ` + -RunnerImage 'self-hosted-windows-docker-lane' ` + -ExpectedContext 'desktop-windows' ` -ExpectedOs 'windows' ` + -RequiredHealthReceipts @() ` + -Notes @( + 'Availability means an online, idle repository runner advertises the ingress plus docker-lane labels.', + 'The lane may mutate Docker Desktop into the Windows engine and then restores the starting context after proof capture.' + ) ` + -Token $env:GITHUB_TOKEN ` -OutputJsonPath $planPath ` -GitHubOutputPath $env:GITHUB_OUTPUT ` -StepSummaryPath $env:GITHUB_STEP_SUMMARY @@ -1534,15 +1556,16 @@ jobs: run: | if ($env:GITHUB_STEP_SUMMARY) { $lines = @( - '### VI History Scenarios (Windows hosted plan)', + '### VI History Scenarios (Windows self-hosted Docker plan)', '', ('- execute_lanes: `{0}`' -f '${{ needs.vi-history-scenarios-plan.outputs.execute_lanes }}'), ('- lane_available: `{0}`' -f '${{ steps.plan.outputs.available }}'), ('- lane_status: `{0}`' -f '${{ steps.plan.outputs.status }}'), ('- skip_reason: `{0}`' -f '${{ steps.plan.outputs.skip_reason }}'), - ('- hosted_model: `{0}`' -f '${{ steps.plan.outputs.execution_model }}'), + ('- execution_model: `{0}`' -f '${{ steps.plan.outputs.execution_model }}'), ('- runner_image: `{0}`' -f '${{ steps.plan.outputs.runner_image }}'), - '- hosted model: agents can dispatch this portable hosted Windows lane while continuing on local Linux or Windows Docker Desktop lanes.' + ('- required_labels: `{0}`' -f '${{ steps.plan.outputs.required_labels }}'), + '- self-hosted docker model: validate 64-bit LabVIEW in the Windows container while the LV32 lane runs in parallel as the native host-plane reference.' ) $lines -join "`n" | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Encoding utf8 -Append } @@ -1551,20 +1574,24 @@ jobs: if: always() uses: actions/upload-artifact@v7 with: - name: validate-vi-history-windows-hosted-plan - path: tests/results/_agent/vi-history-dispatch/validate-vi-history-windows-hosted-plan.json + name: validate-vi-history-windows-docker-plan + path: tests/results/_agent/vi-history-dispatch/validate-vi-history-windows-docker-plan.json if-no-files-found: error vi-history-scenarios-windows: needs: [smoke-gate, lint, session-index, session-index-v2-contract, vi-history-scenarios-plan, vi-history-scenarios-windows-plan] if: needs.smoke-gate.outputs.skip != 'true' && needs.vi-history-scenarios-plan.outputs.execute_lanes == 'true' && needs.vi-history-scenarios-windows-plan.outputs.available == 'true' - runs-on: windows-2022 + runs-on: [self-hosted, Windows, X64, comparevi, capability-ingress, docker-lane] + continue-on-error: true timeout-minutes: 50 permissions: + actions: read contents: read env: NI_WINDOWS_IMAGE: nationalinstruments/labview:2026q1-windows NI_WINDOWS_LABVIEW_PATH: C:\Program Files\National Instruments\LabVIEW 2026\LabVIEW.exe + GITHUB_TOKEN: ${{ secrets.GH_TOKEN || secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GH_TOKEN || secrets.GITHUB_TOKEN }} defaults: run: shell: pwsh @@ -1573,13 +1600,14 @@ jobs: run: | if ($env:GITHUB_STEP_SUMMARY) { $lines = @( - '### VI History Scenarios (Windows lane)', + '### VI History Scenarios (Windows self-hosted Docker lane)', '', ('- execute_lanes: `{0}`' -f '${{ needs.vi-history-scenarios-plan.outputs.execute_lanes }}'), - ('- hosted_status: `{0}`' -f '${{ needs.vi-history-scenarios-windows-plan.outputs.status }}'), + ('- plan_status: `{0}`' -f '${{ needs.vi-history-scenarios-windows-plan.outputs.status }}'), ('- runner_image: `{0}`' -f '${{ needs.vi-history-scenarios-windows-plan.outputs.runner_image }}'), ('- expected_context: `{0}`' -f '${{ needs.vi-history-scenarios-windows-plan.outputs.expected_context }}'), - '- image pull budget: Windows image hydration is materially slower than the Linux lane, so keep local lanes active while hosted proof runs.' + ('- required_labels: `{0}`' -f '${{ needs.vi-history-scenarios-windows-plan.outputs.required_labels }}'), + '- self-hosted 64-bit proof runs on the ingress host and pairs with the parallel LV32 lane so 32-bit host-plane drift has an immediate native reference.' ) $lines -join "`n" | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Encoding utf8 -Append } @@ -1601,7 +1629,31 @@ jobs: with: phase: J2 results-dir: tests/results - - name: Collect hosted Windows runner health + - name: Validate self-hosted runner label contract + shell: pwsh + run: | + $resultsRoot = Join-Path $env:GITHUB_WORKSPACE 'tests/results/local-parity/windows' + New-Item -ItemType Directory -Path $resultsRoot -Force | Out-Null + $requiredLabels = @( + 'self-hosted', + 'Windows', + 'X64', + 'comparevi', + 'capability-ingress', + 'docker-lane' + ) + foreach ($label in $requiredLabels) { + $contractPath = Join-Path $resultsRoot ("runner-label-contract-{0}.json" -f $label) + pwsh -NoLogo -NoProfile -File tools/Assert-RunnerLabelContract.ps1 ` + -Repository '${{ github.repository }}' ` + -RunnerName $env:RUNNER_NAME ` + -RequiredLabel $label ` + -Token $env:GITHUB_TOKEN ` + -OutputJsonPath $contractPath ` + -GitHubOutputPath $env:GITHUB_OUTPUT ` + -StepSummaryPath $env:GITHUB_STEP_SUMMARY + } + - name: Collect self-hosted Windows runner health shell: pwsh run: | pwsh -NoLogo -NoProfile -File tools/Collect-RunnerHealth.ps1 ` @@ -1618,8 +1670,9 @@ jobs: pwsh -NoLogo -NoProfile -File tools/Test-WindowsNI2026q1HostPreflight.ps1 ` -Image $env:NI_WINDOWS_IMAGE ` -ResultsDir $resultsRoot ` - -ExecutionSurface 'github-hosted-windows' ` - -AllowUnavailable ` + -ExecutionSurface 'desktop-local' ` + -ManageDockerEngine:$true ` + -AllowHostEngineMutation:$true ` -OutputJsonPath $summaryPath ` -GitHubOutputPath $env:GITHUB_OUTPUT ` -StepSummaryPath $env:GITHUB_STEP_SUMMARY @@ -1650,6 +1703,8 @@ jobs: -LabVIEWPath $env:NI_WINDOWS_LABVIEW_PATH ` -ReportPath $reportPath ` -TimeoutSeconds 600 ` + -ManageDockerEngine:$false ` + -AllowHostEngineMutation:$false ` -RuntimeEngineReadyTimeoutSeconds 180 ` -RuntimeEngineReadyPollSeconds 5 ` -RuntimeSnapshotPath $runtimeSnapshot @@ -1743,7 +1798,33 @@ jobs: if: always() && steps.windows-preflight.outputs.windows_host_preflight_status != 'ready' shell: pwsh run: | - Write-Host ("::notice::Skipping hosted Windows compare because preflight status was {0} ({1})." -f '${{ steps.windows-preflight.outputs.windows_host_preflight_status }}', '${{ steps.windows-preflight.outputs.windows_host_preflight_failure_class }}') + Write-Host ("::notice::Skipping self-hosted Windows Docker compare because preflight status was {0} ({1})." -f '${{ steps.windows-preflight.outputs.windows_host_preflight_status }}', '${{ steps.windows-preflight.outputs.windows_host_preflight_failure_class }}') + + - name: Restore Docker Desktop context after Windows proof + if: always() + shell: pwsh + run: | + $resultsRoot = 'tests/results/local-parity/windows' + $preflightPath = Join-Path $resultsRoot 'windows-ni-2026q1-host-preflight.json' + if (-not (Test-Path -LiteralPath $preflightPath -PathType Leaf)) { + Write-Host ("::notice::Skipping Docker Desktop restore because preflight artifact was not found at {0}" -f $preflightPath) + exit 0 + } + $preflight = Get-Content -LiteralPath $preflightPath -Raw | ConvertFrom-Json -Depth 20 + $restoreContext = if ($preflight.contexts.PSObject.Properties['start'] -and -not [string]::IsNullOrWhiteSpace([string]$preflight.contexts.start)) { + [string]$preflight.contexts.start + } else { + 'desktop-windows' + } + $restorePath = Join-Path $resultsRoot 'runtime-manager-restore-windows.json' + pwsh -NoLogo -NoProfile -File tools/Invoke-DockerRuntimeManager.ps1 ` + -ProbeScope 'windows' ` + -BootstrapWindowsImage:$false ` + -BootstrapLinuxImage:$false ` + -RestoreContext $restoreContext ` + -OutputJsonPath $restorePath ` + -GitHubOutputPath '' ` + -StepSummaryPath $env:GITHUB_STEP_SUMMARY | Out-Null - name: Upload VI history scenario artifacts (Windows lane) if: always() @@ -1759,6 +1840,8 @@ jobs: tests/results/local-parity/windows/container-export/** tests/results/local-parity/windows/windows-ni-2026q1-host-preflight.json tests/results/local-parity/windows/runtime-manager-compare-windows.json + tests/results/local-parity/windows/runtime-manager-restore-windows.json + tests/results/local-parity/windows/runner-label-contract-*.json tests/results/local-parity/windows/_agent/runner-health.json if-no-files-found: warn diff --git a/docs/DEVELOPER_GUIDE.md b/docs/DEVELOPER_GUIDE.md index df1423f90..43158d490 100644 --- a/docs/DEVELOPER_GUIDE.md +++ b/docs/DEVELOPER_GUIDE.md @@ -478,14 +478,12 @@ For each cut: `comparevi-history-bundle-certification` follows the same routing. `vi-history-scenarios-*` runs for `compare-engine-history`, `docker-vi-history`, `mixed-runtime`, `unclassified`, and explicit manual dispatches; the final VI-history plan still honors `history_scenario_set`. -- Hosted Windows mirror proof now lives in `Validate` as the non-required `vi-history-scenarios-windows` lane. - That lane runs on GitHub-hosted `windows-2022`, hydrates - `nationalinstruments/labview:2026q1-windows` with no repository runner dependency, and is expected to - take materially longer to pull than the Linux lane. -- If the hosted Windows runner cannot expose a Docker Windows daemon, the lane records - `windows_host_preflight_status = unavailable` and skips the heavy compare instead of poisoning - the current queue item with a non-required proof failure. - Agents can dispatch the hosted lane while continuing with the manual Linux or Windows Docker Desktop/WSL2 lanes locally. +- Windows mirror proof now lives in `Validate` as the non-required + `vi-history-scenarios-windows` lane on the self-hosted compare ingress host. + That lane validates the 64-bit Windows Docker image locally instead of burning + GitHub-hosted Windows image-pull time. +- `vi-history-scenarios-windows-lv32` runs in parallel and acts as the native 32-bit + reference for that Windows Docker proof, so 32-bit and 64-bit feedback arrive together. - `node tools/npm/run-script.mjs priority:lane:concurrency:plan` now reads the host-plane report, host RAM budget, and optional Docker runtime snapshot to recommend a safe concurrent hosted/manual bundle before those lanes are dispatched. diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md index 2558e3eb3..727b54b3f 100644 --- a/docs/ENVIRONMENT.md +++ b/docs/ENVIRONMENT.md @@ -144,14 +144,16 @@ Notes: - The `windows-mirror-proof` local VI-history profile is pinned to this same image and is proof-only in the first slice; it is not a warm or accelerated lane. - Hosted CI now has a matching non-required Windows proof lane in `Validate`: - `vi-history-scenarios-windows`. It runs on GitHub-hosted `windows-2022`, bootstraps + `vi-history-scenarios-windows`. It runs on the self-hosted compare ingress + host with the `docker-lane` capability label, bootstraps `nationalinstruments/labview:2026q1-windows`, and uses the same canonical in-container LabVIEW path: `C:\Program Files\National Instruments\LabVIEW 2026\LabVIEW.exe`. -- If the GitHub-hosted runner cannot expose a usable Docker Windows daemon, the lane now records - `status = unavailable` in the preflight artifact and skips the heavy compare instead of blocking - unrelated integration work. -- Expect the hosted Windows image pull to be materially slower than the Linux lane. - Agents can dispatch the hosted lane while manually running the Linux or Windows Docker Desktop/WSL2 lanes on this host. +- `vi-history-scenarios-windows-lv32` runs in parallel on the same ingress host and serves + as the native 32-bit reference for the Windows Docker proof. +- The lane can mutate Docker Desktop into the Windows engine and restores the starting + context after the proof run so the ingress host does not stay stranded on the wrong daemon. +- Expect the self-hosted Windows image proof to remain materially slower than the Linux lane, + but it now spends local host time instead of GitHub-hosted Windows minutes. - Use `node tools/npm/run-script.mjs priority:lane:concurrency:plan` to turn the current host-plane, host-RAM, and Docker-runtime receipts into a recommended concurrent hosted/manual lane bundle before dispatching work. The plan now emits a `dockerRuntimeCutover` contract so you can tell whether a Linux-based @@ -197,6 +199,8 @@ Notes: `vi-history-scenarios-windows` lane locally: it runs the same NI Windows host preflight, then the same fixed fixture compare invocation against the canonical in-container LabVIEW path. +- That replay remains the fastest way to iterate on the Windows Docker proof locally; the + CI lane exists to keep the parallel 64-bit proof aligned with the LV32 reference job. Common remediation: diff --git a/docs/SELFHOSTED_CI_SETUP.md b/docs/SELFHOSTED_CI_SETUP.md index 9bea93442..6c745e827 100644 --- a/docs/SELFHOSTED_CI_SETUP.md +++ b/docs/SELFHOSTED_CI_SETUP.md @@ -99,8 +99,20 @@ today's compare jobs is: - `tools/policy/runner-capability-routing.json` That matrix keeps most compare jobs ingress-only, but -`.github/workflows/labview-cli-compare.yml` is now an explicit native 32-bit -consumer: +`.github/workflows/validate.yml`'s `vi-history-scenarios-windows` lane is now an +explicit Windows Docker proof consumer: + +- `runs-on: [self-hosted, Windows, X64, comparevi, capability-ingress, docker-lane]` +- it may mutate Docker Desktop into the Windows engine to prove `nationalinstruments/labview:2026q1-windows` +- it restores the starting Docker context after the proof completes + +The matrix also keeps the native 32-bit consumers explicit so they can run in +parallel with the 64-bit Windows Docker proof: + +- `.github/workflows/validate.yml` `vi-history-scenarios-windows-lv32` +- `.github/workflows/labview-cli-compare.yml` + +- `.github/workflows/labview-cli-compare.yml` consumer details: - `runs-on: [self-hosted, Windows, X64, comparevi, capability-ingress, labview-2026, lv32]` - emit `node tools/npm/run-script.mjs env:labview:2026:host-planes` diff --git a/docs/knowledgebase/FEATURE_BRANCH_POLICY.md b/docs/knowledgebase/FEATURE_BRANCH_POLICY.md index d37edde8c..5b9366cf3 100644 --- a/docs/knowledgebase/FEATURE_BRANCH_POLICY.md +++ b/docs/knowledgebase/FEATURE_BRANCH_POLICY.md @@ -1,7 +1,7 @@ # Feature Branch Enforcement & Merge Queue -| `develop` (live id may drift) | `refs/heads/develop` | Merge queue enabled (`merge_method=SQUASH`, `grouping=ALLGREEN`, build queue <=20 entries, 1-minute quiet window). Required checks: `lint`, `fixtures`, `Policy Guard (Upstream) / policy-guard`, `vi-history-scenarios-linux`, `commit-integrity`. Non-required hosted proof lanes may run alongside the queue contract, including `session-index`, `issue-snapshot`, `semver`, `agent-review-policy`, `hook-parity`, and `vi-history-scenarios-windows` on GitHub-hosted `windows-2022`. Copilot review settings are no longer enforced through policy; draft/ready review semantics are repo-owned and validated by `agent-review-policy`. | +| `develop` (live id may drift) | `refs/heads/develop` | Merge queue enabled (`merge_method=SQUASH`, `grouping=ALLGREEN`, build queue <=20 entries, 1-minute quiet window). Required checks: `lint`, `fixtures`, `Policy Guard (Upstream) / policy-guard`, `vi-history-scenarios-linux`, `commit-integrity`. Non-required supporting lanes may run alongside the queue contract, including `session-index`, `issue-snapshot`, `semver`, `agent-review-policy`, `hook-parity`, plus the self-hosted Windows Docker and LV32 VI-history proofs that run in parallel on the compare ingress host. Copilot review settings are no longer enforced through policy; draft/ready review semantics are repo-owned and validated by `agent-review-policy`. | ## Purpose @@ -129,10 +129,16 @@ checked into `tools/priority/policy.json` so `priority:policy` stays authoritati - **Required checks**: `lint`, `fixtures`, `Policy Guard (Upstream) / policy-guard`, `vi-history-scenarios-linux`, `commit-integrity`. - **Supporting non-required lanes**: `session-index`, `issue-snapshot`, `semver`, - `agent-review-policy`, `hook-parity`, plus hosted Windows proving. -- **Non-required hosted proof**: `vi-history-scenarios-windows` may run on GitHub-hosted - `windows-2022` to validate `nationalinstruments/labview:2026q1-windows`. Agents may - dispatch that hosted lane while manually running the Linux or Windows Docker Desktop/WSL2 lanes on this host. + `agent-review-policy`, `hook-parity`, plus parallel Windows 64-bit and LV32 proving. +- **Non-required self-hosted Docker proof**: `vi-history-scenarios-windows` runs on the + compare ingress host with the `docker-lane` capability label to validate + `nationalinstruments/labview:2026q1-windows` without paying GitHub-hosted Windows + image-hydration time on every PR. The lane may flip Docker Desktop into the + Windows engine, capture the 64-bit container proof, and restore the starting + context afterward. +- **Parallel LV32 reference proof**: `vi-history-scenarios-windows-lv32` runs beside the + Docker lane so native 32-bit LabVIEW and the 64-bit Windows container can be observed + together without serializing the Windows-side feedback loop. - **Admin bypass**: leave disabled; administrators should only intervene when `priority:policy` confirms parity. - **Reapply**: Use `node tools/npm/run-script.mjs priority:policy -- --apply` to push the manifest configuration when drift is detected. diff --git a/tools/Resolve-SelfHostedWindowsLanePlan.ps1 b/tools/Resolve-SelfHostedWindowsLanePlan.ps1 index 12a8f09d1..2da95068c 100644 --- a/tools/Resolve-SelfHostedWindowsLanePlan.ps1 +++ b/tools/Resolve-SelfHostedWindowsLanePlan.ps1 @@ -1,12 +1,12 @@ #Requires -Version 7.0 <# .SYNOPSIS - Resolves the availability plan for a self-hosted Windows LV32 lane. + Resolves the availability plan for a self-hosted Windows specialized lane. .DESCRIPTION Emits a deterministic planning artifact for a repository runner that must - advertise the required ingress plus LV32 capability labels. The helper uses - the repository runner inventory API or an injected inventory fixture. + advertise the required ingress plus optional capability labels. The helper + uses the repository runner inventory API or an injected inventory fixture. #> [CmdletBinding()] param( @@ -20,6 +20,15 @@ param( 'labview-2026', 'lv32' ), + [string]$ExecutionModel = 'self-hosted-windows-lv32', + [string]$RunnerImage = 'self-hosted-windows-lv32', + [string]$ExpectedContext = 'headless-labview-32', + [string]$ExpectedOs = 'windows', + [string[]]$RequiredHealthReceipts = @('labview-2026-host-plane-report'), + [string[]]$Notes = @( + 'Availability means an online, idle repository runner advertises every required label.', + 'The lane must skip rather than queue indefinitely when no matching runner is available.' + ), [string]$Token = $env:GITHUB_TOKEN, [string]$OutputJsonPath = 'tests/results/_agent/vi-history-dispatch/validate-vi-history-windows-lv32-plan.json', [string]$GitHubOutputPath = $env:GITHUB_OUTPUT, @@ -183,15 +192,12 @@ $summary = [ordered]@{ skipReason = '' failureClass = 'none' failureMessage = '' - executionModel = 'self-hosted-windows-lv32' - runnerImage = 'self-hosted-windows-lv32' - expectedContext = 'headless-labview-32' - expectedOs = 'windows' - requiredHealthReceipts = @('labview-2026-host-plane-report') - notes = @( - 'Availability means an online, idle repository runner advertises every required label.', - 'The lane must skip rather than queue indefinitely when no matching runner is available.' - ) + executionModel = [string]$ExecutionModel + runnerImage = [string]$RunnerImage + expectedContext = [string]$ExpectedContext + expectedOs = [string]$ExpectedOs + requiredHealthReceipts = @($RequiredHealthReceipts) + notes = @($Notes) } try { @@ -277,7 +283,7 @@ try { } else { $summary.available = $false $summary.status = 'missing-label' - $summary.skipReason = 'no online self-hosted Windows LV32 runner matched the required capability labels' + $summary.skipReason = 'no online self-hosted Windows runner matched the required capability labels' } } catch { $summary.available = $false @@ -315,7 +321,7 @@ if (-not [string]::IsNullOrWhiteSpace($StepSummaryPath)) { 'none' } $summaryLines = @( - '### Self-Hosted Windows LV32 Lane Plan', + '### Self-Hosted Windows Specialized Lane Plan', '', ('- status: `{0}`' -f [string]$summary.status), ('- available: `{0}`' -f ([string]([bool]$summary.available)).ToLowerInvariant()), diff --git a/tools/Test-WindowsNI2026q1HostPreflight.ps1 b/tools/Test-WindowsNI2026q1HostPreflight.ps1 index 6ff4f609f..2dbb3b83b 100644 --- a/tools/Test-WindowsNI2026q1HostPreflight.ps1 +++ b/tools/Test-WindowsNI2026q1HostPreflight.ps1 @@ -21,6 +21,8 @@ param( [string]$ResultsDir = 'tests/results/local-parity', [ValidateSet('desktop-local', 'github-hosted-windows')] [string]$ExecutionSurface = 'desktop-local', + [bool]$ManageDockerEngine = $false, + [bool]$AllowHostEngineMutation = $false, [switch]$AllowUnavailable, [string]$OutputJsonPath = '', [string]$GitHubOutputPath = $env:GITHUB_OUTPUT, @@ -292,7 +294,8 @@ try { $summary.contexts.final = [string]$desktopObservation.context $summary.contexts.finalOsType = [string]$desktopObservation.osType - if ([string]::Equals([string]$desktopObservation.osType, 'linux', [System.StringComparison]::OrdinalIgnoreCase)) { + $canMutateDesktopEngine = $ManageDockerEngine -and $AllowHostEngineMutation + if ([string]::Equals([string]$desktopObservation.osType, 'linux', [System.StringComparison]::OrdinalIgnoreCase) -and -not $canMutateDesktopEngine) { $summary.status = 'failure' $summary.failureClass = 'docker-engine-mismatch' $summary.failureMessage = ("desktop-local Windows NI preflight requires Docker Desktop Windows containers. Observed context '{0}' with OSType '{1}'. Switch Docker Desktop to Windows containers (`desktop-windows`) and retry." -f ([string]$desktopObservation.context ?? ''), [string]$desktopObservation.osType) @@ -354,8 +357,8 @@ try { -RuntimeProvider 'desktop' ` -ExpectedContext 'default' ` -AutoRepair:$true ` - -ManageDockerEngine:$false ` - -AllowHostEngineMutation:$false ` + -ManageDockerEngine:$ManageDockerEngine ` + -AllowHostEngineMutation:$AllowHostEngineMutation ` -SnapshotPath $snapshotPath ` -GitHubOutputPath '' if ($LASTEXITCODE -ne 0) { diff --git a/tools/policy/runner-capability-routing.json b/tools/policy/runner-capability-routing.json index c8bcce3e9..16ae1c607 100644 --- a/tools/policy/runner-capability-routing.json +++ b/tools/policy/runner-capability-routing.json @@ -119,6 +119,13 @@ { "workflow": ".github/workflows/validate.yml", "jobs": [ + { + "id": "vi-history-scenarios-windows", + "routingClass": "specialized-opt-in", + "requiredCapabilityLabels": [ + "docker-lane" + ] + }, { "id": "vi-history-scenarios-windows-lv32", "routingClass": "specialized-opt-in", diff --git a/tools/priority/__tests__/docker-labview-path-contract.test.mjs b/tools/priority/__tests__/docker-labview-path-contract.test.mjs index 42bcf9441..dcc0a93a5 100644 --- a/tools/priority/__tests__/docker-labview-path-contract.test.mjs +++ b/tools/priority/__tests__/docker-labview-path-contract.test.mjs @@ -11,7 +11,7 @@ function readRepoFile(relativePath) { return readFileSync(path.join(repoRoot, relativePath), 'utf8'); } -test('validate workflow pins explicit LabVIEW paths for hosted Linux and Windows VI history lanes', () => { +test('validate workflow pins explicit LabVIEW paths for hosted Linux and self-hosted Windows VI history lanes', () => { const workflow = readRepoFile('.github/workflows/validate.yml'); assert.match(workflow, /NI_LINUX_IMAGE:\s*nationalinstruments\/labview:2026q1-linux/); @@ -20,15 +20,19 @@ test('validate workflow pins explicit LabVIEW paths for hosted Linux and Windows assert.match(workflow, /-Image \$env:NI_LINUX_IMAGE/); assert.match(workflow, /-LabVIEWPath \$env:NI_LINUX_LABVIEW_PATH/); assert.match(workflow, /vi-history-scenarios-windows-plan:/); - assert.match(workflow, /Resolve-HostedWindowsLanePlan\.ps1/); + assert.match(workflow, /Resolve-SelfHostedWindowsLanePlan\.ps1/); assert.match(workflow, /vi-history-scenarios-windows:/); - assert.match(workflow, /runs-on:\s*windows-2022/); + assert.match(workflow, /runs-on:\s*\[self-hosted, Windows, X64, comparevi, capability-ingress, docker-lane\]/); assert.match(workflow, /NI_WINDOWS_IMAGE:\s*nationalinstruments\/labview:2026q1-windows/); assert.match(workflow, /NI_WINDOWS_LABVIEW_PATH:\s*C:\\Program Files\\National Instruments\\LabVIEW 2026\\LabVIEW\.exe/); assert.match(workflow, /Test-WindowsNI2026q1HostPreflight\.ps1/); assert.match(workflow, /Write-VIHistoryLaneEvidence\.ps1/); assert.match(workflow, /Run-NIWindowsContainerCompare\.ps1/); - assert.match(workflow, /-ExecutionSurface 'github-hosted-windows'/); + assert.match(workflow, /Assert-RunnerLabelContract\.ps1/); + assert.match(workflow, /-ExecutionSurface 'desktop-local'/); + assert.match(workflow, /-ManageDockerEngine:\$true/); + assert.match(workflow, /-AllowHostEngineMutation:\$true/); + assert.match(workflow, /Restore Docker Desktop context after Windows proof/); assert.match(workflow, /-Image \$env:NI_WINDOWS_IMAGE/); assert.match(workflow, /-LabVIEWPath \$env:NI_WINDOWS_LABVIEW_PATH/); assert.match(workflow, /validate-vi-history-scenarios-windows/); diff --git a/tools/priority/__tests__/runner-capability-routing-policy.test.mjs b/tools/priority/__tests__/runner-capability-routing-policy.test.mjs index 086c842e0..0ba88a41f 100644 --- a/tools/priority/__tests__/runner-capability-routing-policy.test.mjs +++ b/tools/priority/__tests__/runner-capability-routing-policy.test.mjs @@ -53,6 +53,10 @@ test('runner capability routing policy covers all current self-hosted compare wo assert.equal(job.routingClass, 'specialized-opt-in'); assert.deepEqual(job.requiredCapabilityLabels, ['labview-2026', 'lv32']); assert.deepEqual(job.requiredHealthReceipts, ['labview-2026-host-plane-report']); + } else if (entry.workflow === '.github/workflows/validate.yml' && job.id === 'vi-history-scenarios-windows') { + assert.equal(job.routingClass, 'specialized-opt-in'); + assert.deepEqual(job.requiredCapabilityLabels, ['docker-lane']); + assert.equal(job.requiredHealthReceipts, undefined); } else if (entry.workflow === '.github/workflows/validate.yml' && job.id === 'vi-history-scenarios-windows-lv32') { assert.equal(job.routingClass, 'specialized-opt-in'); assert.deepEqual(job.requiredCapabilityLabels, ['labview-2026', 'lv32']); diff --git a/tools/priority/__tests__/validate-vi-history-dispatch-contract.test.mjs b/tools/priority/__tests__/validate-vi-history-dispatch-contract.test.mjs index ebca3b181..d79d868d8 100644 --- a/tools/priority/__tests__/validate-vi-history-dispatch-contract.test.mjs +++ b/tools/priority/__tests__/validate-vi-history-dispatch-contract.test.mjs @@ -57,35 +57,41 @@ test('validate workflow Linux VI-history lane consumes shared dispatch-plan outp assert.doesNotMatch(linuxSection, /pull-requests: read/); }); -test('validate workflow Windows VI-history lane is gated by shared dispatch planning and portable hosted execution', () => { +test('validate workflow Windows VI-history lane is gated by shared dispatch planning and self-hosted docker execution', () => { const workflow = readRepoFile('.github/workflows/validate.yml'); const planSection = extractWorkflowJobSection(workflow, 'vi-history-scenarios-windows-plan', 'vi-history-scenarios-windows'); const windowsSection = extractWorkflowJobSection(workflow, 'vi-history-scenarios-windows', 'vi-history-scenarios-windows-lv32-plan'); assert.match(workflow, /vi-history-scenarios-windows-plan:\s*\r?\n\s+needs:\s*\[smoke-gate, lint, session-index, session-index-v2-contract, vi-history-scenarios-plan\]\r?\n\s+if:\s+needs\.smoke-gate\.outputs\.skip != 'true'/); assert.match(planSection, /permissions:\s*\r?\n\s+contents: read/); - assert.match(planSection, /Resolve portable hosted Windows lane/); - assert.match(planSection, /tools\/Resolve-HostedWindowsLanePlan\.ps1/); - assert.match(planSection, /-RunnerImage 'windows-2022'/); + assert.match(planSection, /Resolve self-hosted Windows Docker lane/); + assert.match(planSection, /tools\/Resolve-SelfHostedWindowsLanePlan\.ps1/); + assert.match(planSection, /-RequiredLabels \$requiredLabels/); + assert.match(planSection, /docker-lane/); assert.match(planSection, /outputs:\s*\r?\n\s+available:\s+\$\{\{\s*steps\.plan\.outputs\.available\s*\}\}/); assert.match(workflow, /vi-history-scenarios-windows:\s*\r?\n\s+needs:\s*\[smoke-gate, lint, session-index, session-index-v2-contract, vi-history-scenarios-plan, vi-history-scenarios-windows-plan\]\r?\n\s+if:\s+needs\.smoke-gate\.outputs\.skip != 'true' && needs\.vi-history-scenarios-plan\.outputs\.execute_lanes == 'true' && needs\.vi-history-scenarios-windows-plan\.outputs\.available == 'true'/); - assert.match(windowsSection, /runs-on:\s*windows-2022/); + assert.match(windowsSection, /runs-on:\s*\[self-hosted, Windows, X64, comparevi, capability-ingress, docker-lane\]/); + assert.match(windowsSection, /continue-on-error:\s*true/); assert.match(windowsSection, /Print VI history Windows runtime alignment/); - assert.match(windowsSection, /Collect hosted Windows runner health/); + assert.match(windowsSection, /Validate self-hosted runner label contract/); + assert.match(windowsSection, /Assert-RunnerLabelContract\.ps1/); + assert.match(windowsSection, /Collect self-hosted Windows runner health/); assert.match(windowsSection, /Collect-RunnerHealth\.ps1/); assert.match(windowsSection, /Project VI history Windows Docker-side evidence/); assert.match(windowsSection, /tools\/Write-VIHistoryLaneEvidence\.ps1/); assert.match(windowsSection, /Test-WindowsNI2026q1HostPreflight\.ps1/); - assert.match(windowsSection, /-ExecutionSurface 'github-hosted-windows'/); - assert.match(windowsSection, /-AllowUnavailable/); + assert.match(windowsSection, /-ExecutionSurface 'desktop-local'/); + assert.match(windowsSection, /-ManageDockerEngine:\$true/); + assert.match(windowsSection, /-AllowHostEngineMutation:\$true/); assert.match(windowsSection, /id:\s*windows-preflight/); assert.match(windowsSection, /Run-NIWindowsContainerCompare\.ps1/); assert.match(windowsSection, /if:\s*steps\.windows-preflight\.outputs\.windows_host_preflight_status == 'ready'/); + assert.match(windowsSection, /Restore Docker Desktop context after Windows proof/); + assert.match(windowsSection, /Invoke-DockerRuntimeManager\.ps1/); assert.match(windowsSection, /tests\/results\/local-parity\/windows\/_agent\/runner-health\.json/); assert.match(windowsSection, /ni-windows-container-stdout\.txt/); assert.match(windowsSection, /ni-windows-container-stderr\.txt/); - assert.doesNotMatch(windowsSection, /Assert-RunnerLabelContract\.ps1/); }); test('validate workflow self-hosted Windows LV32 VI-history lane is gated by inventory planning and headless proof receipts', () => { From afca493c14d51fe7f13e8fd99536c7cdfcb8598c Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Mon, 30 Mar 2026 21:53:01 -0700 Subject: [PATCH 12/44] Honor docker override across NI Linux proof tooling (#2066) Honor docker override in NI Linux proof path Co-authored-by: svelderrainruiz --- .../Assert-DockerRuntimeDeterminism.Tests.ps1 | 43 ++++++++++ tools/Assert-DockerRuntimeDeterminism.ps1 | 61 +++++++++++++-- tools/Run-NILinuxContainerCompare.ps1 | 78 ++++++++++++++++--- 3 files changed, 166 insertions(+), 16 deletions(-) diff --git a/tests/Assert-DockerRuntimeDeterminism.Tests.ps1 b/tests/Assert-DockerRuntimeDeterminism.Tests.ps1 index 9f7e708ba..498c949ff 100644 --- a/tests/Assert-DockerRuntimeDeterminism.Tests.ps1 +++ b/tests/Assert-DockerRuntimeDeterminism.Tests.ps1 @@ -171,6 +171,7 @@ exit 0 DOCKER_STUB_CONTEXT = $env:DOCKER_STUB_CONTEXT DOCKER_STUB_CONTEXT_SHOW_EMPTY = $env:DOCKER_STUB_CONTEXT_SHOW_EMPTY DOCKER_STUB_CONTEXT_USE_FAIL_TARGET = $env:DOCKER_STUB_CONTEXT_USE_FAIL_TARGET + DOCKER_COMMAND_OVERRIDE = $env:DOCKER_COMMAND_OVERRIDE DOCKER_HOST = $env:DOCKER_HOST } } @@ -231,6 +232,48 @@ exit 0 $snapshot.result.reason | Should -Match 'exitCode=1' } + It 'honors DOCKER_COMMAND_OVERRIDE before a failing docker on PATH' { + $work = Join-Path $TestDrive 'docker-command-override' + New-Item -ItemType Directory -Path $work -Force | Out-Null + & $script:CreateDockerWslStubs -WorkRoot $work + + $fallbackBin = Join-Path $work 'fallback-bin' + New-Item -ItemType Directory -Path $fallbackBin -Force | Out-Null + if ($IsWindows) { + @" +@echo off +echo fallback docker should not run 1>&2 +exit /b 9 +"@ | Set-Content -LiteralPath (Join-Path $fallbackBin 'docker.cmd') -Encoding ascii + $env:PATH = "{0};{1}" -f $fallbackBin, $env:PATH + } else { + @' +#!/usr/bin/env bash +echo fallback docker should not run 1>&2 +exit 9 +'@ | Set-Content -LiteralPath (Join-Path $fallbackBin 'docker') -Encoding utf8 + & chmod +x (Join-Path $fallbackBin 'docker') + $env:PATH = "{0}:{1}" -f $fallbackBin, $env:PATH + } + + Set-Item Env:DOCKER_COMMAND_OVERRIDE (Join-Path $work 'bin' 'docker.ps1') + Set-Item Env:DOCKER_STUB_INFO_MODE 'parsed-linux' + Set-Item Env:DOCKER_STUB_CONTEXT 'desktop-linux' + + $snapshotPath = Join-Path $work 'runtime.json' + $output = & pwsh -NoLogo -NoProfile -File $script:GuardScript ` + -ExpectedOsType linux ` + -ExpectedContext desktop-linux ` + -AutoRepair:$false ` + -SnapshotPath $snapshotPath ` + -GitHubOutputPath '' 2>&1 + $LASTEXITCODE | Should -Be 0 -Because ($output -join "`n") + + $snapshot = Get-Content -LiteralPath $snapshotPath -Raw | ConvertFrom-Json -Depth 12 + $snapshot.result.status | Should -Be 'ok' + $snapshot.observed.osType | Should -Be 'linux' + } + It 'classifies parse-defect when docker info output is non-empty but unparseable' { $work = Join-Path $TestDrive 'parse-defect-unparseable' New-Item -ItemType Directory -Path $work -Force | Out-Null diff --git a/tools/Assert-DockerRuntimeDeterminism.ps1 b/tools/Assert-DockerRuntimeDeterminism.ps1 index c516284d9..5de4003c7 100644 --- a/tools/Assert-DockerRuntimeDeterminism.ps1 +++ b/tools/Assert-DockerRuntimeDeterminism.ps1 @@ -113,6 +113,38 @@ function Split-OutputLines { return @($Text -split "(`r`n|`n|`r)" | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) } +function Resolve-DockerCommandSource { + $override = $env:DOCKER_COMMAND_OVERRIDE + if (-not [string]::IsNullOrWhiteSpace($override) -and (Test-Path -LiteralPath $override -PathType Leaf)) { + return [System.IO.Path]::GetFullPath($override) + } + + $pathSeparator = [System.IO.Path]::PathSeparator + $pathEntries = @($env:PATH -split [regex]::Escape([string]$pathSeparator)) + $candidates = if ($IsWindows) { + @('docker.exe', 'docker.cmd', 'docker.ps1', 'docker.bat', 'docker') + } else { + @('docker', 'docker.sh', 'docker.exe', 'docker.ps1', 'docker.cmd') + } + + foreach ($entry in $pathEntries) { + if ([string]::IsNullOrWhiteSpace($entry)) { continue } + foreach ($name in $candidates) { + $candidatePath = Join-Path $entry $name + if (Test-Path -LiteralPath $candidatePath -PathType Leaf) { + return [System.IO.Path]::GetFullPath($candidatePath) + } + } + } + + $command = Get-Command -Name 'docker' -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($null -eq $command -or [string]::IsNullOrWhiteSpace([string]$command.Source)) { + return $null + } + + return [System.IO.Path]::GetFullPath([string]$command.Source) +} + function Invoke-ProcessWithTimeout { [CmdletBinding()] param( @@ -123,16 +155,31 @@ function Invoke-ProcessWithTimeout { $safeTimeout = [math]::Max(5, [int]$TimeoutSeconds) $resolvedFilePath = $FilePath + $effectiveArguments = @($Arguments) + if ([string]::Equals($FilePath, 'docker', [System.StringComparison]::OrdinalIgnoreCase)) { + $dockerCommandSource = Resolve-DockerCommandSource + if (-not [string]::IsNullOrWhiteSpace($dockerCommandSource)) { + $dockerCommandExtension = [System.IO.Path]::GetExtension($dockerCommandSource) + if ([System.StringComparer]::OrdinalIgnoreCase.Equals($dockerCommandExtension, '.ps1')) { + $resolvedFilePath = (Get-Command -Name 'pwsh' -ErrorAction Stop | Select-Object -First 1).Source + $effectiveArguments = @('-NoLogo', '-NoProfile', '-File', $dockerCommandSource) + @($Arguments) + } else { + $resolvedFilePath = $dockerCommandSource + } + } + } try { - $resolvedCommand = Get-Command -Name $FilePath -CommandType Application -ErrorAction Stop | Select-Object -First 1 - if ($resolvedCommand -and $resolvedCommand.Source) { - $resolvedFilePath = [string]$resolvedCommand.Source - } elseif ($resolvedCommand -and $resolvedCommand.Path) { - $resolvedFilePath = [string]$resolvedCommand.Path + if ([string]::Equals($resolvedFilePath, $FilePath, [System.StringComparison]::Ordinal)) { + $resolvedCommand = Get-Command -Name $FilePath -CommandType Application -ErrorAction Stop | Select-Object -First 1 + if ($resolvedCommand -and $resolvedCommand.Source) { + $resolvedFilePath = [string]$resolvedCommand.Source + } elseif ($resolvedCommand -and $resolvedCommand.Path) { + $resolvedFilePath = [string]$resolvedCommand.Path + } } } catch {} - $argText = if ($Arguments -and $Arguments.Count -gt 0) { [string]::Join(' ', $Arguments) } else { '' } + $argText = if ($effectiveArguments -and $effectiveArguments.Count -gt 0) { [string]::Join(' ', $effectiveArguments) } else { '' } $commandText = if ([string]::IsNullOrWhiteSpace($argText)) { $resolvedFilePath } else { "$resolvedFilePath $argText" } $result = [ordered]@{ @@ -150,7 +197,7 @@ function Invoke-ProcessWithTimeout { $psi.RedirectStandardOutput = $true $psi.RedirectStandardError = $true $psi.CreateNoWindow = $true - foreach ($arg in @($Arguments)) { + foreach ($arg in @($effectiveArguments)) { [void]$psi.ArgumentList.Add([string]$arg) } diff --git a/tools/Run-NILinuxContainerCompare.ps1 b/tools/Run-NILinuxContainerCompare.ps1 index 3b2752836..3735b49c8 100644 --- a/tools/Run-NILinuxContainerCompare.ps1 +++ b/tools/Run-NILinuxContainerCompare.ps1 @@ -155,7 +155,7 @@ function Resolve-DockerCommandSource { $candidates = if ($IsWindows) { @('docker.exe', 'docker.cmd', 'docker.ps1', 'docker.bat', 'docker') } else { - @('docker', 'docker.sh') + @('docker', 'docker.sh', 'docker.exe', 'docker.ps1', 'docker.cmd') } foreach ($entry in $pathEntries) { if ([string]::IsNullOrWhiteSpace($entry)) { continue } @@ -173,6 +173,24 @@ function Resolve-DockerCommandSource { return [string]$command.Source } +function Invoke-DirectDockerCommand { + param([Parameter(Mandatory)][string[]]$DockerArgs) + + $dockerCommandSource = Resolve-DockerCommandSource + $dockerCommandExtension = [System.IO.Path]::GetExtension($dockerCommandSource) + $stdout = @() + if ([System.StringComparer]::OrdinalIgnoreCase.Equals($dockerCommandExtension, '.ps1')) { + $stdout = @(& pwsh -NoLogo -NoProfile -File $dockerCommandSource @DockerArgs) + } else { + $stdout = @(& $dockerCommandSource @DockerArgs) + } + + return [pscustomobject]@{ + ExitCode = [int]$LASTEXITCODE + StdOut = @($stdout) + } +} + function Get-EffectiveCompareFlags { param( [AllowNull()][string[]]$InputFlags @@ -377,8 +395,8 @@ function Resolve-OutputReportPath { function Test-DockerImageExists { param([Parameter(Mandatory)][string]$Tag) - & docker image inspect $Tag *> $null - return ($LASTEXITCODE -eq 0) + $inspectResult = Invoke-DirectDockerCommand -DockerArgs @('image', 'inspect', $Tag) 2>$null + return ($inspectResult.ExitCode -eq 0) } function Get-OrAddMountPath { @@ -405,6 +423,44 @@ function Convert-HostFileToContainerPath { return (Join-Path $containerDir (Split-Path -Leaf $HostFilePath)).Replace('\', '/') } +function Resolve-DockerBindMountHostPath { + param( + [Parameter(Mandatory)][string]$HostPath, + [AllowEmptyString()][string]$DockerCommandSource + ) + + $resolvedHostPath = [System.IO.Path]::GetFullPath($HostPath) + if ([string]::IsNullOrWhiteSpace($DockerCommandSource)) { + return $resolvedHostPath + } + + $dockerCommandExtension = [System.IO.Path]::GetExtension($DockerCommandSource) + if (-not [System.StringComparer]::OrdinalIgnoreCase.Equals($dockerCommandExtension, '.exe')) { + return $resolvedHostPath + } + + if (-not $resolvedHostPath.StartsWith('/')) { + return $resolvedHostPath + } + + $wslCommand = Get-Command -Name 'wsl.exe' -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($null -eq $wslCommand -or [string]::IsNullOrWhiteSpace([string]$wslCommand.Source)) { + return $resolvedHostPath + } + + $translatedOutput = & $wslCommand.Source wslpath -w $resolvedHostPath 2>$null + if ($LASTEXITCODE -ne 0) { + return $resolvedHostPath + } + + $translatedPath = [string](@($translatedOutput) | Select-Object -Last 1) + if ([string]::IsNullOrWhiteSpace($translatedPath)) { + return $resolvedHostPath + } + + return $translatedPath.Trim() +} + function Test-HostPathWithinRoot { param( [Parameter(Mandatory)][string]$RootPath, @@ -460,10 +516,11 @@ function Convert-HostPathToExistingContainerPath { function Get-DockerContainerRecord { param([Parameter(Mandatory)][string]$ContainerName) - $inspectOutput = & docker inspect $ContainerName 2>$null - if ($LASTEXITCODE -ne 0) { + $inspectResult = Invoke-DirectDockerCommand -DockerArgs @('inspect', $ContainerName) 2>$null + if ($inspectResult.ExitCode -ne 0) { return $null } + $inspectOutput = @($inspectResult.StdOut) try { $records = $inspectOutput | ConvertFrom-Json -Depth 12 -ErrorAction Stop @@ -1739,8 +1796,8 @@ function Invoke-DockerExecWithTimeout { while (-not $process.HasExited) { if ((Get-Date) -ge $deadline) { try { Stop-Process -Id $process.Id -Force -ErrorAction SilentlyContinue } catch {} - try { & docker exec $ContainerName sh -lc 'pkill -f LabVIEWCLI || true' *> $null } catch {} - try { & docker stop --time 1 $ContainerName *> $null } catch {} + try { Invoke-DirectDockerCommand -DockerArgs @('exec', $ContainerName, 'sh', '-lc', 'pkill -f LabVIEWCLI || true') *> $null } catch {} + try { Invoke-DirectDockerCommand -DockerArgs @('stop', '--time', '1', $ContainerName) *> $null } catch {} $capturedOutput = Complete-DockerProcessInvocation -Invocation $invocation return [pscustomobject]@{ TimedOut = $true @@ -2467,6 +2524,7 @@ try { $containerBaseVi = '' $containerHeadVi = '' $containerReportPath = '' + $dockerCommandSource = Resolve-DockerCommandSource if ($useExistingContainer) { $reuseRepoHostPathResolved = if ([string]::IsNullOrWhiteSpace($ReuseRepoHostPath)) { if ($viHistoryEnabled -and -not [string]::IsNullOrWhiteSpace([string]$resolvedRuntimeBootstrap.viHistory.repoHostPath)) { @@ -2661,11 +2719,13 @@ try { } if (-not $useExistingContainer) { foreach ($entry in ($mounts.GetEnumerator() | Sort-Object Name)) { - $volumeSpec = '{0}:{1}' -f $entry.Name, $entry.Value + $mountHostPath = Resolve-DockerBindMountHostPath -HostPath ([string]$entry.Name) -DockerCommandSource $dockerCommandSource + $volumeSpec = '{0}:{1}' -f $mountHostPath, $entry.Value $dockerArgs += @('-v', $volumeSpec) } foreach ($runtimeMount in $resolvedRuntimeInjectionMounts) { - $dockerArgs += @('-v', ('{0}:{1}' -f $runtimeMount.hostPath, $runtimeMount.containerPath)) + $runtimeMountHostPath = Resolve-DockerBindMountHostPath -HostPath ([string]$runtimeMount.hostPath) -DockerCommandSource $dockerCommandSource + $dockerArgs += @('-v', ('{0}:{1}' -f $runtimeMountHostPath, $runtimeMount.containerPath)) } } if (-not [string]::IsNullOrWhiteSpace($containerBaseVi)) { From 78e578f9e825240d8000d8e4e1a2413059969f3e Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Tue, 31 Mar 2026 07:14:53 -0700 Subject: [PATCH 13/44] Introduce Pester service-model pilot (#2068) * Introduce Pester service-model pilot * Fix Windows VI history planner invocation * Make Pester service model consume receipts * Add trusted PR entrypoint for Pester service model * Fix trusted pilot workflow lint contract --------- Co-authored-by: svelderrainruiz --- .github/workflows/pester-evidence.yml | 313 +++++++++++++++++ .github/workflows/pester-gate.yml | 88 +++++ .github/workflows/pester-run.yml | 332 ++++++++++++++++++ .../pester-service-model-on-label.yml | 114 ++++++ .github/workflows/selfhosted-readiness.yml | 238 +++++++++++++ .github/workflows/validate.yml | 54 ++- docs/knowledgebase/Pester-Service-Model.md | 44 +++ tools/policy/runner-capability-routing.json | 20 ++ ...r-service-model-workflow-contract.test.mjs | 111 ++++++ ...date-vi-history-dispatch-contract.test.mjs | 7 +- 10 files changed, 1304 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/pester-evidence.yml create mode 100644 .github/workflows/pester-gate.yml create mode 100644 .github/workflows/pester-run.yml create mode 100644 .github/workflows/pester-service-model-on-label.yml create mode 100644 .github/workflows/selfhosted-readiness.yml create mode 100644 docs/knowledgebase/Pester-Service-Model.md create mode 100644 tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs diff --git a/.github/workflows/pester-evidence.yml b/.github/workflows/pester-evidence.yml new file mode 100644 index 000000000..a68ec5e21 --- /dev/null +++ b/.github/workflows/pester-evidence.yml @@ -0,0 +1,313 @@ +name: Pester evidence + +on: + workflow_call: + inputs: + raw_artifact_name: + required: false + type: string + default: 'pester-run-raw' + dispatcher_exit_code: + required: false + type: string + default: '' + readiness_status: + required: false + type: string + default: 'unknown' + execution_job_result: + required: false + type: string + default: '' + continue_on_error: + required: false + type: string + default: 'false' + outputs: + classification: + description: 'Final evidence classification' + value: ${{ jobs.evidence.outputs.classification }} + total: + description: 'Total tests discovered' + value: ${{ jobs.evidence.outputs.total }} + passed: + description: 'Passed tests count' + value: ${{ jobs.evidence.outputs.passed }} + failed: + description: 'Failed tests count' + value: ${{ jobs.evidence.outputs.failed }} + errors: + description: 'Errors count' + value: ${{ jobs.evidence.outputs.errors }} + duration_s: + description: 'Duration in seconds' + value: ${{ jobs.evidence.outputs.duration_s }} + workflow_dispatch: + inputs: + raw_artifact_name: + description: 'Raw execution artifact name' + required: false + default: 'pester-run-raw' + type: string + dispatcher_exit_code: + description: 'Dispatcher exit code from execution workflow' + required: false + default: '' + type: string + readiness_status: + description: 'Readiness receipt status' + required: false + default: 'unknown' + type: string + execution_job_result: + description: 'Execution job result' + required: false + default: '' + type: string + continue_on_error: + description: 'Treat failing evidence as notice-only' + required: false + default: 'false' + type: choice + options: ['false', 'true'] + +jobs: + evidence: + runs-on: ubuntu-latest + outputs: + classification: ${{ steps.classify.outputs.classification }} + total: ${{ steps.export.outputs.total }} + passed: ${{ steps.export.outputs.passed }} + failed: ${{ steps.export.outputs.failed }} + errors: ${{ steps.export.outputs.errors }} + duration_s: ${{ steps.export.outputs.duration_s }} + steps: + - uses: actions/checkout@v5 + + - name: Resolve raw artifact name + id: artifact_name + shell: pwsh + run: | + $artifactName = '${{ inputs.raw_artifact_name }}' + if ([string]::IsNullOrWhiteSpace($artifactName)) { $artifactName = 'pester-run-raw' } + "name=$artifactName" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + + - name: Download raw execution artifact + id: download + continue-on-error: true + uses: actions/download-artifact@v5 + with: + name: ${{ steps.artifact_name.outputs.name }} + path: . + + - name: Ensure results directory + shell: pwsh + run: | + if (-not (Test-Path -LiteralPath 'tests/results')) { + New-Item -ItemType Directory -Path 'tests/results' -Force | Out-Null + } + + - name: Validate execution receipt artifact + id: execution_receipt + if: always() + shell: pwsh + run: | + $receiptPath = Join-Path 'tests/results' 'pester-run-receipt.json' + if (-not (Test-Path -LiteralPath $receiptPath)) { + "present=false" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "status=missing" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + exit 0 + } + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -ErrorAction Stop + if ($receipt.schema -ne 'pester-execution-receipt@v1') { + throw ("Unexpected execution receipt schema: {0}" -f $receipt.schema) + } + "present=true" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "status=$($receipt.status)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "dispatcher_exit_code=$($receipt.dispatcherExitCode)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + + - name: Publish Pester summary + if: always() + continue-on-error: true + shell: pwsh + run: pwsh -File scripts/Write-PesterSummaryToStepSummary.ps1 -ResultsDir 'tests/results' + + - name: Validate Pester summary schema-lite (notice-only) + if: always() + continue-on-error: true + shell: pwsh + run: | + $json = Join-Path 'tests/results' 'pester-summary.json' + if (Test-Path $json) { + $schemas = @( + 'docs/schemas/pester-summary-v1_7_1.schema.json', + 'docs/schemas/pester-summary-v1_7.schema.json', + 'docs/schemas/pester-summary-v1_6.schema.json', + 'docs/schemas/pester-summary-v1_5.schema.json', + 'docs/schemas/pester-summary-v1_4.schema.json', + 'docs/schemas/pester-summary-v1_3.schema.json', + 'docs/schemas/pester-summary-v1_2.schema.json', + 'docs/schemas/pester-summary-v1_1.schema.json' + ) + foreach ($schema in $schemas) { + if (Test-Path $schema) { + pwsh -File tools/Invoke-JsonSchemaLite.ps1 -JsonPath $json -SchemaPath $schema + if ($LASTEXITCODE -eq 0) { break } + } + } + } + + - name: Ensure session index (fallback) + if: always() + continue-on-error: true + shell: pwsh + run: pwsh -File tools/Ensure-SessionIndex.ps1 -ResultsDir 'tests/results' -SummaryJson 'pester-summary.json' + + - name: Wire Probe (S1) + if: always() + uses: ./.github/actions/wire-probe + with: + phase: S1 + results-dir: tests/results + + - name: Export Pester totals as outputs + id: export + if: always() + shell: pwsh + run: | + $sum = Join-Path 'tests/results' 'pester-summary.json' + if (Test-Path $sum) { + $js = Get-Content $sum -Raw | ConvertFrom-Json + "total=$($js.total)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "passed=$($js.passed)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "failed=$($js.failed)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "errors=$($js.errors)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "duration_s=$($js.duration_s)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + } else { + "total=0" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "passed=0" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "failed=0" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "errors=0" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "duration_s=0" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + } + + - name: Session index post + if: always() + uses: ./.github/actions/session-index-post + with: + results-dir: tests/results + validate-schema: true + upload: true + artifact-name: session-index + + - name: Append session summary + if: always() + continue-on-error: true + shell: pwsh + run: pwsh -File tools/Write-SessionIndexSummary.ps1 -ResultsDir 'tests/results' + + - name: Append top Pester failures + if: always() + continue-on-error: true + shell: pwsh + run: pwsh -File tools/Write-PesterTopFailures.ps1 -ResultsDir 'tests/results' -Top 10 + + - name: Write compact totals JSON + if: always() + shell: pwsh + run: | + $outDir = 'tests/results' + $sum = Join-Path $outDir 'pester-summary.json' + $obj = [ordered]@{ + schema = 'pester-totals/v1' + includeIntegration = $null + status = 'missing-summary' + } + if (Test-Path $sum) { + try { + $js = Get-Content $sum -Raw | ConvertFrom-Json -ErrorAction Stop + $obj.total = $js.total + $obj.passed = $js.passed + $obj.failed = $js.failed + $obj.errors = $js.errors + $obj.duration_s = $js.duration_s + $obj.status = if (($js.failed + $js.errors) -gt 0) { 'fail' } else { 'ok' } + } catch { + $obj.status = 'unknown' + } + } + $obj | ConvertTo-Json -Depth 5 | Out-File -FilePath (Join-Path $outDir 'pester-totals.json') -Encoding utf8 + + - name: Classify evidence outcome + id: classify + if: always() + shell: pwsh + run: | + $resultsDir = 'tests/results' + $summaryPath = Join-Path $resultsDir 'pester-summary.json' + $classification = 'seam-defect' + $reasons = New-Object System.Collections.Generic.List[string] + $executionReceiptPresent = '${{ steps.execution_receipt.outputs.present }}' + $executionReceiptStatus = '${{ steps.execution_receipt.outputs.status }}' + if ('${{ inputs.readiness_status }}' -ne 'ready') { + $reasons.Add(("readiness-status={0}" -f '${{ inputs.readiness_status }}')) | Out-Null + } + if ('${{ inputs.execution_job_result }}' -eq 'skipped') { + $reasons.Add('execution-job-skipped') | Out-Null + } + $dispatcherExitCode = '${{ inputs.dispatcher_exit_code }}' + if ([string]::IsNullOrWhiteSpace($dispatcherExitCode)) { $dispatcherExitCode = '-1' } + if ($executionReceiptPresent -ne 'true') { + $reasons.Add('execution-receipt-missing') | Out-Null + } elseif ($executionReceiptStatus -eq 'seam-defect') { + $reasons.Add('execution-receipt-seam-defect') | Out-Null + } elseif ($executionReceiptStatus -eq 'test-failures') { + $classification = 'test-failures' + } elseif (Test-Path -LiteralPath $summaryPath) { + try { + $summary = Get-Content -LiteralPath $summaryPath -Raw | ConvertFrom-Json -ErrorAction Stop + if ('${{ steps.execution_receipt.outputs.dispatcher_exit_code }}' -and '${{ steps.execution_receipt.outputs.dispatcher_exit_code }}' -ne $dispatcherExitCode) { + $reasons.Add('dispatcher-exit-mismatch') | Out-Null + } + if (($summary.failed + $summary.errors) -gt 0 -or $dispatcherExitCode -ne '0') { + $classification = 'test-failures' + } else { + $classification = 'ok' + } + } catch { + $classification = 'seam-defect' + $reasons.Add('summary-unparseable') | Out-Null + } + } else { + $reasons.Add('summary-missing') | Out-Null + } + $receipt = [ordered]@{ + schema = 'pester-evidence-classification@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + readinessStatus = '${{ inputs.readiness_status }}' + executionJobResult = '${{ inputs.execution_job_result }}' + dispatcherExitCode = [int]$dispatcherExitCode + summaryPresent = Test-Path -LiteralPath $summaryPath + classification = $classification + reasons = @($reasons) + } + $receipt | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-evidence-classification.json') -Encoding UTF8 + "classification=$classification" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + + - name: Generate dev dashboard report + if: always() + continue-on-error: true + shell: pwsh + run: pwsh -File tools/Invoke-DevDashboard.ps1 -Group 'pester-selfhosted' -ResultsRoot 'tests/results' + + - name: Upload evidence artifact + if: always() + uses: actions/upload-artifact@v7 + with: + name: pester-evidence + path: tests/results + if-no-files-found: warn + + - name: Propagate gate outcome + if: ${{ steps.classify.outputs.classification != 'ok' && inputs.continue_on_error != 'true' }} + run: exit 1 diff --git a/.github/workflows/pester-gate.yml b/.github/workflows/pester-gate.yml new file mode 100644 index 000000000..0824f9af8 --- /dev/null +++ b/.github/workflows/pester-gate.yml @@ -0,0 +1,88 @@ +name: Pester gate (service model pilot) + +on: + workflow_call: + inputs: + include_integration: + required: false + default: 'false' + type: string + include_patterns: + required: false + default: '' + type: string + sample_id: + required: false + default: '' + type: string + checkout_repository: + required: false + default: '' + type: string + checkout_ref: + required: false + default: '' + type: string + workflow_dispatch: + inputs: + include_integration: + description: "Include Integration-tagged tests in the execution pack" + required: false + default: 'false' + type: choice + options: ['false', 'true'] + include_patterns: + description: 'Optional repo-relative IncludePatterns selector' + required: false + default: '' + type: string + sample_id: + description: 'Sampling correlation id (prevents cancels)' + required: false + default: '' + type: string + checkout_repository: + description: 'Repository to checkout for the pilot run' + required: false + default: '' + type: string + checkout_ref: + description: 'Git ref or SHA to checkout for the pilot run' + required: false + default: '' + type: string + +concurrency: + group: ${{ github.workflow }}-${{ github.event.inputs.sample_id || github.ref }} + cancel-in-progress: true + +jobs: + readiness: + uses: ./.github/workflows/selfhosted-readiness.yml + with: + sample_id: ${{ inputs.sample_id || '' }} + checkout_repository: ${{ inputs.checkout_repository || github.repository }} + checkout_ref: ${{ inputs.checkout_ref || github.sha }} + + pester-run: + needs: readiness + if: always() + uses: ./.github/workflows/pester-run.yml + with: + include_integration: ${{ inputs.include_integration || 'false' }} + include_patterns: ${{ inputs.include_patterns || '' }} + sample_id: ${{ inputs.sample_id || '' }} + readiness_status: ${{ needs.readiness.outputs.receipt_status }} + readiness_artifact_name: ${{ needs.readiness.outputs.receipt_artifact_name }} + checkout_repository: ${{ inputs.checkout_repository || github.repository }} + checkout_ref: ${{ inputs.checkout_ref || github.sha }} + + pester-evidence: + needs: [readiness, pester-run] + if: always() + uses: ./.github/workflows/pester-evidence.yml + with: + raw_artifact_name: ${{ needs.pester-run.outputs.raw_artifact_name }} + dispatcher_exit_code: ${{ needs.pester-run.outputs.dispatcher_exit_code }} + readiness_status: ${{ needs.readiness.outputs.receipt_status }} + execution_job_result: ${{ needs.pester-run.result }} diff --git a/.github/workflows/pester-run.yml b/.github/workflows/pester-run.yml new file mode 100644 index 000000000..5aef347c8 --- /dev/null +++ b/.github/workflows/pester-run.yml @@ -0,0 +1,332 @@ +name: Pester run + +on: + workflow_call: + inputs: + include_integration: + required: false + type: string + default: 'false' + include_patterns: + required: false + type: string + default: '' + sample_id: + required: false + type: string + readiness_status: + required: false + type: string + default: 'unknown' + readiness_artifact_name: + required: false + type: string + default: 'pester-readiness' + checkout_repository: + required: false + type: string + checkout_ref: + required: false + type: string + outputs: + dispatcher_exit_code: + description: 'Dispatcher exit code from the self-hosted execution pass' + value: ${{ jobs.pester.outputs.dispatcher_exit_code }} + raw_artifact_name: + description: 'Artifact name containing raw execution outputs' + value: ${{ jobs.pester.outputs.raw_artifact_name }} + workflow_dispatch: + inputs: + include_integration: + description: "Include Integration-tagged tests in the execution pack" + required: false + default: 'false' + type: choice + options: ['false', 'true'] + include_patterns: + description: 'Optional repo-relative IncludePatterns selector' + required: false + default: '' + type: string + sample_id: + description: 'Sampling correlation id (prevents cancels)' + required: false + default: '' + type: string + readiness_status: + description: 'Readiness receipt status' + required: false + default: 'unknown' + type: string + readiness_artifact_name: + description: 'Readiness receipt artifact name' + required: false + default: 'pester-readiness' + type: string + checkout_repository: + description: 'Repository to checkout for execution' + required: false + default: '' + type: string + checkout_ref: + description: 'Git ref or SHA to checkout for execution' + required: false + default: '' + type: string + +concurrency: + group: ${{ github.workflow }}-pester-run-${{ github.event.inputs.sample_id || inputs.sample_id || github.ref }} + cancel-in-progress: true + +jobs: + normalize: + runs-on: ubuntu-latest + outputs: + include_integration: ${{ steps.b.outputs.normalized }} + steps: + - uses: actions/checkout@v5 + with: + repository: ${{ inputs.checkout_repository || github.repository }} + ref: ${{ inputs.checkout_ref || github.sha }} + - name: Apply determinism profile + uses: ./.github/actions/determinism-profile + with: + strict: 'true' + - name: Normalize include_integration + id: b + uses: ./.github/actions/bool-normalize + with: + value: ${{ inputs.include_integration || 'false' }} + + pester: + name: Pester (execution only) + if: ${{ inputs.readiness_status == 'ready' }} + runs-on: [self-hosted, Windows, X64, comparevi, capability-ingress] + needs: normalize + env: + LV_SUPPRESS_UI: ${{ vars.LV_SUPPRESS_UI || '1' }} + LV_NO_ACTIVATE: ${{ vars.LV_NO_ACTIVATE || '1' }} + LV_CURSOR_RESTORE: ${{ vars.LV_CURSOR_RESTORE || '1' }} + LV_IDLE_WAIT_SECONDS: ${{ vars.LV_IDLE_WAIT_SECONDS || '2' }} + LV_IDLE_MAX_WAIT_SECONDS: ${{ vars.LV_IDLE_MAX_WAIT_SECONDS || '5' }} + STUCK_GUARD: ${{ vars.STUCK_GUARD || '1' }} + CLEAN_LV_BEFORE: ${{ vars.CLEAN_LV_BEFORE || 'true' }} + CLEAN_LV_AFTER: ${{ vars.CLEAN_LV_AFTER || 'true' }} + CLEAN_LV_INCLUDE_COMPARE: ${{ vars.CLEAN_LV_INCLUDE_COMPARE || vars.CLEAN_LVCOMPARE || 'true' }} + LVCOMPARE_PATH: ${{ vars.LVCOMPARE_PATH || '' }} + outputs: + dispatcher_exit_code: ${{ steps.execution_receipt.outputs.dispatcher_exit_code }} + raw_artifact_name: ${{ steps.execution_receipt.outputs.raw_artifact_name }} + steps: + - uses: actions/checkout@v5 + + - name: Install Node dependencies + shell: pwsh + run: node tools/npm/cli.mjs ci + + - name: Export workflow token for priority sync + shell: pwsh + env: + WORKFLOW_TOKEN: ${{ github.token }} + run: | + if (-not $env:WORKFLOW_TOKEN) { throw 'github.token is empty' } + "GH_TOKEN=$env:WORKFLOW_TOKEN" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + "GITHUB_TOKEN=$env:WORKFLOW_TOKEN" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + + - name: Download readiness receipt artifact + uses: actions/download-artifact@v5 + with: + name: ${{ inputs.readiness_artifact_name }} + path: tests/readiness + + - name: Validate readiness receipt + id: readiness_receipt + shell: pwsh + run: | + $receiptPath = 'tests/readiness/selfhosted-readiness.json' + if (-not (Test-Path -LiteralPath $receiptPath)) { + throw "Readiness receipt missing: $receiptPath" + } + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -ErrorAction Stop + if ($receipt.schema -ne 'pester-selfhosted-readiness-receipt@v1') { + throw ("Unexpected readiness receipt schema: {0}" -f $receipt.schema) + } + if ($receipt.status -ne 'ready') { + throw ("Readiness receipt status is not ready: {0}" -f $receipt.status) + } + $freshnessWindowSeconds = 900 + if ($receipt.PSObject.Properties.Name -contains 'freshnessWindowSeconds') { + $freshnessWindowSeconds = [int]$receipt.freshnessWindowSeconds + } + $generatedAtUtc = [DateTime]::Parse($receipt.generatedAtUtc).ToUniversalTime() + $ageSeconds = [math]::Floor(([DateTime]::UtcNow - $generatedAtUtc).TotalSeconds) + if ($ageSeconds -gt $freshnessWindowSeconds) { + throw ("Readiness receipt stale: age {0}s exceeds freshness window {1}s" -f $ageSeconds, $freshnessWindowSeconds) + } + "path=$receiptPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "generated_at_utc=$($generatedAtUtc.ToString('o'))" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "freshness_window_seconds=$freshnessWindowSeconds" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + + - name: Acquire session lock + shell: pwsh + run: pwsh -NoLogo -NoProfile -File tools/Session-Lock.ps1 -Action Acquire -Group 'pester-selfhosted' -QueueWaitSeconds 15 -QueueMaxAttempts 40 -StaleSeconds 300 -HeartbeatSeconds 15 + + - name: LV Guard (pre) + uses: ./.github/actions/runner-unblock-guard + with: + snapshot-path: tests/results/lv-guard-pre.json + cleanup: ${{ env.CLEAN_LV_BEFORE == 'true' }} + process-names: 'LVCompare,LabVIEW' + + - name: Prepare fixture copies (base/head) + if: ${{ needs.normalize.outputs.include_integration == 'true' }} + id: fixtures + uses: ./.github/actions/prepare-fixtures + + - name: Export fixture env for tests + if: ${{ needs.normalize.outputs.include_integration == 'true' }} + shell: pwsh + run: | + if ('${{ steps.fixtures.outputs.base }}' -and '${{ steps.fixtures.outputs.head }}') { + "LV_BASE_VI=${{ steps.fixtures.outputs.base }}" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + "LV_HEAD_VI=${{ steps.fixtures.outputs.head }}" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + } else { + Write-Host '::warning::prepare-fixtures did not emit expected outputs; proceeding without LV_* env.' + } + + - name: Apply dispatcher profile + id: dprofile + uses: ./.github/actions/dispatcher-profile + with: + timeout-seconds: '0' + emit-failures-json-always: 'true' + detect-leaks: 'true' + fail-on-leaks: 'false' + kill-leaks: 'false' + leak-grace-seconds: '3' + clean-labview-before: 'false' + clean-after: 'false' + track-artifacts: 'true' + + - name: Wire Probe (T1) + if: ${{ vars.WIRE_PROBES != '0' }} + uses: ./.github/actions/wire-probe + with: + phase: T1 + results-dir: tests/results + + - name: Run Pester tests via local dispatcher + id: dispatcher + continue-on-error: true + shell: pwsh + run: | + $logPath = 'tests/results/pester-dispatcher.log' + $logDir = Split-Path -Parent $logPath + if (-not (Test-Path $logDir)) { New-Item -ItemType Directory -Force -Path $logDir | Out-Null } + if (Test-Path -LiteralPath $logPath) { Remove-Item -LiteralPath $logPath -Force } + $dispatcherPath = Join-Path (Get-Location) 'Invoke-PesterTests.ps1' + if (-not (Test-Path -LiteralPath $dispatcherPath -PathType Leaf)) { + throw "Dispatcher script not found at $dispatcherPath" + } + $bound = [ordered]@{} + $bound.TestsPath = 'tests' + $includeIntegration = '${{ needs.normalize.outputs.include_integration }}' + if ($includeIntegration) { + switch ($includeIntegration.ToString().ToLowerInvariant()) { + { $_ -in @('true','1','yes','y','on','include') } { $bound.IntegrationMode = 'include' } + { $_ -in @('false','0','no','n','off','exclude') } { $bound.IntegrationMode = 'exclude' } + 'auto' { $bound.IntegrationMode = 'auto' } + } + } + $bound.ResultsPath = 'tests/results' + if ($env:DISPATCHER_LIVE_OUTPUT -ne '0') { $bound.LiveOutput = $true } + if ('${{ steps.dprofile.outputs.emit_failures_json_always }}' -eq 'true') { $bound.EmitFailuresJsonAlways = $true } + if ('${{ inputs.include_patterns }}' -ne '') { + $bound.IncludePatterns = (Split-Path -Leaf '${{ inputs.include_patterns }}') + } + $timeoutSeconds = '${{ steps.dprofile.outputs.timeout_seconds }}' + if ($timeoutSeconds -and $timeoutSeconds -match '^-?\d+(\.\d+)?$') { + $bound.TimeoutSeconds = [double]$timeoutSeconds + } + $lockScript = Join-Path (Get-Location) 'tools/Session-Lock.ps1' + $heartbeatSeconds = 15 + $heartbeatJob = Start-ThreadJob -ScriptBlock { + param($scriptPath,$seconds) + while ($true) { + pwsh -NoLogo -NoProfile -File $scriptPath -Action Heartbeat | Out-Null + Start-Sleep -Seconds $seconds + } + } -ArgumentList $lockScript, $heartbeatSeconds + $exitCode = 0 + try { + & $dispatcherPath @bound 2>&1 | Tee-Object -FilePath $logPath + $exitCode = $LASTEXITCODE + } finally { + if ($heartbeatJob) { + Stop-Job -Id $heartbeatJob.Id | Out-Null + Remove-Job -Id $heartbeatJob.Id -Force | Out-Null + } + pwsh -NoLogo -NoProfile -File $lockScript -Action Heartbeat | Out-Null + } + if (-not $env:GITHUB_OUTPUT) { + $fallbackOutput = Join-Path (Get-Location) 'tests/results/dispatcher-github-output.txt' + $fallbackDir = Split-Path -Parent $fallbackOutput + if (-not (Test-Path -LiteralPath $fallbackDir)) { + New-Item -ItemType Directory -Path $fallbackDir -Force | Out-Null + } + $env:GITHUB_OUTPUT = [System.IO.Path]::GetFullPath($fallbackOutput) + } + "exit_code=$exitCode" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + $global:LASTEXITCODE = 0 + + - name: Release session lock + if: always() + shell: pwsh + run: pwsh -NoLogo -NoProfile -File tools/Session-Lock.ps1 -Action Release + + - name: Write execution receipt + id: execution_receipt + if: always() + shell: pwsh + run: | + $resultsDir = 'tests/results' + if (-not (Test-Path -LiteralPath $resultsDir)) { + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + } + $summaryPath = Join-Path $resultsDir 'pester-summary.json' + $dispatcherExitCode = '${{ steps.dispatcher.outputs.exit_code }}' + $status = 'seam-defect' + if ($dispatcherExitCode -eq '') { + $dispatcherExitCode = '-1' + } + $readinessReceiptPresent = Test-Path -LiteralPath '${{ steps.readiness_receipt.outputs.path }}' + if ((Test-Path -LiteralPath $summaryPath) -and $dispatcherExitCode -eq '0') { + $status = 'completed' + } elseif (Test-Path -LiteralPath $summaryPath) { + $status = 'test-failures' + } + $receipt = [ordered]@{ + schema = 'pester-execution-receipt@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + readinessStatus = '${{ inputs.readiness_status }}' + readinessReceiptPath = '${{ steps.readiness_receipt.outputs.path }}' + readinessReceiptPresent = $readinessReceiptPresent + readinessReceiptGeneratedAtUtc = '${{ steps.readiness_receipt.outputs.generated_at_utc }}' + readinessReceiptFreshnessWindowSeconds = '${{ steps.readiness_receipt.outputs.freshness_window_seconds }}' + dispatcherExitCode = [int]$dispatcherExitCode + summaryPresent = Test-Path -LiteralPath $summaryPath + status = $status + rawArtifactName = 'pester-run-raw' + } + $receiptPath = Join-Path $resultsDir 'pester-run-receipt.json' + $receipt | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $receiptPath -Encoding UTF8 + "dispatcher_exit_code=$dispatcherExitCode" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "raw_artifact_name=pester-run-raw" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + + - name: Upload raw Pester execution artifact + if: always() + uses: actions/upload-artifact@v7 + with: + name: pester-run-raw + path: tests/results + if-no-files-found: warn diff --git a/.github/workflows/pester-service-model-on-label.yml b/.github/workflows/pester-service-model-on-label.yml new file mode 100644 index 000000000..f9728fe44 --- /dev/null +++ b/.github/workflows/pester-service-model-on-label.yml @@ -0,0 +1,114 @@ +name: Pester service-model pilot on trusted PR label + +on: + pull_request_target: + types: [labeled, reopened, synchronize] + branches: [main, develop, release/**] + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + sample_id: + description: 'Sampling correlation id (prevents cancels)' + required: false + default: '' + type: string + checkout_repository: + description: 'Repository to checkout for the pilot run' + required: false + default: '' + type: string + checkout_ref: + description: 'Git ref or SHA to checkout for the pilot run' + required: false + default: '' + type: string + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.inputs.sample_id || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: read + +jobs: + trust-context: + runs-on: ubuntu-latest + outputs: + should_run: ${{ steps.decide.outputs.should_run }} + checkout_repository: ${{ steps.decide.outputs.checkout_repository }} + checkout_ref: ${{ steps.decide.outputs.checkout_ref }} + trust_mode: ${{ steps.decide.outputs.trust_mode }} + reason: ${{ steps.decide.outputs.reason }} + steps: + - name: Decide trusted pilot routing + id: decide + shell: pwsh + env: + PR_LABELS_JSON: ${{ toJson(github.event.pull_request.labels.*.name) }} + run: | + $eventName = '${{ github.event_name }}' + $shouldRun = 'false' + $checkoutRepository = '${{ github.repository }}' + $checkoutRef = '${{ github.sha }}' + $trustMode = 'base-ref' + $reason = 'workflow-dispatch-default' + if ($eventName -eq 'workflow_dispatch') { + if ('${{ inputs.checkout_repository }}') { $checkoutRepository = '${{ inputs.checkout_repository }}' } + if ('${{ inputs.checkout_ref }}') { $checkoutRef = '${{ inputs.checkout_ref }}' } + $shouldRun = 'true' + } else { + $labels = @() + if ($env:PR_LABELS_JSON) { + try { + $labels = @((ConvertFrom-Json -InputObject $env:PR_LABELS_JSON -ErrorAction Stop) | Where-Object { $_ }) + } catch { + throw ("Unable to parse PR labels JSON: {0}" -f $_.Exception.Message) + } + } + $hasLabel = $labels -contains 'pester-service-model' + $sameOwner = '${{ github.event.pull_request.head.repo.owner.login }}' -eq '${{ github.repository_owner }}' + if (-not $hasLabel) { + $reason = 'pilot-label-missing' + } elseif (-not $sameOwner) { + $reason = 'untrusted-cross-owner-fork' + } else { + $checkoutRepository = '${{ github.event.pull_request.head.repo.full_name }}' + $checkoutRef = '${{ github.event.pull_request.head.sha }}' + $trustMode = 'same-owner-head' + $shouldRun = 'true' + $reason = 'trusted-same-owner-head' + } + } + "should_run=$shouldRun" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "checkout_repository=$checkoutRepository" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "checkout_ref=$checkoutRef" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "trust_mode=$trustMode" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "reason=$reason" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + + pilot: + needs: trust-context + if: ${{ needs.trust-context.outputs.should_run == 'true' }} + uses: ./.github/workflows/pester-gate.yml + with: + include_integration: ${{ 'true' }} + sample_id: ${{ github.event.inputs.sample_id || '' }} + checkout_repository: ${{ needs.trust-context.outputs.checkout_repository }} + checkout_ref: ${{ needs.trust-context.outputs.checkout_ref }} + + skipped: + needs: trust-context + if: ${{ needs.trust-context.outputs.should_run != 'true' }} + runs-on: ubuntu-latest + steps: + - name: Append skip summary + shell: pwsh + run: | + if ($env:GITHUB_STEP_SUMMARY) { + $lines = @('### Pester service-model pilot', '') + $lines += ('- Status: skipped') + $lines += ('- Reason: {0}' -f '${{ needs.trust-context.outputs.reason }}') + $lines += ('- Trust mode: {0}' -f '${{ needs.trust-context.outputs.trust_mode }}') + $lines -join "`n" | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append -Encoding utf8 + } diff --git a/.github/workflows/selfhosted-readiness.yml b/.github/workflows/selfhosted-readiness.yml new file mode 100644 index 000000000..a9a15c716 --- /dev/null +++ b/.github/workflows/selfhosted-readiness.yml @@ -0,0 +1,238 @@ +name: Self-hosted readiness + +on: + workflow_call: + inputs: + sample_id: + required: false + type: string + checkout_repository: + required: false + type: string + checkout_ref: + required: false + type: string + outputs: + receipt_status: + description: 'Overall readiness status for the self-hosted ingress plane' + value: ${{ jobs.readiness.outputs.receipt_status }} + receipt_artifact_name: + description: 'Artifact name containing the readiness receipt bundle' + value: ${{ jobs.readiness.outputs.receipt_artifact_name }} + workflow_dispatch: + inputs: + sample_id: + description: 'Sampling correlation id (prevents cancels)' + required: false + default: '' + type: string + checkout_repository: + description: 'Repository to checkout for readiness and execution context' + required: false + default: '' + type: string + checkout_ref: + description: 'Git ref or SHA to checkout for readiness and execution context' + required: false + default: '' + type: string + +concurrency: + group: ${{ github.workflow }}-${{ github.event.inputs.sample_id || inputs.sample_id || github.ref }} + cancel-in-progress: true + +jobs: + readiness: + runs-on: [self-hosted, Windows, X64, comparevi, capability-ingress] + outputs: + receipt_status: ${{ steps.receipt.outputs.status }} + receipt_artifact_name: ${{ steps.receipt.outputs.artifact_name }} + env: + LVCOMPARE_PATH: ${{ vars.LVCOMPARE_PATH || '' }} + LV_IDLE_WAIT_SECONDS: ${{ vars.LV_IDLE_WAIT_SECONDS || '2' }} + LV_IDLE_MAX_WAIT_SECONDS: ${{ vars.LV_IDLE_MAX_WAIT_SECONDS || '5' }} + steps: + - uses: actions/checkout@v5 + with: + repository: ${{ inputs.checkout_repository || github.repository }} + ref: ${{ inputs.checkout_ref || github.sha }} + + - name: Install Node dependencies + shell: pwsh + run: node tools/npm/cli.mjs ci + + - name: Validate runner label contract + id: runner_labels + continue-on-error: true + shell: pwsh + run: | + $outDir = 'tests/results/pester-readiness' + New-Item -ItemType Directory -Force -Path $outDir | Out-Null + $labels = if ($env:RUNNER_LABELS) { + @($env:RUNNER_LABELS -split ',' | ForEach-Object { $_.Trim() } | Where-Object { $_ }) + } else { + @() + } + if ($labels.Count -eq 0) { + $labels = @('self-hosted', 'Windows', 'X64', 'comparevi', 'capability-ingress') + } + $required = @('self-hosted', 'Windows', 'X64', 'comparevi', 'capability-ingress') + $missing = @($required | Where-Object { $labels -notcontains $_ }) + $status = if ($missing.Count -eq 0) { 'ready' } else { 'not-ready' } + $report = [ordered]@{ + schema = 'pester/selfhosted-runner-labels@v1' + status = $status + labels = $labels + required = $required + missing = $missing + } + $reportPath = Join-Path $outDir 'runner-label-contract.json' + $report | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $reportPath -Encoding UTF8 + if ($status -ne 'ready') { + throw ("Runner label contract missing required labels: {0}" -f ($missing -join ', ')) + } + + - name: Probe session-lock health + id: session_lock + continue-on-error: true + shell: pwsh + run: | + $lockScript = Join-Path (Get-Location) 'tools/Session-Lock.ps1' + if (-not (Test-Path -LiteralPath $lockScript -PathType Leaf)) { + throw "Session-Lock.ps1 not found: $lockScript" + } + pwsh -NoLogo -NoProfile -File $lockScript -Action Acquire -Group 'pester-selfhosted-readiness' -QueueWaitSeconds 5 -QueueMaxAttempts 3 -StaleSeconds 120 -HeartbeatSeconds 5 + try { + pwsh -NoLogo -NoProfile -File $lockScript -Action Heartbeat | Out-Null + } finally { + pwsh -NoLogo -NoProfile -File $lockScript -Action Release | Out-Null + } + + - name: Resolve .NET host toolchain + id: dotnet + continue-on-error: true + shell: pwsh + run: | + $outDir = 'tests/results/pester-readiness' + New-Item -ItemType Directory -Force -Path $outDir | Out-Null + $candidates = New-Object System.Collections.Generic.List[string] + if ($env:COMPAREVI_DOTNET_EXE) { $candidates.Add($env:COMPAREVI_DOTNET_EXE) | Out-Null } + try { + $cmd = Get-Command dotnet -ErrorAction Stop | Select-Object -First 1 + if ($cmd -and $cmd.Source) { $candidates.Add([string]$cmd.Source) | Out-Null } + } catch {} + $candidates.Add('C:\Program Files\dotnet\dotnet.exe') | Out-Null + $candidates.Add('C:\Program Files (x86)\dotnet\dotnet.exe') | Out-Null + $resolved = $null + foreach ($candidate in $candidates) { + if ([string]::IsNullOrWhiteSpace($candidate)) { continue } + if (Test-Path -LiteralPath $candidate) { + $resolved = (Resolve-Path -LiteralPath $candidate).Path + break + } + } + $status = if ($resolved) { 'ready' } else { 'not-ready' } + $report = [ordered]@{ + schema = 'pester/selfhosted-dotnet@v1' + status = $status + resolved = $resolved + candidates = @($candidates | Where-Object { $_ } | Select-Object -Unique) + } + $reportPath = Join-Path $outDir 'dotnet-readiness.json' + $report | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $reportPath -Encoding UTF8 + if (-not $resolved) { + throw '.NET SDK/CLI not available on the self-hosted ingress plane.' + } + & $resolved --info | Out-Host + "path=$resolved" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + + - name: Probe Windows Docker runtime + id: docker_runtime + continue-on-error: true + shell: pwsh + run: | + $outDir = 'tests/results/pester-readiness' + New-Item -ItemType Directory -Force -Path $outDir | Out-Null + $reportPath = Join-Path $outDir 'docker-runtime-manager.json' + pwsh -NoLogo -NoProfile -File tools/Invoke-DockerRuntimeManager.ps1 ` + -ProbeScope windows ` + -BootstrapWindowsImage:$false ` + -OutputJsonPath $reportPath ` + -SwitchRetryCount 1 ` + -SwitchTimeoutSeconds 30 + + - name: Verify LVCompare and idle LabVIEW state + id: lvcompare + continue-on-error: true + shell: pwsh + run: | + $outDir = 'tests/results/pester-readiness' + New-Item -ItemType Directory -Force -Path $outDir | Out-Null + $defaultCli = 'C:\Program Files\National Instruments\Shared\LabVIEW Compare\LVCompare.exe' + $candidates = @() + if ($env:LVCOMPARE_PATH) { $candidates += $env:LVCOMPARE_PATH } + $candidates += $defaultCli + $cli = $null + foreach ($p in $candidates) { + if ($p -and (Test-Path -LiteralPath $p)) { $cli = $p; break } + } + $lv = @(Get-Process -Name 'LabVIEW' -ErrorAction SilentlyContinue) + $status = if ($cli -and $lv.Count -eq 0) { 'ready' } else { 'not-ready' } + $report = [ordered]@{ + schema = 'pester/selfhosted-lvcompare@v1' + status = $status + resolvedCliPath = $cli + detectedLabVIEW = @($lv | ForEach-Object { [ordered]@{ pid = $_.Id; sessionId = $_.SessionId } }) + candidates = $candidates + } + $reportPath = Join-Path $outDir 'lvcompare-readiness.json' + $report | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $reportPath -Encoding UTF8 + if (-not $cli) { + throw ("LVCompare.exe not found on the ingress plane. Tried: {0}" -f (($candidates | Where-Object { $_ }) -join '; ')) + } + if ($lv.Count -gt 0) { + $pids = $lv | ForEach-Object { '{0}(session={1})' -f $_.Id,$_.SessionId } + throw ("LabVIEW.exe is running during readiness probe: {0}" -f ($pids -join ', ')) + } + + - name: Write readiness receipt + id: receipt + if: always() + shell: pwsh + run: | + $outDir = 'tests/results/pester-readiness' + New-Item -ItemType Directory -Force -Path $outDir | Out-Null + $probeOutcomes = [ordered]@{ + runnerLabels = '${{ steps.runner_labels.outcome }}' + sessionLock = '${{ steps.session_lock.outcome }}' + dotnet = '${{ steps.dotnet.outcome }}' + dockerRuntime = '${{ steps.docker_runtime.outcome }}' + lvcompare = '${{ steps.lvcompare.outcome }}' + } + $status = if (@($probeOutcomes.Values | Where-Object { $_ -ne 'success' }).Count -eq 0) { 'ready' } else { 'not-ready' } + $receipt = [ordered]@{ + schema = 'pester-selfhosted-readiness-receipt@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + freshnessWindowSeconds = 900 + status = $status + sampleId = '${{ inputs.sample_id || github.event.inputs.sample_id || '' }}' + probes = [ordered]@{ + runnerLabels = [ordered]@{ outcome = $probeOutcomes.runnerLabels; reportPath = 'tests/results/pester-readiness/runner-label-contract.json' } + sessionLock = [ordered]@{ outcome = $probeOutcomes.sessionLock } + dotnet = [ordered]@{ outcome = $probeOutcomes.dotnet; reportPath = 'tests/results/pester-readiness/dotnet-readiness.json'; path = '${{ steps.dotnet.outputs.path }}' } + dockerRuntime = [ordered]@{ outcome = $probeOutcomes.dockerRuntime; reportPath = 'tests/results/pester-readiness/docker-runtime-manager.json' } + lvcompare = [ordered]@{ outcome = $probeOutcomes.lvcompare; reportPath = 'tests/results/pester-readiness/lvcompare-readiness.json' } + } + } + $receiptPath = Join-Path $outDir 'selfhosted-readiness.json' + $receipt | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $receiptPath -Encoding UTF8 + "status=$status" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "artifact_name=pester-readiness" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + + - name: Upload readiness receipt + if: always() + uses: actions/upload-artifact@v7 + with: + name: pester-readiness + path: tests/results/pester-readiness + if-no-files-found: error diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index e7def6867..2aa6e22e5 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -1534,22 +1534,44 @@ jobs: 'capability-ingress', 'docker-lane' ) - pwsh -NoLogo -NoProfile -File tools/Resolve-SelfHostedWindowsLanePlan.ps1 ` - -Repository '${{ github.repository }}' ` - -RequiredLabels $requiredLabels ` - -ExecutionModel 'self-hosted-windows-docker-lane' ` - -RunnerImage 'self-hosted-windows-docker-lane' ` - -ExpectedContext 'desktop-windows' ` - -ExpectedOs 'windows' ` - -RequiredHealthReceipts @() ` - -Notes @( - 'Availability means an online, idle repository runner advertises the ingress plus docker-lane labels.', - 'The lane may mutate Docker Desktop into the Windows engine and then restores the starting context after proof capture.' - ) ` - -Token $env:GITHUB_TOKEN ` - -OutputJsonPath $planPath ` - -GitHubOutputPath $env:GITHUB_OUTPUT ` - -StepSummaryPath $env:GITHUB_STEP_SUMMARY + $requiredHealthReceipts = @() + $notes = @( + 'Availability means an online, idle repository runner advertises the ingress plus docker-lane labels.', + 'The lane may mutate Docker Desktop into the Windows engine and then restores the starting context after proof capture.' + ) + $args = @( + '-NoLogo', + '-NoProfile', + '-File', + 'tools/Resolve-SelfHostedWindowsLanePlan.ps1', + '-Repository', + '${{ github.repository }}', + '-RequiredLabels' + ) + $requiredLabels + @( + '-ExecutionModel', + 'self-hosted-windows-docker-lane', + '-RunnerImage', + 'self-hosted-windows-docker-lane', + '-ExpectedContext', + 'desktop-windows', + '-ExpectedOs', + 'windows', + '-Notes' + ) + $notes + @( + '-Token', + $env:GITHUB_TOKEN, + '-OutputJsonPath', + $planPath, + '-GitHubOutputPath', + $env:GITHUB_OUTPUT, + '-StepSummaryPath', + $env:GITHUB_STEP_SUMMARY + ) + if ($requiredHealthReceipts.Count -gt 0) { + $args += '-RequiredHealthReceipts' + $args += $requiredHealthReceipts + } + & pwsh @args - name: Append VI history Windows runner plan shell: pwsh diff --git a/docs/knowledgebase/Pester-Service-Model.md b/docs/knowledgebase/Pester-Service-Model.md new file mode 100644 index 000000000..5fe3d0d3a --- /dev/null +++ b/docs/knowledgebase/Pester-Service-Model.md @@ -0,0 +1,44 @@ +# Pester Service Model + +The legacy Pester control plane couples four concerns into one self-hosted transaction: + +1. policy routing +2. host-plane readiness +3. test execution +4. evidence generation + +That coupling is what makes the monolithic self-hosted seam expensive to reproduce and hard to localize when it stalls or emits `missing-summary`. + +## Pilot Split + +The additive pilot introduces four workflow surfaces: + +- `.github/workflows/pester-gate.yml` + - top-level router for the pilot service model +- `.github/workflows/pester-service-model-on-label.yml` + - trusted PR/dispatch entrypoint for proving the pilot without exposing the self-hosted ingress plane to untrusted fork heads +- `.github/workflows/selfhosted-readiness.yml` + - host-plane readiness receipts for the self-hosted ingress surface +- `.github/workflows/pester-run.yml` + - receipt-driven Pester execution only +- `.github/workflows/pester-evidence.yml` + - summary, classification, session-index, dashboard, and artifact publication + +## Design Rules + +- Readiness certifies the environment. It does not execute the test pack. +- Readiness emits a bounded-freshness receipt artifact that execution must download and validate before dispatch. +- Execution consumes readiness. It does not bootstrap Docker runtimes or install core toolchains. +- Execution writes an execution receipt before uploading raw artifacts so evidence can classify the real seam outcome. +- Evidence consumes raw execution output plus the execution receipt. It classifies `seam-defect` explicitly when execution never yields a valid summary or never yields a valid execution receipt. +- The existing required gate remains in place until the pilot proves equivalent or better behavior. +- Trusted PR proving must stay on `pull_request_target` with same-owner gating. Cross-owner fork heads are not allowed to drive self-hosted execution. + +## Promotion Rule + +The pilot can replace the monolith only after: + +- readiness receipts are stable on the ingress host +- execution runs the declared pack without host bootstrap +- evidence produces deterministic classifications +- PR/release comparisons show better failure localization and lower operator ambiguity diff --git a/tools/policy/runner-capability-routing.json b/tools/policy/runner-capability-routing.json index 16ae1c607..7d7af2346 100644 --- a/tools/policy/runner-capability-routing.json +++ b/tools/policy/runner-capability-routing.json @@ -81,6 +81,26 @@ } ] }, + { + "workflow": ".github/workflows/selfhosted-readiness.yml", + "jobs": [ + { + "id": "readiness", + "routingClass": "ingress-only", + "requiredCapabilityLabels": [] + } + ] + }, + { + "workflow": ".github/workflows/pester-run.yml", + "jobs": [ + { + "id": "pester", + "routingClass": "ingress-only", + "requiredCapabilityLabels": [] + } + ] + }, { "workflow": ".github/workflows/runbook-validation.yml", "jobs": [ diff --git a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs new file mode 100644 index 000000000..7b48ca9df --- /dev/null +++ b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs @@ -0,0 +1,111 @@ +#!/usr/bin/env node + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { readFileSync } from 'node:fs'; + +const repoRoot = process.cwd(); + +function readRepoFile(relativePath) { + return readFileSync(path.join(repoRoot, relativePath), 'utf8'); +} + +test('pester gate pilot routes readiness, execution, and evidence through separate reusable workflows', () => { + const workflow = readRepoFile('.github/workflows/pester-gate.yml'); + + assert.match(workflow, /name:\s+Pester gate \(service model pilot\)/); + assert.match(workflow, /workflow_call:/); + assert.match(workflow, /workflow_dispatch:/); + assert.match(workflow, /jobs:\s*\n\s*readiness:\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/selfhosted-readiness\.yml/); + assert.match(workflow, /\n\s*pester-run:\s*\n\s+needs:\s+readiness\s*\n\s+if:\s+always\(\)\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/pester-run\.yml/); + assert.match(workflow, /readiness_artifact_name:\s+\$\{\{\s*needs\.readiness\.outputs\.receipt_artifact_name\s*\}\}/); + assert.match(workflow, /checkout_repository:\s+\$\{\{\s*inputs\.checkout_repository \|\| github\.repository\s*\}\}/); + assert.match(workflow, /checkout_ref:\s+\$\{\{\s*inputs\.checkout_ref \|\| github\.sha\s*\}\}/); + assert.match(workflow, /\n\s*pester-evidence:\s*\n\s+needs:\s+\[readiness, pester-run\]\s*\n\s+if:\s+always\(\)\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/pester-evidence\.yml/); +}); + +test('selfhosted readiness owns host-plane certification and emits a receipt artifact', () => { + const workflow = readRepoFile('.github/workflows/selfhosted-readiness.yml'); + + assert.match(workflow, /name:\s+Self-hosted readiness/); + assert.match(workflow, /workflow_call:/); + assert.match(workflow, /receipt_status:/); + assert.match(workflow, /runs-on:\s*\[self-hosted, Windows, X64, comparevi, capability-ingress\]/); + assert.match(workflow, /repository:\s+\$\{\{\s*inputs\.checkout_repository \|\| github\.repository\s*\}\}/); + assert.match(workflow, /ref:\s+\$\{\{\s*inputs\.checkout_ref \|\| github\.sha\s*\}\}/); + assert.match(workflow, /Validate runner label contract/); + assert.match(workflow, /Probe session-lock health/); + assert.match(workflow, /Resolve \.NET host toolchain/); + assert.match(workflow, /Invoke-DockerRuntimeManager\.ps1/); + assert.match(workflow, /Verify LVCompare and idle LabVIEW state/); + assert.match(workflow, /Upload readiness receipt/); + assert.match(workflow, /pester-selfhosted-readiness/); + assert.match(workflow, /freshnessWindowSeconds = 900/); +}); + +test('pester run is execution-only and validates the readiness receipt before dispatch', () => { + const workflow = readRepoFile('.github/workflows/pester-run.yml'); + + assert.match(workflow, /name:\s+Pester run/); + assert.match(workflow, /name:\s+Pester \(execution only\)/); + assert.match(workflow, /if:\s+\$\{\{\s*inputs\.readiness_status == 'ready'\s*\}\}/); + assert.match(workflow, /repository:\s+\$\{\{\s*inputs\.checkout_repository \|\| github\.repository\s*\}\}/); + assert.match(workflow, /ref:\s+\$\{\{\s*inputs\.checkout_ref \|\| github\.sha\s*\}\}/); + assert.match(workflow, /Download readiness receipt artifact/); + assert.match(workflow, /Validate readiness receipt/); + assert.match(workflow, /selfhosted-readiness\.json/); + assert.match(workflow, /Run Pester tests via local dispatcher/); + assert.match(workflow, /pester-run-receipt\.json/); + assert.match(workflow, /Upload raw Pester execution artifact/); + assert.doesNotMatch(workflow, /Install Pester/); + assert.doesNotMatch(workflow, /Invoke-DockerRuntimeManager\.ps1/); + assert.doesNotMatch(workflow, /Write-PesterSummaryToStepSummary\.ps1/); + assert.doesNotMatch(workflow, /Invoke-DevDashboard\.ps1/); +}); + +test('pester evidence classifies seam defects explicitly from raw execution outputs', () => { + const workflow = readRepoFile('.github/workflows/pester-evidence.yml'); + + assert.match(workflow, /name:\s+Pester evidence/); + assert.match(workflow, /runs-on:\s+ubuntu-latest/); + assert.match(workflow, /Download raw execution artifact/); + assert.match(workflow, /Validate execution receipt artifact/); + assert.match(workflow, /execution-receipt-missing/); + assert.match(workflow, /Write-PesterSummaryToStepSummary\.ps1/); + assert.match(workflow, /Ensure-SessionIndex\.ps1/); + assert.match(workflow, /Invoke-DevDashboard\.ps1/); + assert.match(workflow, /classification = 'seam-defect'/); + assert.match(workflow, /execution-receipt-seam-defect/); + assert.match(workflow, /Upload evidence artifact/); + assert.match(workflow, /Propagate gate outcome/); +}); + +test('knowledgebase documents the additive service model and keeps the monolith as the current baseline', () => { + const doc = readRepoFile('docs/knowledgebase/Pester-Service-Model.md'); + + assert.match(doc, /legacy Pester control plane couples four concerns into one self-hosted transaction/i); + assert.match(doc, /selfhosted-readiness\.yml/); + assert.match(doc, /pester-run\.yml/); + assert.match(doc, /pester-evidence\.yml/); + assert.match(doc, /readiness receipt/i); + assert.match(doc, /execution receipt/i); + assert.match(doc, /existing required gate remains in place/i); +}); + +test('trusted PR pilot router only runs self-hosted service-model proof for workflow dispatch or same-owner labeled PR heads', () => { + const workflow = readRepoFile('.github/workflows/pester-service-model-on-label.yml'); + + assert.match(workflow, /name:\s+Pester service-model pilot on trusted PR label/); + assert.match(workflow, /pull_request_target:/); + assert.match(workflow, /types:\s*\[labeled, reopened, synchronize\]/); + assert.match(workflow, /workflow_dispatch:/); + assert.match(workflow, /labels -contains 'pester-service-model'/); + assert.match(workflow, /PR_LABELS_JSON:\s+\$\{\{\s*toJson\(github\.event\.pull_request\.labels\.\*\.name\)\s*\}\}/); + assert.match(workflow, /ConvertFrom-Json -InputObject \$env:PR_LABELS_JSON/); + assert.match(workflow, /head\.repo\.owner\.login/); + assert.match(workflow, /\$trustMode = 'same-owner-head'/); + assert.match(workflow, /reason = 'untrusted-cross-owner-fork'/); + assert.match(workflow, /uses:\s+\.\s*\/\.github\/workflows\/pester-gate\.yml/); + assert.match(workflow, /include_integration:\s+\$\{\{\s*'true'\s*\}\}/); +}); diff --git a/tools/priority/__tests__/validate-vi-history-dispatch-contract.test.mjs b/tools/priority/__tests__/validate-vi-history-dispatch-contract.test.mjs index d79d868d8..4d869a5e7 100644 --- a/tools/priority/__tests__/validate-vi-history-dispatch-contract.test.mjs +++ b/tools/priority/__tests__/validate-vi-history-dispatch-contract.test.mjs @@ -66,7 +66,12 @@ test('validate workflow Windows VI-history lane is gated by shared dispatch plan assert.match(planSection, /permissions:\s*\r?\n\s+contents: read/); assert.match(planSection, /Resolve self-hosted Windows Docker lane/); assert.match(planSection, /tools\/Resolve-SelfHostedWindowsLanePlan\.ps1/); - assert.match(planSection, /-RequiredLabels \$requiredLabels/); + assert.match(planSection, /\$args = @\(/); + assert.match(planSection, /'-RequiredLabels'/); + assert.match(planSection, /\) \+ \$requiredLabels \+ @\(/); + assert.match(planSection, /if \(\$requiredHealthReceipts\.Count -gt 0\)/); + assert.match(planSection, /\$args \+= '-RequiredHealthReceipts'/); + assert.match(planSection, /& pwsh @args/); assert.match(planSection, /docker-lane/); assert.match(planSection, /outputs:\s*\r?\n\s+available:\s+\$\{\{\s*steps\.plan\.outputs\.available\s*\}\}/); From 0b2c3cd2340784ff63051d3672cbf4f25ce15ccf Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Tue, 31 Mar 2026 07:52:05 -0700 Subject: [PATCH 14/44] [ops]: harden Pester service-model skip-path contracts (#2070) * Preserve Pester service-model outputs on skipped execution * Separate Pester execution contract from raw outputs --------- Co-authored-by: svelderrainruiz --- .github/workflows/pester-evidence.yml | 38 +++++++- .github/workflows/pester-gate.yml | 9 +- .github/workflows/pester-run.yml | 93 ++++++++++++++++++- docs/knowledgebase/Pester-Service-Model.md | 1 + ...r-service-model-workflow-contract.test.mjs | 12 ++- 5 files changed, 143 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pester-evidence.yml b/.github/workflows/pester-evidence.yml index a68ec5e21..30bb75d75 100644 --- a/.github/workflows/pester-evidence.yml +++ b/.github/workflows/pester-evidence.yml @@ -7,6 +7,10 @@ on: required: false type: string default: 'pester-run-raw' + execution_receipt_artifact_name: + required: false + type: string + default: 'pester-execution-contract' dispatcher_exit_code: required: false type: string @@ -49,6 +53,11 @@ on: required: false default: 'pester-run-raw' type: string + execution_receipt_artifact_name: + description: 'Execution contract artifact name' + required: false + default: 'pester-execution-contract' + type: string dispatcher_exit_code: description: 'Dispatcher exit code from execution workflow' required: false @@ -60,7 +69,7 @@ on: default: 'unknown' type: string execution_job_result: - description: 'Execution job result' + description: 'Execution contract outcome' required: false default: '' type: string @@ -89,11 +98,24 @@ jobs: shell: pwsh run: | $artifactName = '${{ inputs.raw_artifact_name }}' - if ([string]::IsNullOrWhiteSpace($artifactName)) { $artifactName = 'pester-run-raw' } + $shouldDownload = 'true' + if ('${{ inputs.execution_job_result }}' -in @('skipped','cancelled')) { + $shouldDownload = 'false' + } elseif ([string]::IsNullOrWhiteSpace($artifactName)) { + $artifactName = 'pester-run-raw' + } "name=$artifactName" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "should_download=$shouldDownload" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + + - name: Download execution receipt artifact + uses: actions/download-artifact@v5 + with: + name: ${{ inputs.execution_receipt_artifact_name }} + path: tests/execution-contract - name: Download raw execution artifact id: download + if: ${{ steps.artifact_name.outputs.should_download == 'true' }} continue-on-error: true uses: actions/download-artifact@v5 with: @@ -112,7 +134,7 @@ jobs: if: always() shell: pwsh run: | - $receiptPath = Join-Path 'tests/results' 'pester-run-receipt.json' + $receiptPath = Join-Path 'tests/execution-contract' 'pester-run-receipt.json' if (-not (Test-Path -LiteralPath $receiptPath)) { "present=false" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 "status=missing" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 @@ -254,6 +276,15 @@ jobs: } if ('${{ inputs.execution_job_result }}' -eq 'skipped') { $reasons.Add('execution-job-skipped') | Out-Null + } elseif ('${{ inputs.execution_job_result }}' -eq 'cancelled') { + $reasons.Add('execution-job-cancelled') | Out-Null + } elseif ('${{ inputs.execution_job_result }}' -eq 'seam-defect') { + $reasons.Add('execution-job-seam-defect') | Out-Null + } elseif ('${{ inputs.execution_job_result }}' -eq 'unknown') { + $reasons.Add('execution-job-unknown') | Out-Null + } + if ('${{ steps.artifact_name.outputs.should_download }}' -eq 'true' -and '${{ steps.download.outcome }}' -ne 'success') { + $reasons.Add(("raw-artifact-download={0}" -f '${{ steps.download.outcome }}')) | Out-Null } $dispatcherExitCode = '${{ inputs.dispatcher_exit_code }}' if ([string]::IsNullOrWhiteSpace($dispatcherExitCode)) { $dispatcherExitCode = '-1' } @@ -286,6 +317,7 @@ jobs: generatedAtUtc = [DateTime]::UtcNow.ToString('o') readinessStatus = '${{ inputs.readiness_status }}' executionJobResult = '${{ inputs.execution_job_result }}' + rawArtifactDownload = if ('${{ steps.artifact_name.outputs.should_download }}' -eq 'true') { '${{ steps.download.outcome }}' } else { 'skipped' } dispatcherExitCode = [int]$dispatcherExitCode summaryPresent = Test-Path -LiteralPath $summaryPath classification = $classification diff --git a/.github/workflows/pester-gate.yml b/.github/workflows/pester-gate.yml index 0824f9af8..6ab8a48c5 100644 --- a/.github/workflows/pester-gate.yml +++ b/.github/workflows/pester-gate.yml @@ -72,8 +72,8 @@ jobs: include_integration: ${{ inputs.include_integration || 'false' }} include_patterns: ${{ inputs.include_patterns || '' }} sample_id: ${{ inputs.sample_id || '' }} - readiness_status: ${{ needs.readiness.outputs.receipt_status }} - readiness_artifact_name: ${{ needs.readiness.outputs.receipt_artifact_name }} + readiness_status: ${{ needs.readiness.outputs.receipt_status || needs.readiness.result || 'unknown' }} + readiness_artifact_name: ${{ needs.readiness.outputs.receipt_artifact_name || 'pester-readiness' }} checkout_repository: ${{ inputs.checkout_repository || github.repository }} checkout_ref: ${{ inputs.checkout_ref || github.sha }} @@ -83,6 +83,7 @@ jobs: uses: ./.github/workflows/pester-evidence.yml with: raw_artifact_name: ${{ needs.pester-run.outputs.raw_artifact_name }} + execution_receipt_artifact_name: ${{ needs.pester-run.outputs.execution_receipt_artifact_name || 'pester-execution-contract' }} dispatcher_exit_code: ${{ needs.pester-run.outputs.dispatcher_exit_code }} - readiness_status: ${{ needs.readiness.outputs.receipt_status }} - execution_job_result: ${{ needs.pester-run.result }} + readiness_status: ${{ needs.readiness.outputs.receipt_status || needs.readiness.result || 'unknown' }} + execution_job_result: ${{ needs.pester-run.outputs.execution_status || needs.pester-run.result }} diff --git a/.github/workflows/pester-run.yml b/.github/workflows/pester-run.yml index 5aef347c8..39bb309d5 100644 --- a/.github/workflows/pester-run.yml +++ b/.github/workflows/pester-run.yml @@ -31,10 +31,16 @@ on: outputs: dispatcher_exit_code: description: 'Dispatcher exit code from the self-hosted execution pass' - value: ${{ jobs.pester.outputs.dispatcher_exit_code }} + value: ${{ jobs.finalize.outputs.dispatcher_exit_code }} raw_artifact_name: description: 'Artifact name containing raw execution outputs' - value: ${{ jobs.pester.outputs.raw_artifact_name }} + value: ${{ jobs.finalize.outputs.raw_artifact_name }} + execution_status: + description: 'Execution contract outcome for the self-hosted execution pass' + value: ${{ jobs.finalize.outputs.execution_status }} + execution_receipt_artifact_name: + description: 'Artifact name containing the execution contract receipt' + value: ${{ jobs.finalize.outputs.execution_receipt_artifact_name }} workflow_dispatch: inputs: include_integration: @@ -117,6 +123,7 @@ jobs: outputs: dispatcher_exit_code: ${{ steps.execution_receipt.outputs.dispatcher_exit_code }} raw_artifact_name: ${{ steps.execution_receipt.outputs.raw_artifact_name }} + execution_receipt_status: ${{ steps.execution_receipt.outputs.status }} steps: - uses: actions/checkout@v5 @@ -320,6 +327,7 @@ jobs: } $receiptPath = Join-Path $resultsDir 'pester-run-receipt.json' $receipt | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $receiptPath -Encoding UTF8 + "status=$status" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 "dispatcher_exit_code=$dispatcherExitCode" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 "raw_artifact_name=pester-run-raw" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 @@ -330,3 +338,84 @@ jobs: name: pester-run-raw path: tests/results if-no-files-found: warn + + finalize: + runs-on: ubuntu-latest + needs: [normalize, pester] + if: always() + outputs: + dispatcher_exit_code: ${{ steps.emit.outputs.dispatcher_exit_code }} + raw_artifact_name: ${{ steps.emit.outputs.raw_artifact_name }} + execution_status: ${{ steps.emit.outputs.execution_status }} + execution_receipt_artifact_name: ${{ steps.emit.outputs.execution_receipt_artifact_name }} + steps: + - name: Emit execution contract + id: emit + shell: pwsh + run: | + $resultsDir = Join-Path 'tests/results' 'pester-execution-contract' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + $executionStatus = 'unknown' + $receiptStatus = '${{ needs.pester.outputs.execution_receipt_status }}' + $rawArtifactName = '' + if ('${{ inputs.readiness_status }}' -ne 'ready') { + $executionStatus = 'skipped' + if ([string]::IsNullOrWhiteSpace($receiptStatus)) { $receiptStatus = 'skipped' } + } else { + switch ('${{ needs.pester.result }}') { + 'success' { + $executionStatus = 'completed' + $rawArtifactName = 'pester-run-raw' + if ([string]::IsNullOrWhiteSpace($receiptStatus)) { $receiptStatus = 'completed' } + } + 'failure' { + $executionStatus = 'seam-defect' + $rawArtifactName = 'pester-run-raw' + if ([string]::IsNullOrWhiteSpace($receiptStatus)) { $receiptStatus = 'seam-defect' } + } + 'cancelled' { + $executionStatus = 'cancelled' + if ([string]::IsNullOrWhiteSpace($receiptStatus)) { $receiptStatus = 'cancelled' } + } + 'skipped' { + $executionStatus = 'skipped' + if ([string]::IsNullOrWhiteSpace($receiptStatus)) { $receiptStatus = 'skipped' } + } + default { + $executionStatus = 'unknown' + if ([string]::IsNullOrWhiteSpace($receiptStatus)) { $receiptStatus = 'unknown' } + } + } + } + $dispatcherExitCode = '${{ needs.pester.outputs.dispatcher_exit_code }}' + if ([string]::IsNullOrWhiteSpace($dispatcherExitCode)) { + if ($receiptStatus -eq 'completed') { + $dispatcherExitCode = '0' + } else { + $dispatcherExitCode = '-1' + } + } + $receipt = [ordered]@{ + schema = 'pester-execution-receipt@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + readinessStatus = '${{ inputs.readiness_status }}' + executionJobResult = '${{ needs.pester.result }}' + dispatcherExitCode = [int]$dispatcherExitCode + status = $receiptStatus + rawArtifactName = $rawArtifactName + rawArtifactExpected = -not [string]::IsNullOrWhiteSpace($rawArtifactName) + source = 'pester-run/finalize' + } + $receiptPath = Join-Path $resultsDir 'pester-run-receipt.json' + $receipt | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $receiptPath -Encoding UTF8 + "execution_status=$executionStatus" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "dispatcher_exit_code=$dispatcherExitCode" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "raw_artifact_name=$rawArtifactName" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "execution_receipt_artifact_name=pester-execution-contract" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + + - name: Upload execution contract artifact + uses: actions/upload-artifact@v7 + with: + name: pester-execution-contract + path: tests/results/pester-execution-contract + if-no-files-found: error diff --git a/docs/knowledgebase/Pester-Service-Model.md b/docs/knowledgebase/Pester-Service-Model.md index 5fe3d0d3a..1359ec5c1 100644 --- a/docs/knowledgebase/Pester-Service-Model.md +++ b/docs/knowledgebase/Pester-Service-Model.md @@ -30,6 +30,7 @@ The additive pilot introduces four workflow surfaces: - Readiness emits a bounded-freshness receipt artifact that execution must download and validate before dispatch. - Execution consumes readiness. It does not bootstrap Docker runtimes or install core toolchains. - Execution writes an execution receipt before uploading raw artifacts so evidence can classify the real seam outcome. +- Execution must also emit a skip-safe execution contract from an always-on finalize path so reusable-workflow outputs do not collapse when the execution job never starts. - Evidence consumes raw execution output plus the execution receipt. It classifies `seam-defect` explicitly when execution never yields a valid summary or never yields a valid execution receipt. - The existing required gate remains in place until the pilot proves equivalent or better behavior. - Trusted PR proving must stay on `pull_request_target` with same-owner gating. Cross-owner fork heads are not allowed to drive self-hosted execution. diff --git a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs index 7b48ca9df..82bfe682e 100644 --- a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs +++ b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs @@ -19,10 +19,13 @@ test('pester gate pilot routes readiness, execution, and evidence through separa assert.match(workflow, /workflow_dispatch:/); assert.match(workflow, /jobs:\s*\n\s*readiness:\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/selfhosted-readiness\.yml/); assert.match(workflow, /\n\s*pester-run:\s*\n\s+needs:\s+readiness\s*\n\s+if:\s+always\(\)\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/pester-run\.yml/); - assert.match(workflow, /readiness_artifact_name:\s+\$\{\{\s*needs\.readiness\.outputs\.receipt_artifact_name\s*\}\}/); + assert.match(workflow, /readiness_artifact_name:\s+\$\{\{\s*needs\.readiness\.outputs\.receipt_artifact_name\s*\|\|\s*'pester-readiness'/); + assert.match(workflow, /readiness_status:\s+\$\{\{\s*needs\.readiness\.outputs\.receipt_status\s*\|\|\s*needs\.readiness\.result/); assert.match(workflow, /checkout_repository:\s+\$\{\{\s*inputs\.checkout_repository \|\| github\.repository\s*\}\}/); assert.match(workflow, /checkout_ref:\s+\$\{\{\s*inputs\.checkout_ref \|\| github\.sha\s*\}\}/); assert.match(workflow, /\n\s*pester-evidence:\s*\n\s+needs:\s+\[readiness, pester-run\]\s*\n\s+if:\s+always\(\)\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/pester-evidence\.yml/); + assert.match(workflow, /execution_job_result:\s+\$\{\{\s*needs\.pester-run\.outputs\.execution_status\s*\|\|\s*needs\.pester-run\.result/); + assert.match(workflow, /execution_receipt_artifact_name:\s+\$\{\{\s*needs\.pester-run\.outputs\.execution_receipt_artifact_name/); }); test('selfhosted readiness owns host-plane certification and emits a receipt artifact', () => { @@ -50,6 +53,8 @@ test('pester run is execution-only and validates the readiness receipt before di assert.match(workflow, /name:\s+Pester run/); assert.match(workflow, /name:\s+Pester \(execution only\)/); assert.match(workflow, /if:\s+\$\{\{\s*inputs\.readiness_status == 'ready'\s*\}\}/); + assert.match(workflow, /execution_status:/); + assert.match(workflow, /execution_receipt_artifact_name:/); assert.match(workflow, /repository:\s+\$\{\{\s*inputs\.checkout_repository \|\| github\.repository\s*\}\}/); assert.match(workflow, /ref:\s+\$\{\{\s*inputs\.checkout_ref \|\| github\.sha\s*\}\}/); assert.match(workflow, /Download readiness receipt artifact/); @@ -58,6 +63,8 @@ test('pester run is execution-only and validates the readiness receipt before di assert.match(workflow, /Run Pester tests via local dispatcher/); assert.match(workflow, /pester-run-receipt\.json/); assert.match(workflow, /Upload raw Pester execution artifact/); + assert.match(workflow, /Emit execution contract/); + assert.match(workflow, /Upload execution contract artifact/); assert.doesNotMatch(workflow, /Install Pester/); assert.doesNotMatch(workflow, /Invoke-DockerRuntimeManager\.ps1/); assert.doesNotMatch(workflow, /Write-PesterSummaryToStepSummary\.ps1/); @@ -69,6 +76,8 @@ test('pester evidence classifies seam defects explicitly from raw execution outp assert.match(workflow, /name:\s+Pester evidence/); assert.match(workflow, /runs-on:\s+ubuntu-latest/); + assert.match(workflow, /execution_receipt_artifact_name:/); + assert.match(workflow, /Download execution receipt artifact/); assert.match(workflow, /Download raw execution artifact/); assert.match(workflow, /Validate execution receipt artifact/); assert.match(workflow, /execution-receipt-missing/); @@ -76,6 +85,7 @@ test('pester evidence classifies seam defects explicitly from raw execution outp assert.match(workflow, /Ensure-SessionIndex\.ps1/); assert.match(workflow, /Invoke-DevDashboard\.ps1/); assert.match(workflow, /classification = 'seam-defect'/); + assert.match(workflow, /raw-artifact-download=/); assert.match(workflow, /execution-receipt-seam-defect/); assert.match(workflow, /Upload evidence artifact/); assert.match(workflow, /Propagate gate outcome/); From 1e11ac1d426053a2db06148fc1b065926deb8120 Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Tue, 31 Mar 2026 08:07:28 -0700 Subject: [PATCH 15/44] [ops]: allow trusted Pester router on integration branches (#2073) * Allow trusted Pester router on integration branches * Fix Windows Docker planner label binding --------- Co-authored-by: svelderrainruiz --- .../pester-service-model-on-label.yml | 2 +- .github/workflows/validate.yml | 53 +++++++------------ ...date-vi-history-dispatch-contract.test.mjs | 11 ++-- 3 files changed, 25 insertions(+), 41 deletions(-) diff --git a/.github/workflows/pester-service-model-on-label.yml b/.github/workflows/pester-service-model-on-label.yml index f9728fe44..d4358695f 100644 --- a/.github/workflows/pester-service-model-on-label.yml +++ b/.github/workflows/pester-service-model-on-label.yml @@ -3,7 +3,7 @@ name: Pester service-model pilot on trusted PR label on: pull_request_target: types: [labeled, reopened, synchronize] - branches: [main, develop, release/**] + branches: [main, develop, integration/**, release/**] paths-ignore: - '**/*.md' workflow_dispatch: diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 2aa6e22e5..6e950e414 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -1535,43 +1535,26 @@ jobs: 'docker-lane' ) $requiredHealthReceipts = @() - $notes = @( - 'Availability means an online, idle repository runner advertises the ingress plus docker-lane labels.', - 'The lane may mutate Docker Desktop into the Windows engine and then restores the starting context after proof capture.' - ) - $args = @( - '-NoLogo', - '-NoProfile', - '-File', - 'tools/Resolve-SelfHostedWindowsLanePlan.ps1', - '-Repository', - '${{ github.repository }}', - '-RequiredLabels' - ) + $requiredLabels + @( - '-ExecutionModel', - 'self-hosted-windows-docker-lane', - '-RunnerImage', - 'self-hosted-windows-docker-lane', - '-ExpectedContext', - 'desktop-windows', - '-ExpectedOs', - 'windows', - '-Notes' - ) + $notes + @( - '-Token', - $env:GITHUB_TOKEN, - '-OutputJsonPath', - $planPath, - '-GitHubOutputPath', - $env:GITHUB_OUTPUT, - '-StepSummaryPath', - $env:GITHUB_STEP_SUMMARY - ) + $plannerArgs = @{ + Repository = '${{ github.repository }}' + RequiredLabels = $requiredLabels + ExecutionModel = 'self-hosted-windows-docker-lane' + RunnerImage = 'self-hosted-windows-docker-lane' + ExpectedContext = 'desktop-windows' + ExpectedOs = 'windows' + Notes = @( + 'Availability means an online, idle repository runner advertises the ingress plus docker-lane labels.', + 'The lane may mutate Docker Desktop into the Windows engine and then restores the starting context after proof capture.' + ) + Token = $env:GITHUB_TOKEN + OutputJsonPath = $planPath + GitHubOutputPath = $env:GITHUB_OUTPUT + StepSummaryPath = $env:GITHUB_STEP_SUMMARY + } if ($requiredHealthReceipts.Count -gt 0) { - $args += '-RequiredHealthReceipts' - $args += $requiredHealthReceipts + $plannerArgs.RequiredHealthReceipts = $requiredHealthReceipts } - & pwsh @args + & tools/Resolve-SelfHostedWindowsLanePlan.ps1 @plannerArgs - name: Append VI history Windows runner plan shell: pwsh diff --git a/tools/priority/__tests__/validate-vi-history-dispatch-contract.test.mjs b/tools/priority/__tests__/validate-vi-history-dispatch-contract.test.mjs index 4d869a5e7..37aa5b794 100644 --- a/tools/priority/__tests__/validate-vi-history-dispatch-contract.test.mjs +++ b/tools/priority/__tests__/validate-vi-history-dispatch-contract.test.mjs @@ -66,12 +66,13 @@ test('validate workflow Windows VI-history lane is gated by shared dispatch plan assert.match(planSection, /permissions:\s*\r?\n\s+contents: read/); assert.match(planSection, /Resolve self-hosted Windows Docker lane/); assert.match(planSection, /tools\/Resolve-SelfHostedWindowsLanePlan\.ps1/); - assert.match(planSection, /\$args = @\(/); - assert.match(planSection, /'-RequiredLabels'/); - assert.match(planSection, /\) \+ \$requiredLabels \+ @\(/); + assert.match(planSection, /\$plannerArgs = @\{/); + assert.match(planSection, /RequiredLabels = \$requiredLabels/); + assert.match(planSection, /Notes = @\(/); assert.match(planSection, /if \(\$requiredHealthReceipts\.Count -gt 0\)/); - assert.match(planSection, /\$args \+= '-RequiredHealthReceipts'/); - assert.match(planSection, /& pwsh @args/); + assert.match(planSection, /\$plannerArgs\.RequiredHealthReceipts = \$requiredHealthReceipts/); + assert.match(planSection, /& tools\/Resolve-SelfHostedWindowsLanePlan\.ps1 @plannerArgs/); + assert.doesNotMatch(planSection, /& pwsh @args/); assert.match(planSection, /docker-lane/); assert.match(planSection, /outputs:\s*\r?\n\s+available:\s+\$\{\{\s*steps\.plan\.outputs\.available\s*\}\}/); From 1a89659c35ccf06cf7f06d424e1fa656cc311179 Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Tue, 31 Mar 2026 08:15:01 -0700 Subject: [PATCH 16/44] [ops]: fix auto-merge helper for workflow-edit PRs (#2074) Fix auto-merge helper for workflow-edit PRs Co-authored-by: svelderrainruiz --- .github/workflows/pr-automerge.yml | 10 ++++---- .../pr-automerge-workflow-contract.test.mjs | 23 +++++++++++++++++++ 2 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 tools/priority/__tests__/pr-automerge-workflow-contract.test.mjs diff --git a/.github/workflows/pr-automerge.yml b/.github/workflows/pr-automerge.yml index 67d6db0f5..ea1b3c94f 100644 --- a/.github/workflows/pr-automerge.yml +++ b/.github/workflows/pr-automerge.yml @@ -14,9 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Enable auto-merge (merge method) - uses: peter-evans/enable-pull-request-automerge@v3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - pull-request-number: ${{ github.event.pull_request.number }} - merge-method: MERGE - + env: + GH_TOKEN: ${{ secrets.GH_TOKEN || secrets.GITHUB_TOKEN }} + run: | + gh pr merge -R "${{ github.repository }}" --auto "${{ github.event.pull_request.number }}" diff --git a/tools/priority/__tests__/pr-automerge-workflow-contract.test.mjs b/tools/priority/__tests__/pr-automerge-workflow-contract.test.mjs new file mode 100644 index 000000000..5014a81af --- /dev/null +++ b/tools/priority/__tests__/pr-automerge-workflow-contract.test.mjs @@ -0,0 +1,23 @@ +#!/usr/bin/env node + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { readFileSync } from 'node:fs'; + +const repoRoot = process.cwd(); + +function readRepoFile(relativePath) { + return readFileSync(path.join(repoRoot, relativePath), 'utf8'); +} + +test('pr-automerge workflow uses gh CLI with GH_TOKEN fallback', () => { + const workflow = readRepoFile('.github/workflows/pr-automerge.yml'); + + assert.match(workflow, /name:\s*PR Auto-merge \(on label\)/); + assert.match(workflow, /pull_request_target:\s*\r?\n\s+types:\s*\[labeled, synchronize, reopened\]/); + assert.match(workflow, /if:\s*contains\(github\.event\.pull_request\.labels\.\*\.name,\s*'automerge'\)/); + assert.match(workflow, /GH_TOKEN:\s*\$\{\{\s*secrets\.GH_TOKEN \|\| secrets\.GITHUB_TOKEN\s*\}\}/); + assert.match(workflow, /gh pr merge -R "\$\{\{\s*github\.repository\s*\}\}" --auto "\$\{\{\s*github\.event\.pull_request\.number\s*\}\}"/); + assert.doesNotMatch(workflow, /enable-pull-request-automerge@v3/); +}); From b913d83be79792a1fda81eb66f1fe83fe1a301be Mon Sep 17 00:00:00 2001 From: svelderrainruiz Date: Tue, 31 Mar 2026 08:15:57 -0700 Subject: [PATCH 17/44] Classify readiness-blocked Pester evidence explicitly --- .github/workflows/pester-evidence.yml | 20 +++++++++++-------- ...r-service-model-workflow-contract.test.mjs | 4 +++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pester-evidence.yml b/.github/workflows/pester-evidence.yml index 30bb75d75..ea757c3dc 100644 --- a/.github/workflows/pester-evidence.yml +++ b/.github/workflows/pester-evidence.yml @@ -269,18 +269,20 @@ jobs: $summaryPath = Join-Path $resultsDir 'pester-summary.json' $classification = 'seam-defect' $reasons = New-Object System.Collections.Generic.List[string] + $readinessStatus = '${{ inputs.readiness_status }}' + $executionJobResult = '${{ inputs.execution_job_result }}' $executionReceiptPresent = '${{ steps.execution_receipt.outputs.present }}' $executionReceiptStatus = '${{ steps.execution_receipt.outputs.status }}' - if ('${{ inputs.readiness_status }}' -ne 'ready') { - $reasons.Add(("readiness-status={0}" -f '${{ inputs.readiness_status }}')) | Out-Null + if ($readinessStatus -ne 'ready') { + $reasons.Add(("readiness-status={0}" -f $readinessStatus)) | Out-Null } - if ('${{ inputs.execution_job_result }}' -eq 'skipped') { + if ($executionJobResult -eq 'skipped') { $reasons.Add('execution-job-skipped') | Out-Null - } elseif ('${{ inputs.execution_job_result }}' -eq 'cancelled') { + } elseif ($executionJobResult -eq 'cancelled') { $reasons.Add('execution-job-cancelled') | Out-Null - } elseif ('${{ inputs.execution_job_result }}' -eq 'seam-defect') { + } elseif ($executionJobResult -eq 'seam-defect') { $reasons.Add('execution-job-seam-defect') | Out-Null - } elseif ('${{ inputs.execution_job_result }}' -eq 'unknown') { + } elseif ($executionJobResult -eq 'unknown') { $reasons.Add('execution-job-unknown') | Out-Null } if ('${{ steps.artifact_name.outputs.should_download }}' -eq 'true' -and '${{ steps.download.outcome }}' -ne 'success') { @@ -290,6 +292,8 @@ jobs: if ([string]::IsNullOrWhiteSpace($dispatcherExitCode)) { $dispatcherExitCode = '-1' } if ($executionReceiptPresent -ne 'true') { $reasons.Add('execution-receipt-missing') | Out-Null + } elseif ($readinessStatus -ne 'ready' -and $executionJobResult -in @('skipped','cancelled')) { + $classification = 'readiness-blocked' } elseif ($executionReceiptStatus -eq 'seam-defect') { $reasons.Add('execution-receipt-seam-defect') | Out-Null } elseif ($executionReceiptStatus -eq 'test-failures') { @@ -315,8 +319,8 @@ jobs: $receipt = [ordered]@{ schema = 'pester-evidence-classification@v1' generatedAtUtc = [DateTime]::UtcNow.ToString('o') - readinessStatus = '${{ inputs.readiness_status }}' - executionJobResult = '${{ inputs.execution_job_result }}' + readinessStatus = $readinessStatus + executionJobResult = $executionJobResult rawArtifactDownload = if ('${{ steps.artifact_name.outputs.should_download }}' -eq 'true') { '${{ steps.download.outcome }}' } else { 'skipped' } dispatcherExitCode = [int]$dispatcherExitCode summaryPresent = Test-Path -LiteralPath $summaryPath diff --git a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs index 82bfe682e..ef764cffc 100644 --- a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs +++ b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs @@ -71,7 +71,7 @@ test('pester run is execution-only and validates the readiness receipt before di assert.doesNotMatch(workflow, /Invoke-DevDashboard\.ps1/); }); -test('pester evidence classifies seam defects explicitly from raw execution outputs', () => { +test('pester evidence distinguishes readiness-blocked skips from seam defects', () => { const workflow = readRepoFile('.github/workflows/pester-evidence.yml'); assert.match(workflow, /name:\s+Pester evidence/); @@ -85,6 +85,8 @@ test('pester evidence classifies seam defects explicitly from raw execution outp assert.match(workflow, /Ensure-SessionIndex\.ps1/); assert.match(workflow, /Invoke-DevDashboard\.ps1/); assert.match(workflow, /classification = 'seam-defect'/); + assert.match(workflow, /\$classification = 'readiness-blocked'/); + assert.match(workflow, /\$readinessStatus -ne 'ready' -and \$executionJobResult -in @\('skipped','cancelled'\)/); assert.match(workflow, /raw-artifact-download=/); assert.match(workflow, /execution-receipt-seam-defect/); assert.match(workflow, /Upload evidence artifact/); From 0a20e5c70ec145960cab610ae9af74dd6f116f66 Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Tue, 31 Mar 2026 08:19:52 -0700 Subject: [PATCH 18/44] [ops]: classify readiness-blocked Pester evidence explicitly (#2075) Classify readiness-blocked Pester evidence explicitly Co-authored-by: svelderrainruiz --- .github/workflows/pester-evidence.yml | 20 +++++++++++-------- ...r-service-model-workflow-contract.test.mjs | 4 +++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pester-evidence.yml b/.github/workflows/pester-evidence.yml index 30bb75d75..ea757c3dc 100644 --- a/.github/workflows/pester-evidence.yml +++ b/.github/workflows/pester-evidence.yml @@ -269,18 +269,20 @@ jobs: $summaryPath = Join-Path $resultsDir 'pester-summary.json' $classification = 'seam-defect' $reasons = New-Object System.Collections.Generic.List[string] + $readinessStatus = '${{ inputs.readiness_status }}' + $executionJobResult = '${{ inputs.execution_job_result }}' $executionReceiptPresent = '${{ steps.execution_receipt.outputs.present }}' $executionReceiptStatus = '${{ steps.execution_receipt.outputs.status }}' - if ('${{ inputs.readiness_status }}' -ne 'ready') { - $reasons.Add(("readiness-status={0}" -f '${{ inputs.readiness_status }}')) | Out-Null + if ($readinessStatus -ne 'ready') { + $reasons.Add(("readiness-status={0}" -f $readinessStatus)) | Out-Null } - if ('${{ inputs.execution_job_result }}' -eq 'skipped') { + if ($executionJobResult -eq 'skipped') { $reasons.Add('execution-job-skipped') | Out-Null - } elseif ('${{ inputs.execution_job_result }}' -eq 'cancelled') { + } elseif ($executionJobResult -eq 'cancelled') { $reasons.Add('execution-job-cancelled') | Out-Null - } elseif ('${{ inputs.execution_job_result }}' -eq 'seam-defect') { + } elseif ($executionJobResult -eq 'seam-defect') { $reasons.Add('execution-job-seam-defect') | Out-Null - } elseif ('${{ inputs.execution_job_result }}' -eq 'unknown') { + } elseif ($executionJobResult -eq 'unknown') { $reasons.Add('execution-job-unknown') | Out-Null } if ('${{ steps.artifact_name.outputs.should_download }}' -eq 'true' -and '${{ steps.download.outcome }}' -ne 'success') { @@ -290,6 +292,8 @@ jobs: if ([string]::IsNullOrWhiteSpace($dispatcherExitCode)) { $dispatcherExitCode = '-1' } if ($executionReceiptPresent -ne 'true') { $reasons.Add('execution-receipt-missing') | Out-Null + } elseif ($readinessStatus -ne 'ready' -and $executionJobResult -in @('skipped','cancelled')) { + $classification = 'readiness-blocked' } elseif ($executionReceiptStatus -eq 'seam-defect') { $reasons.Add('execution-receipt-seam-defect') | Out-Null } elseif ($executionReceiptStatus -eq 'test-failures') { @@ -315,8 +319,8 @@ jobs: $receipt = [ordered]@{ schema = 'pester-evidence-classification@v1' generatedAtUtc = [DateTime]::UtcNow.ToString('o') - readinessStatus = '${{ inputs.readiness_status }}' - executionJobResult = '${{ inputs.execution_job_result }}' + readinessStatus = $readinessStatus + executionJobResult = $executionJobResult rawArtifactDownload = if ('${{ steps.artifact_name.outputs.should_download }}' -eq 'true') { '${{ steps.download.outcome }}' } else { 'skipped' } dispatcherExitCode = [int]$dispatcherExitCode summaryPresent = Test-Path -LiteralPath $summaryPath diff --git a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs index 82bfe682e..ef764cffc 100644 --- a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs +++ b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs @@ -71,7 +71,7 @@ test('pester run is execution-only and validates the readiness receipt before di assert.doesNotMatch(workflow, /Invoke-DevDashboard\.ps1/); }); -test('pester evidence classifies seam defects explicitly from raw execution outputs', () => { +test('pester evidence distinguishes readiness-blocked skips from seam defects', () => { const workflow = readRepoFile('.github/workflows/pester-evidence.yml'); assert.match(workflow, /name:\s+Pester evidence/); @@ -85,6 +85,8 @@ test('pester evidence classifies seam defects explicitly from raw execution outp assert.match(workflow, /Ensure-SessionIndex\.ps1/); assert.match(workflow, /Invoke-DevDashboard\.ps1/); assert.match(workflow, /classification = 'seam-defect'/); + assert.match(workflow, /\$classification = 'readiness-blocked'/); + assert.match(workflow, /\$readinessStatus -ne 'ready' -and \$executionJobResult -in @\('skipped','cancelled'\)/); assert.match(workflow, /raw-artifact-download=/); assert.match(workflow, /execution-receipt-seam-defect/); assert.match(workflow, /Upload evidence artifact/); From 6eda190494c8e7d66ba4586d30b03b94b093d164 Mon Sep 17 00:00:00 2001 From: svelderrainruiz Date: Tue, 31 Mar 2026 08:28:37 -0700 Subject: [PATCH 19/44] [ops]: remove trusted pilot path filter --- .github/workflows/pester-service-model-on-label.yml | 2 -- .../__tests__/pester-service-model-workflow-contract.test.mjs | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pester-service-model-on-label.yml b/.github/workflows/pester-service-model-on-label.yml index d4358695f..1df0161fc 100644 --- a/.github/workflows/pester-service-model-on-label.yml +++ b/.github/workflows/pester-service-model-on-label.yml @@ -4,8 +4,6 @@ on: pull_request_target: types: [labeled, reopened, synchronize] branches: [main, develop, integration/**, release/**] - paths-ignore: - - '**/*.md' workflow_dispatch: inputs: sample_id: diff --git a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs index ef764cffc..5f275aead 100644 --- a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs +++ b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs @@ -111,6 +111,7 @@ test('trusted PR pilot router only runs self-hosted service-model proof for work assert.match(workflow, /name:\s+Pester service-model pilot on trusted PR label/); assert.match(workflow, /pull_request_target:/); assert.match(workflow, /types:\s*\[labeled, reopened, synchronize\]/); + assert.doesNotMatch(workflow, /paths-ignore:/); assert.match(workflow, /workflow_dispatch:/); assert.match(workflow, /labels -contains 'pester-service-model'/); assert.match(workflow, /PR_LABELS_JSON:\s+\$\{\{\s*toJson\(github\.event\.pull_request\.labels\.\*\.name\)\s*\}\}/); From d1a957bbe667adaa71eb412f1260071098a8344f Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Tue, 31 Mar 2026 08:31:50 -0700 Subject: [PATCH 20/44] [ops]: remove trusted pilot path filter (#2076) Co-authored-by: svelderrainruiz --- .github/workflows/pester-service-model-on-label.yml | 2 -- .../__tests__/pester-service-model-workflow-contract.test.mjs | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pester-service-model-on-label.yml b/.github/workflows/pester-service-model-on-label.yml index d4358695f..1df0161fc 100644 --- a/.github/workflows/pester-service-model-on-label.yml +++ b/.github/workflows/pester-service-model-on-label.yml @@ -4,8 +4,6 @@ on: pull_request_target: types: [labeled, reopened, synchronize] branches: [main, develop, integration/**, release/**] - paths-ignore: - - '**/*.md' workflow_dispatch: inputs: sample_id: diff --git a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs index ef764cffc..5f275aead 100644 --- a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs +++ b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs @@ -111,6 +111,7 @@ test('trusted PR pilot router only runs self-hosted service-model proof for work assert.match(workflow, /name:\s+Pester service-model pilot on trusted PR label/); assert.match(workflow, /pull_request_target:/); assert.match(workflow, /types:\s*\[labeled, reopened, synchronize\]/); + assert.doesNotMatch(workflow, /paths-ignore:/); assert.match(workflow, /workflow_dispatch:/); assert.match(workflow, /labels -contains 'pester-service-model'/); assert.match(workflow, /PR_LABELS_JSON:\s+\$\{\{\s*toJson\(github\.event\.pull_request\.labels\.\*\.name\)\s*\}\}/); From 168c3ac209caa5c9f7dd1f7a906c028f16c88170 Mon Sep 17 00:00:00 2001 From: svelderrainruiz Date: Tue, 31 Mar 2026 08:35:22 -0700 Subject: [PATCH 21/44] [ops]: harden trusted pilot routing outputs --- .../pester-service-model-on-label.yml | 24 +++++++++++++------ ...r-service-model-workflow-contract.test.mjs | 4 ++++ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pester-service-model-on-label.yml b/.github/workflows/pester-service-model-on-label.yml index 1df0161fc..042893dd9 100644 --- a/.github/workflows/pester-service-model-on-label.yml +++ b/.github/workflows/pester-service-model-on-label.yml @@ -79,15 +79,25 @@ jobs: $reason = 'trusted-same-owner-head' } } - "should_run=$shouldRun" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 - "checkout_repository=$checkoutRepository" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 - "checkout_ref=$checkoutRef" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 - "trust_mode=$trustMode" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 - "reason=$reason" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + Add-Content -Path $env:GITHUB_OUTPUT -Value "should_run=$shouldRun" + Add-Content -Path $env:GITHUB_OUTPUT -Value "checkout_repository=$checkoutRepository" + Add-Content -Path $env:GITHUB_OUTPUT -Value "checkout_ref=$checkoutRef" + Add-Content -Path $env:GITHUB_OUTPUT -Value "trust_mode=$trustMode" + Add-Content -Path $env:GITHUB_OUTPUT -Value "reason=$reason" + if ($env:GITHUB_STEP_SUMMARY) { + $lines = @('### Trusted pilot routing', '') + $lines += ('- Event: {0}' -f $eventName) + $lines += ('- Should run: {0}' -f $shouldRun) + $lines += ('- Reason: {0}' -f $reason) + $lines += ('- Trust mode: {0}' -f $trustMode) + $lines += ('- Checkout repository: {0}' -f $checkoutRepository) + $lines += ('- Checkout ref: {0}' -f $checkoutRef) + $lines -join "`n" | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append -Encoding utf8 + } pilot: needs: trust-context - if: ${{ needs.trust-context.outputs.should_run == 'true' }} + if: ${{ fromJSON(needs.trust-context.outputs.should_run || 'false') }} uses: ./.github/workflows/pester-gate.yml with: include_integration: ${{ 'true' }} @@ -97,7 +107,7 @@ jobs: skipped: needs: trust-context - if: ${{ needs.trust-context.outputs.should_run != 'true' }} + if: ${{ !fromJSON(needs.trust-context.outputs.should_run || 'false') }} runs-on: ubuntu-latest steps: - name: Append skip summary diff --git a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs index 5f275aead..83160d3e8 100644 --- a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs +++ b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs @@ -117,6 +117,10 @@ test('trusted PR pilot router only runs self-hosted service-model proof for work assert.match(workflow, /PR_LABELS_JSON:\s+\$\{\{\s*toJson\(github\.event\.pull_request\.labels\.\*\.name\)\s*\}\}/); assert.match(workflow, /ConvertFrom-Json -InputObject \$env:PR_LABELS_JSON/); assert.match(workflow, /head\.repo\.owner\.login/); + assert.match(workflow, /Add-Content -Path \$env:GITHUB_OUTPUT -Value "should_run=\$shouldRun"/); + assert.match(workflow, /### Trusted pilot routing/); + assert.match(workflow, /if:\s+\$\{\{\s*fromJSON\(needs\.trust-context\.outputs\.should_run \|\| 'false'\)\s*\}\}/); + assert.match(workflow, /if:\s+\$\{\{\s*!fromJSON\(needs\.trust-context\.outputs\.should_run \|\| 'false'\)\s*\}\}/); assert.match(workflow, /\$trustMode = 'same-owner-head'/); assert.match(workflow, /reason = 'untrusted-cross-owner-fork'/); assert.match(workflow, /uses:\s+\.\s*\/\.github\/workflows\/pester-gate\.yml/); From 092b893dce162900b88d10936364d729c5966d8a Mon Sep 17 00:00:00 2001 From: svelderrainruiz Date: Tue, 31 Mar 2026 08:38:20 -0700 Subject: [PATCH 22/44] [ops]: move pilot routing inside pester gate --- .github/workflows/pester-gate.yml | 50 ++++++++++++++++++- .../pester-service-model-on-label.yml | 20 ++------ ...r-service-model-workflow-contract.test.mjs | 16 ++++-- 3 files changed, 62 insertions(+), 24 deletions(-) diff --git a/.github/workflows/pester-gate.yml b/.github/workflows/pester-gate.yml index 6ab8a48c5..50d5927b9 100644 --- a/.github/workflows/pester-gate.yml +++ b/.github/workflows/pester-gate.yml @@ -3,6 +3,18 @@ name: Pester gate (service model pilot) on: workflow_call: inputs: + route_should_run: + required: false + default: 'true' + type: string + route_reason: + required: false + default: '' + type: string + route_trust_mode: + required: false + default: '' + type: string include_integration: required: false default: 'false' @@ -25,6 +37,22 @@ on: type: string workflow_dispatch: inputs: + route_should_run: + description: 'Whether the service-model pilot should run or emit a routed skip' + required: false + default: 'true' + type: choice + options: ['false', 'true'] + route_reason: + description: 'Optional skip reason emitted by the outer router' + required: false + default: '' + type: string + route_trust_mode: + description: 'Optional trust-mode emitted by the outer router' + required: false + default: '' + type: string include_integration: description: "Include Integration-tagged tests in the execution pack" required: false @@ -57,7 +85,25 @@ concurrency: cancel-in-progress: true jobs: + skipped: + if: ${{ !fromJSON(inputs.route_should_run || 'true') }} + runs-on: ubuntu-latest + steps: + - name: Append routed skip summary + shell: pwsh + run: | + if ($env:GITHUB_STEP_SUMMARY) { + $lines = @('### Pester gate (service model pilot)', '') + $lines += ('- Status: skipped') + $lines += ('- Reason: {0}' -f '${{ inputs.route_reason || 'route-should-not-run' }}') + $lines += ('- Trust mode: {0}' -f '${{ inputs.route_trust_mode || 'unspecified' }}') + $lines += ('- Checkout repository: {0}' -f '${{ inputs.checkout_repository || github.repository }}') + $lines += ('- Checkout ref: {0}' -f '${{ inputs.checkout_ref || github.sha }}') + $lines -join "`n" | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append -Encoding utf8 + } + readiness: + if: ${{ fromJSON(inputs.route_should_run || 'true') }} uses: ./.github/workflows/selfhosted-readiness.yml with: sample_id: ${{ inputs.sample_id || '' }} @@ -66,7 +112,7 @@ jobs: pester-run: needs: readiness - if: always() + if: ${{ always() && fromJSON(inputs.route_should_run || 'true') }} uses: ./.github/workflows/pester-run.yml with: include_integration: ${{ inputs.include_integration || 'false' }} @@ -79,7 +125,7 @@ jobs: pester-evidence: needs: [readiness, pester-run] - if: always() + if: ${{ always() && fromJSON(inputs.route_should_run || 'true') }} uses: ./.github/workflows/pester-evidence.yml with: raw_artifact_name: ${{ needs.pester-run.outputs.raw_artifact_name }} diff --git a/.github/workflows/pester-service-model-on-label.yml b/.github/workflows/pester-service-model-on-label.yml index 042893dd9..aea023aa5 100644 --- a/.github/workflows/pester-service-model-on-label.yml +++ b/.github/workflows/pester-service-model-on-label.yml @@ -97,26 +97,12 @@ jobs: pilot: needs: trust-context - if: ${{ fromJSON(needs.trust-context.outputs.should_run || 'false') }} uses: ./.github/workflows/pester-gate.yml with: + route_should_run: ${{ needs.trust-context.outputs.should_run || 'false' }} + route_reason: ${{ needs.trust-context.outputs.reason || '' }} + route_trust_mode: ${{ needs.trust-context.outputs.trust_mode || '' }} include_integration: ${{ 'true' }} sample_id: ${{ github.event.inputs.sample_id || '' }} checkout_repository: ${{ needs.trust-context.outputs.checkout_repository }} checkout_ref: ${{ needs.trust-context.outputs.checkout_ref }} - - skipped: - needs: trust-context - if: ${{ !fromJSON(needs.trust-context.outputs.should_run || 'false') }} - runs-on: ubuntu-latest - steps: - - name: Append skip summary - shell: pwsh - run: | - if ($env:GITHUB_STEP_SUMMARY) { - $lines = @('### Pester service-model pilot', '') - $lines += ('- Status: skipped') - $lines += ('- Reason: {0}' -f '${{ needs.trust-context.outputs.reason }}') - $lines += ('- Trust mode: {0}' -f '${{ needs.trust-context.outputs.trust_mode }}') - $lines -join "`n" | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append -Encoding utf8 - } diff --git a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs index 83160d3e8..9bb6c91ea 100644 --- a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs +++ b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs @@ -17,15 +17,20 @@ test('pester gate pilot routes readiness, execution, and evidence through separa assert.match(workflow, /name:\s+Pester gate \(service model pilot\)/); assert.match(workflow, /workflow_call:/); assert.match(workflow, /workflow_dispatch:/); - assert.match(workflow, /jobs:\s*\n\s*readiness:\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/selfhosted-readiness\.yml/); - assert.match(workflow, /\n\s*pester-run:\s*\n\s+needs:\s+readiness\s*\n\s+if:\s+always\(\)\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/pester-run\.yml/); + assert.match(workflow, /route_should_run:/); + assert.match(workflow, /route_reason:/); + assert.match(workflow, /route_trust_mode:/); + assert.match(workflow, /jobs:\s*\n\s*skipped:\s*\n\s+if:\s+\$\{\{\s*!fromJSON\(inputs\.route_should_run \|\| 'true'\)\s*\}\}/); + assert.match(workflow, /\n\s*readiness:\s*\n\s+if:\s+\$\{\{\s*fromJSON\(inputs\.route_should_run \|\| 'true'\)\s*\}\}\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/selfhosted-readiness\.yml/); + assert.match(workflow, /\n\s*pester-run:\s*\n\s+needs:\s+readiness\s*\n\s+if:\s+\$\{\{\s*always\(\) && fromJSON\(inputs\.route_should_run \|\| 'true'\)\s*\}\}\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/pester-run\.yml/); assert.match(workflow, /readiness_artifact_name:\s+\$\{\{\s*needs\.readiness\.outputs\.receipt_artifact_name\s*\|\|\s*'pester-readiness'/); assert.match(workflow, /readiness_status:\s+\$\{\{\s*needs\.readiness\.outputs\.receipt_status\s*\|\|\s*needs\.readiness\.result/); assert.match(workflow, /checkout_repository:\s+\$\{\{\s*inputs\.checkout_repository \|\| github\.repository\s*\}\}/); assert.match(workflow, /checkout_ref:\s+\$\{\{\s*inputs\.checkout_ref \|\| github\.sha\s*\}\}/); - assert.match(workflow, /\n\s*pester-evidence:\s*\n\s+needs:\s+\[readiness, pester-run\]\s*\n\s+if:\s+always\(\)\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/pester-evidence\.yml/); + assert.match(workflow, /\n\s*pester-evidence:\s*\n\s+needs:\s+\[readiness, pester-run\]\s*\n\s+if:\s+\$\{\{\s*always\(\) && fromJSON\(inputs\.route_should_run \|\| 'true'\)\s*\}\}\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/pester-evidence\.yml/); assert.match(workflow, /execution_job_result:\s+\$\{\{\s*needs\.pester-run\.outputs\.execution_status\s*\|\|\s*needs\.pester-run\.result/); assert.match(workflow, /execution_receipt_artifact_name:\s+\$\{\{\s*needs\.pester-run\.outputs\.execution_receipt_artifact_name/); + assert.match(workflow, /### Pester gate \(service model pilot\)/); }); test('selfhosted readiness owns host-plane certification and emits a receipt artifact', () => { @@ -119,10 +124,11 @@ test('trusted PR pilot router only runs self-hosted service-model proof for work assert.match(workflow, /head\.repo\.owner\.login/); assert.match(workflow, /Add-Content -Path \$env:GITHUB_OUTPUT -Value "should_run=\$shouldRun"/); assert.match(workflow, /### Trusted pilot routing/); - assert.match(workflow, /if:\s+\$\{\{\s*fromJSON\(needs\.trust-context\.outputs\.should_run \|\| 'false'\)\s*\}\}/); - assert.match(workflow, /if:\s+\$\{\{\s*!fromJSON\(needs\.trust-context\.outputs\.should_run \|\| 'false'\)\s*\}\}/); assert.match(workflow, /\$trustMode = 'same-owner-head'/); assert.match(workflow, /reason = 'untrusted-cross-owner-fork'/); assert.match(workflow, /uses:\s+\.\s*\/\.github\/workflows\/pester-gate\.yml/); + assert.match(workflow, /route_should_run:\s+\$\{\{\s*needs\.trust-context\.outputs\.should_run \|\| 'false'\s*\}\}/); + assert.match(workflow, /route_reason:\s+\$\{\{\s*needs\.trust-context\.outputs\.reason \|\| ''\s*\}\}/); + assert.match(workflow, /route_trust_mode:\s+\$\{\{\s*needs\.trust-context\.outputs\.trust_mode \|\| ''\s*\}\}/); assert.match(workflow, /include_integration:\s+\$\{\{\s*'true'\s*\}\}/); }); From e3fbc9aefe301e2dd25fb3e05c4ce2f8084e4e95 Mon Sep 17 00:00:00 2001 From: svelderrainruiz Date: Tue, 31 Mar 2026 08:44:13 -0700 Subject: [PATCH 23/44] [ops]: split trusted pilot concurrency domains --- .github/workflows/pester-gate.yml | 2 +- .github/workflows/pester-service-model-on-label.yml | 2 +- .../__tests__/pester-service-model-workflow-contract.test.mjs | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pester-gate.yml b/.github/workflows/pester-gate.yml index 50d5927b9..3beac533f 100644 --- a/.github/workflows/pester-gate.yml +++ b/.github/workflows/pester-gate.yml @@ -81,7 +81,7 @@ on: type: string concurrency: - group: ${{ github.workflow }}-${{ github.event.inputs.sample_id || github.ref }} + group: pester-gate-${{ inputs.sample_id || github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/pester-service-model-on-label.yml b/.github/workflows/pester-service-model-on-label.yml index aea023aa5..2e16eb7e1 100644 --- a/.github/workflows/pester-service-model-on-label.yml +++ b/.github/workflows/pester-service-model-on-label.yml @@ -23,7 +23,7 @@ on: type: string concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.inputs.sample_id || github.ref }} + group: trusted-pilot-router-${{ github.event.pull_request.number || github.event.inputs.sample_id || github.ref }} cancel-in-progress: true permissions: diff --git a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs index 9bb6c91ea..30f180517 100644 --- a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs +++ b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs @@ -20,6 +20,7 @@ test('pester gate pilot routes readiness, execution, and evidence through separa assert.match(workflow, /route_should_run:/); assert.match(workflow, /route_reason:/); assert.match(workflow, /route_trust_mode:/); + assert.match(workflow, /group:\s+pester-gate-\$\{\{\s*inputs\.sample_id \|\| github\.ref\s*\}\}/); assert.match(workflow, /jobs:\s*\n\s*skipped:\s*\n\s+if:\s+\$\{\{\s*!fromJSON\(inputs\.route_should_run \|\| 'true'\)\s*\}\}/); assert.match(workflow, /\n\s*readiness:\s*\n\s+if:\s+\$\{\{\s*fromJSON\(inputs\.route_should_run \|\| 'true'\)\s*\}\}\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/selfhosted-readiness\.yml/); assert.match(workflow, /\n\s*pester-run:\s*\n\s+needs:\s+readiness\s*\n\s+if:\s+\$\{\{\s*always\(\) && fromJSON\(inputs\.route_should_run \|\| 'true'\)\s*\}\}\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/pester-run\.yml/); @@ -117,6 +118,7 @@ test('trusted PR pilot router only runs self-hosted service-model proof for work assert.match(workflow, /pull_request_target:/); assert.match(workflow, /types:\s*\[labeled, reopened, synchronize\]/); assert.doesNotMatch(workflow, /paths-ignore:/); + assert.match(workflow, /group:\s+trusted-pilot-router-\$\{\{\s*github\.event\.pull_request\.number \|\| github\.event\.inputs\.sample_id \|\| github\.ref\s*\}\}/); assert.match(workflow, /workflow_dispatch:/); assert.match(workflow, /labels -contains 'pester-service-model'/); assert.match(workflow, /PR_LABELS_JSON:\s+\$\{\{\s*toJson\(github\.event\.pull_request\.labels\.\*\.name\)\s*\}\}/); From 2e9363ecf692268f9813b743f60497ac3c33eba7 Mon Sep 17 00:00:00 2001 From: svelderrainruiz Date: Tue, 31 Mar 2026 09:18:01 -0700 Subject: [PATCH 24/44] ci(pester): split service-model context layer (#2078) --- .github/workflows/pester-context.yml | 186 ++++++++++++++++++ .github/workflows/pester-evidence.yml | 16 ++ .github/workflows/pester-gate.yml | 18 +- .github/workflows/pester-run.yml | 57 +++++- docs/knowledgebase/Pester-Service-Model.md | 11 +- ...r-service-model-workflow-contract.test.mjs | 41 +++- 6 files changed, 314 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/pester-context.yml diff --git a/.github/workflows/pester-context.yml b/.github/workflows/pester-context.yml new file mode 100644 index 000000000..a7f4bc57b --- /dev/null +++ b/.github/workflows/pester-context.yml @@ -0,0 +1,186 @@ +name: Pester context + +on: + workflow_call: + inputs: + sample_id: + required: false + type: string + checkout_repository: + required: false + type: string + checkout_ref: + required: false + type: string + outputs: + receipt_status: + description: 'Overall context status for the repo/control-plane layer' + value: ${{ jobs.context.outputs.receipt_status }} + receipt_artifact_name: + description: 'Artifact name containing the context receipt bundle' + value: ${{ jobs.context.outputs.receipt_artifact_name }} + repository: + description: 'Repository slug resolved for context classification' + value: ${{ jobs.context.outputs.repository }} + standing_priority_issue: + description: 'Standing-priority issue number when context is ready' + value: ${{ jobs.context.outputs.standing_priority_issue }} + standing_priority_reason: + description: 'Reason emitted by the context classifier' + value: ${{ jobs.context.outputs.standing_priority_reason }} + workflow_dispatch: + inputs: + sample_id: + description: 'Sampling correlation id (prevents cancels)' + required: false + default: '' + type: string + checkout_repository: + description: 'Repository to checkout for context resolution' + required: false + default: '' + type: string + checkout_ref: + description: 'Git ref or SHA to checkout for context resolution' + required: false + default: '' + type: string + +concurrency: + group: ${{ github.workflow }}-${{ github.event.inputs.sample_id || inputs.sample_id || github.ref }} + cancel-in-progress: true + +jobs: + context: + runs-on: ubuntu-latest + outputs: + receipt_status: ${{ steps.receipt.outputs.status }} + receipt_artifact_name: ${{ steps.receipt.outputs.artifact_name }} + repository: ${{ steps.receipt.outputs.repository }} + standing_priority_issue: ${{ steps.receipt.outputs.standing_priority_issue }} + standing_priority_reason: ${{ steps.receipt.outputs.reason }} + steps: + - uses: actions/checkout@v5 + with: + repository: ${{ inputs.checkout_repository || github.repository }} + ref: ${{ inputs.checkout_ref || github.sha }} + + - name: Install Node dependencies + shell: pwsh + run: node tools/npm/cli.mjs ci + + - name: Validate repository context + shell: pwsh + run: | + $repository = '${{ inputs.checkout_repository || github.repository }}' + if ([string]::IsNullOrWhiteSpace($repository)) { + throw 'Repository context is empty.' + } + + - name: Export workflow token for context sync + shell: pwsh + env: + WORKFLOW_TOKEN: ${{ github.token }} + run: | + if (-not $env:WORKFLOW_TOKEN) { throw 'github.token is empty' } + "GH_TOKEN=$env:WORKFLOW_TOKEN" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + "GITHUB_TOKEN=$env:WORKFLOW_TOKEN" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + + - name: Resolve standing-priority context + id: standing_context + continue-on-error: true + shell: pwsh + run: node tools/priority/run-sync-standing-priority.mjs --materialize-cache + + - name: Write context receipt + id: receipt + if: always() + shell: pwsh + run: | + $outDir = 'tests/results/pester-context' + New-Item -ItemType Directory -Force -Path $outDir | Out-Null + + $issueDir = 'tests/results/_agent/issue' + $routerPath = Join-Path $issueDir 'router.json' + $noStandingPath = Join-Path $issueDir 'no-standing-priority.json' + $repository = '${{ inputs.checkout_repository || github.repository }}' + if ([string]::IsNullOrWhiteSpace($repository)) { + $repository = $env:GITHUB_REPOSITORY + } + + $status = 'blocked' + $reason = 'context-sync-missing' + $standingIssue = '' + $issueSummaryPath = $null + $syncOutcome = '${{ steps.standing_context.outcome }}' + + if (Test-Path -LiteralPath $noStandingPath) { + $report = Get-Content -LiteralPath $noStandingPath -Raw | ConvertFrom-Json -ErrorAction Stop + $status = 'blocked' + $reason = if ($report.reason) { [string]$report.reason } elseif ($report.message) { [string]$report.message } else { 'standing-priority-missing' } + } elseif (Test-Path -LiteralPath $routerPath) { + $router = Get-Content -LiteralPath $routerPath -Raw | ConvertFrom-Json -ErrorAction Stop + $issueValue = 0 + if ([int]::TryParse([string]$router.issue, [ref]$issueValue) -and $issueValue -gt 0) { + $standingIssue = [string]$issueValue + $issueSummaryPath = Join-Path $issueDir ("{0}.json" -f $standingIssue) + if (Test-Path -LiteralPath $issueSummaryPath) { + $issueSummary = Get-Content -LiteralPath $issueSummaryPath -Raw | ConvertFrom-Json -ErrorAction Stop + if ($issueSummary.schema -eq 'standing-priority/issue@v1') { + $status = 'ready' + $reason = 'standing-priority-available' + if ($issueSummary.url -match 'https://github.com/(?[^/]+/[^/]+)/issues/') { + $repository = $matches.slug + } + } else { + $status = 'warning' + $reason = ("unexpected-issue-schema:{0}" -f $issueSummary.schema) + } + } else { + $status = if ($syncOutcome -eq 'success') { 'warning' } else { 'blocked' } + $reason = if ($syncOutcome -eq 'success') { 'standing-priority-summary-missing' } else { 'context-sync-failed' } + } + } else { + $status = if ($syncOutcome -eq 'success') { 'warning' } else { 'blocked' } + $reason = if ($syncOutcome -eq 'success') { 'standing-priority-router-missing-issue' } else { 'context-sync-failed' } + } + } elseif ($syncOutcome -eq 'success') { + $status = 'warning' + $reason = 'standing-priority-router-missing' + } else { + $status = 'blocked' + $reason = 'context-sync-failed' + } + + $receipt = [ordered]@{ + schema = 'pester-context-receipt@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + status = $status + repository = $repository + sampleId = '${{ inputs.sample_id || github.event.inputs.sample_id || '' }}' + standingPriority = [ordered]@{ + issueNumber = if ($standingIssue) { [int]$standingIssue } else { $null } + reason = $reason + routerPath = if (Test-Path -LiteralPath $routerPath) { 'tests/results/_agent/issue/router.json' } else { $null } + issueSummaryPath = if ($issueSummaryPath -and (Test-Path -LiteralPath $issueSummaryPath)) { "tests/results/_agent/issue/$standingIssue.json" } else { $null } + noStandingPath = if (Test-Path -LiteralPath $noStandingPath) { 'tests/results/_agent/issue/no-standing-priority.json' } else { $null } + } + sync = [ordered]@{ + outcome = $syncOutcome + } + } + $receiptPath = Join-Path $outDir 'pester-context.json' + $receipt | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $receiptPath -Encoding UTF8 + "status=$status" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "artifact_name=pester-context" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "repository=$repository" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "standing_priority_issue=$standingIssue" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "reason=$reason" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + + - name: Upload context receipt + if: always() + uses: actions/upload-artifact@v7 + with: + name: pester-context + path: tests/results/pester-context + if-no-files-found: error diff --git a/.github/workflows/pester-evidence.yml b/.github/workflows/pester-evidence.yml index ea757c3dc..40dcf7309 100644 --- a/.github/workflows/pester-evidence.yml +++ b/.github/workflows/pester-evidence.yml @@ -15,6 +15,10 @@ on: required: false type: string default: '' + context_status: + required: false + type: string + default: 'unknown' readiness_status: required: false type: string @@ -63,6 +67,11 @@ on: required: false default: '' type: string + context_status: + description: 'Context receipt status' + required: false + default: 'unknown' + type: string readiness_status: description: 'Readiness receipt status' required: false @@ -269,10 +278,14 @@ jobs: $summaryPath = Join-Path $resultsDir 'pester-summary.json' $classification = 'seam-defect' $reasons = New-Object System.Collections.Generic.List[string] + $contextStatus = '${{ inputs.context_status }}' $readinessStatus = '${{ inputs.readiness_status }}' $executionJobResult = '${{ inputs.execution_job_result }}' $executionReceiptPresent = '${{ steps.execution_receipt.outputs.present }}' $executionReceiptStatus = '${{ steps.execution_receipt.outputs.status }}' + if ($contextStatus -ne 'ready') { + $reasons.Add(("context-status={0}" -f $contextStatus)) | Out-Null + } if ($readinessStatus -ne 'ready') { $reasons.Add(("readiness-status={0}" -f $readinessStatus)) | Out-Null } @@ -292,6 +305,8 @@ jobs: if ([string]::IsNullOrWhiteSpace($dispatcherExitCode)) { $dispatcherExitCode = '-1' } if ($executionReceiptPresent -ne 'true') { $reasons.Add('execution-receipt-missing') | Out-Null + } elseif (($contextStatus -ne 'ready' -and $executionJobResult -in @('skipped','cancelled')) -or $executionReceiptStatus -eq 'context-blocked') { + $classification = 'context-blocked' } elseif ($readinessStatus -ne 'ready' -and $executionJobResult -in @('skipped','cancelled')) { $classification = 'readiness-blocked' } elseif ($executionReceiptStatus -eq 'seam-defect') { @@ -319,6 +334,7 @@ jobs: $receipt = [ordered]@{ schema = 'pester-evidence-classification@v1' generatedAtUtc = [DateTime]::UtcNow.ToString('o') + contextStatus = $contextStatus readinessStatus = $readinessStatus executionJobResult = $executionJobResult rawArtifactDownload = if ('${{ steps.artifact_name.outputs.should_download }}' -eq 'true') { '${{ steps.download.outcome }}' } else { 'skipped' } diff --git a/.github/workflows/pester-gate.yml b/.github/workflows/pester-gate.yml index 3beac533f..a795c0397 100644 --- a/.github/workflows/pester-gate.yml +++ b/.github/workflows/pester-gate.yml @@ -102,8 +102,17 @@ jobs: $lines -join "`n" | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append -Encoding utf8 } - readiness: + context: if: ${{ fromJSON(inputs.route_should_run || 'true') }} + uses: ./.github/workflows/pester-context.yml + with: + sample_id: ${{ inputs.sample_id || '' }} + checkout_repository: ${{ inputs.checkout_repository || github.repository }} + checkout_ref: ${{ inputs.checkout_ref || github.sha }} + + readiness: + needs: context + if: ${{ always() && fromJSON(inputs.route_should_run || 'true') && needs.context.outputs.receipt_status == 'ready' }} uses: ./.github/workflows/selfhosted-readiness.yml with: sample_id: ${{ inputs.sample_id || '' }} @@ -111,25 +120,28 @@ jobs: checkout_ref: ${{ inputs.checkout_ref || github.sha }} pester-run: - needs: readiness + needs: [context, readiness] if: ${{ always() && fromJSON(inputs.route_should_run || 'true') }} uses: ./.github/workflows/pester-run.yml with: include_integration: ${{ inputs.include_integration || 'false' }} include_patterns: ${{ inputs.include_patterns || '' }} sample_id: ${{ inputs.sample_id || '' }} + context_status: ${{ needs.context.outputs.receipt_status || needs.context.result || 'unknown' }} + context_artifact_name: ${{ needs.context.outputs.receipt_artifact_name || 'pester-context' }} readiness_status: ${{ needs.readiness.outputs.receipt_status || needs.readiness.result || 'unknown' }} readiness_artifact_name: ${{ needs.readiness.outputs.receipt_artifact_name || 'pester-readiness' }} checkout_repository: ${{ inputs.checkout_repository || github.repository }} checkout_ref: ${{ inputs.checkout_ref || github.sha }} pester-evidence: - needs: [readiness, pester-run] + needs: [context, readiness, pester-run] if: ${{ always() && fromJSON(inputs.route_should_run || 'true') }} uses: ./.github/workflows/pester-evidence.yml with: raw_artifact_name: ${{ needs.pester-run.outputs.raw_artifact_name }} execution_receipt_artifact_name: ${{ needs.pester-run.outputs.execution_receipt_artifact_name || 'pester-execution-contract' }} dispatcher_exit_code: ${{ needs.pester-run.outputs.dispatcher_exit_code }} + context_status: ${{ needs.context.outputs.receipt_status || needs.context.result || 'unknown' }} readiness_status: ${{ needs.readiness.outputs.receipt_status || needs.readiness.result || 'unknown' }} execution_job_result: ${{ needs.pester-run.outputs.execution_status || needs.pester-run.result }} diff --git a/.github/workflows/pester-run.yml b/.github/workflows/pester-run.yml index 39bb309d5..8d1502f9c 100644 --- a/.github/workflows/pester-run.yml +++ b/.github/workflows/pester-run.yml @@ -11,6 +11,14 @@ on: required: false type: string default: '' + context_status: + required: false + type: string + default: 'unknown' + context_artifact_name: + required: false + type: string + default: 'pester-context' sample_id: required: false type: string @@ -54,6 +62,16 @@ on: required: false default: '' type: string + context_status: + description: 'Context receipt status' + required: false + default: 'unknown' + type: string + context_artifact_name: + description: 'Context receipt artifact name' + required: false + default: 'pester-context' + type: string sample_id: description: 'Sampling correlation id (prevents cancels)' required: false @@ -106,7 +124,7 @@ jobs: pester: name: Pester (execution only) - if: ${{ inputs.readiness_status == 'ready' }} + if: ${{ inputs.context_status == 'ready' && inputs.readiness_status == 'ready' }} runs-on: [self-hosted, Windows, X64, comparevi, capability-ingress] needs: normalize env: @@ -140,6 +158,31 @@ jobs: "GH_TOKEN=$env:WORKFLOW_TOKEN" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 "GITHUB_TOKEN=$env:WORKFLOW_TOKEN" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + - name: Download context receipt artifact + uses: actions/download-artifact@v5 + with: + name: ${{ inputs.context_artifact_name }} + path: tests/context + + - name: Validate context receipt + id: context_receipt + shell: pwsh + run: | + $receiptPath = 'tests/context/pester-context.json' + if (-not (Test-Path -LiteralPath $receiptPath)) { + throw "Context receipt missing: $receiptPath" + } + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -ErrorAction Stop + if ($receipt.schema -ne 'pester-context-receipt@v1') { + throw ("Unexpected context receipt schema: {0}" -f $receipt.schema) + } + if ($receipt.status -ne 'ready') { + throw ("Context receipt status is not ready: {0}" -f $receipt.status) + } + "path=$receiptPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "repository=$($receipt.repository)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "standing_priority_issue=$($receipt.standingPriority.issueNumber)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + - name: Download readiness receipt artifact uses: actions/download-artifact@v5 with: @@ -306,6 +349,7 @@ jobs: if ($dispatcherExitCode -eq '') { $dispatcherExitCode = '-1' } + $contextReceiptPresent = Test-Path -LiteralPath '${{ steps.context_receipt.outputs.path }}' $readinessReceiptPresent = Test-Path -LiteralPath '${{ steps.readiness_receipt.outputs.path }}' if ((Test-Path -LiteralPath $summaryPath) -and $dispatcherExitCode -eq '0') { $status = 'completed' @@ -315,6 +359,11 @@ jobs: $receipt = [ordered]@{ schema = 'pester-execution-receipt@v1' generatedAtUtc = [DateTime]::UtcNow.ToString('o') + contextStatus = '${{ inputs.context_status }}' + contextReceiptPath = '${{ steps.context_receipt.outputs.path }}' + contextReceiptPresent = $contextReceiptPresent + repository = '${{ steps.context_receipt.outputs.repository }}' + standingPriorityIssue = '${{ steps.context_receipt.outputs.standing_priority_issue }}' readinessStatus = '${{ inputs.readiness_status }}' readinessReceiptPath = '${{ steps.readiness_receipt.outputs.path }}' readinessReceiptPresent = $readinessReceiptPresent @@ -358,7 +407,10 @@ jobs: $executionStatus = 'unknown' $receiptStatus = '${{ needs.pester.outputs.execution_receipt_status }}' $rawArtifactName = '' - if ('${{ inputs.readiness_status }}' -ne 'ready') { + if ('${{ inputs.context_status }}' -ne 'ready') { + $executionStatus = 'skipped' + $receiptStatus = 'context-blocked' + } elseif ('${{ inputs.readiness_status }}' -ne 'ready') { $executionStatus = 'skipped' if ([string]::IsNullOrWhiteSpace($receiptStatus)) { $receiptStatus = 'skipped' } } else { @@ -398,6 +450,7 @@ jobs: $receipt = [ordered]@{ schema = 'pester-execution-receipt@v1' generatedAtUtc = [DateTime]::UtcNow.ToString('o') + contextStatus = '${{ inputs.context_status }}' readinessStatus = '${{ inputs.readiness_status }}' executionJobResult = '${{ needs.pester.result }}' dispatcherExitCode = [int]$dispatcherExitCode diff --git a/docs/knowledgebase/Pester-Service-Model.md b/docs/knowledgebase/Pester-Service-Model.md index 1359ec5c1..50bbb8706 100644 --- a/docs/knowledgebase/Pester-Service-Model.md +++ b/docs/knowledgebase/Pester-Service-Model.md @@ -11,8 +11,10 @@ That coupling is what makes the monolithic self-hosted seam expensive to reprodu ## Pilot Split -The additive pilot introduces four workflow surfaces: +The additive pilot introduces five workflow surfaces: +- `.github/workflows/pester-context.yml` + - repo/control-plane receipts for repository slug, token-backed standing-priority sync, and context classification - `.github/workflows/pester-gate.yml` - top-level router for the pilot service model - `.github/workflows/pester-service-model-on-label.yml` @@ -26,12 +28,15 @@ The additive pilot introduces four workflow surfaces: ## Design Rules +- Context certifies repo/control-plane assumptions. It does not probe host readiness or execute tests. - Readiness certifies the environment. It does not execute the test pack. +- Readiness consumes context. It does not discover standing-priority state itself. +- Selection is still internal to execution in the current pilot baseline; pulling it into its own receipt is the next planned decomposition slice. - Readiness emits a bounded-freshness receipt artifact that execution must download and validate before dispatch. -- Execution consumes readiness. It does not bootstrap Docker runtimes or install core toolchains. +- Execution consumes context and readiness. It does not bootstrap Docker runtimes, install core toolchains, or discover standing-priority state. - Execution writes an execution receipt before uploading raw artifacts so evidence can classify the real seam outcome. - Execution must also emit a skip-safe execution contract from an always-on finalize path so reusable-workflow outputs do not collapse when the execution job never starts. -- Evidence consumes raw execution output plus the execution receipt. It classifies `seam-defect` explicitly when execution never yields a valid summary or never yields a valid execution receipt. +- Evidence consumes raw execution output plus the execution receipt. It classifies `context-blocked`, `readiness-blocked`, and `seam-defect` explicitly instead of collapsing them into one execution symptom. - The existing required gate remains in place until the pilot proves equivalent or better behavior. - Trusted PR proving must stay on `pull_request_target` with same-owner gating. Cross-owner fork heads are not allowed to drive self-hosted execution. diff --git a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs index 30f180517..4a84abfa6 100644 --- a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs +++ b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs @@ -11,7 +11,7 @@ function readRepoFile(relativePath) { return readFileSync(path.join(repoRoot, relativePath), 'utf8'); } -test('pester gate pilot routes readiness, execution, and evidence through separate reusable workflows', () => { +test('pester gate pilot routes context, readiness, execution, and evidence through separate reusable workflows', () => { const workflow = readRepoFile('.github/workflows/pester-gate.yml'); assert.match(workflow, /name:\s+Pester gate \(service model pilot\)/); @@ -22,18 +22,37 @@ test('pester gate pilot routes readiness, execution, and evidence through separa assert.match(workflow, /route_trust_mode:/); assert.match(workflow, /group:\s+pester-gate-\$\{\{\s*inputs\.sample_id \|\| github\.ref\s*\}\}/); assert.match(workflow, /jobs:\s*\n\s*skipped:\s*\n\s+if:\s+\$\{\{\s*!fromJSON\(inputs\.route_should_run \|\| 'true'\)\s*\}\}/); - assert.match(workflow, /\n\s*readiness:\s*\n\s+if:\s+\$\{\{\s*fromJSON\(inputs\.route_should_run \|\| 'true'\)\s*\}\}\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/selfhosted-readiness\.yml/); - assert.match(workflow, /\n\s*pester-run:\s*\n\s+needs:\s+readiness\s*\n\s+if:\s+\$\{\{\s*always\(\) && fromJSON\(inputs\.route_should_run \|\| 'true'\)\s*\}\}\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/pester-run\.yml/); + assert.match(workflow, /\n\s*context:\s*\n\s+if:\s+\$\{\{\s*fromJSON\(inputs\.route_should_run \|\| 'true'\)\s*\}\}\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/pester-context\.yml/); + assert.match(workflow, /\n\s*readiness:\s*\n\s+needs:\s+context\s*\n\s+if:\s+\$\{\{\s*always\(\) && fromJSON\(inputs\.route_should_run \|\| 'true'\) && needs\.context\.outputs\.receipt_status == 'ready'\s*\}\}\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/selfhosted-readiness\.yml/); + assert.match(workflow, /\n\s*pester-run:\s*\n\s+needs:\s+\[context, readiness\]\s*\n\s+if:\s+\$\{\{\s*always\(\) && fromJSON\(inputs\.route_should_run \|\| 'true'\)\s*\}\}\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/pester-run\.yml/); + assert.match(workflow, /context_status:\s+\$\{\{\s*needs\.context\.outputs\.receipt_status\s*\|\|\s*needs\.context\.result/); + assert.match(workflow, /context_artifact_name:\s+\$\{\{\s*needs\.context\.outputs\.receipt_artifact_name\s*\|\|\s*'pester-context'/); assert.match(workflow, /readiness_artifact_name:\s+\$\{\{\s*needs\.readiness\.outputs\.receipt_artifact_name\s*\|\|\s*'pester-readiness'/); assert.match(workflow, /readiness_status:\s+\$\{\{\s*needs\.readiness\.outputs\.receipt_status\s*\|\|\s*needs\.readiness\.result/); assert.match(workflow, /checkout_repository:\s+\$\{\{\s*inputs\.checkout_repository \|\| github\.repository\s*\}\}/); assert.match(workflow, /checkout_ref:\s+\$\{\{\s*inputs\.checkout_ref \|\| github\.sha\s*\}\}/); - assert.match(workflow, /\n\s*pester-evidence:\s*\n\s+needs:\s+\[readiness, pester-run\]\s*\n\s+if:\s+\$\{\{\s*always\(\) && fromJSON\(inputs\.route_should_run \|\| 'true'\)\s*\}\}\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/pester-evidence\.yml/); + assert.match(workflow, /\n\s*pester-evidence:\s*\n\s+needs:\s+\[context, readiness, pester-run\]\s*\n\s+if:\s+\$\{\{\s*always\(\) && fromJSON\(inputs\.route_should_run \|\| 'true'\)\s*\}\}\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/pester-evidence\.yml/); + assert.match(workflow, /context_status:\s+\$\{\{\s*needs\.context\.outputs\.receipt_status\s*\|\|\s*needs\.context\.result/); assert.match(workflow, /execution_job_result:\s+\$\{\{\s*needs\.pester-run\.outputs\.execution_status\s*\|\|\s*needs\.pester-run\.result/); assert.match(workflow, /execution_receipt_artifact_name:\s+\$\{\{\s*needs\.pester-run\.outputs\.execution_receipt_artifact_name/); assert.match(workflow, /### Pester gate \(service model pilot\)/); }); +test('pester context owns repo/control-plane receipts before host readiness begins', () => { + const workflow = readRepoFile('.github/workflows/pester-context.yml'); + + assert.match(workflow, /name:\s+Pester context/); + assert.match(workflow, /workflow_call:/); + assert.match(workflow, /receipt_status:/); + assert.match(workflow, /standing_priority_issue:/); + assert.match(workflow, /standing_priority_reason:/); + assert.match(workflow, /runs-on:\s+ubuntu-latest/); + assert.match(workflow, /Resolve standing-priority context/); + assert.match(workflow, /run-sync-standing-priority\.mjs --materialize-cache/); + assert.match(workflow, /pester-context-receipt@v1/); + assert.match(workflow, /Upload context receipt/); +}); + test('selfhosted readiness owns host-plane certification and emits a receipt artifact', () => { const workflow = readRepoFile('.github/workflows/selfhosted-readiness.yml'); @@ -53,16 +72,19 @@ test('selfhosted readiness owns host-plane certification and emits a receipt art assert.match(workflow, /freshnessWindowSeconds = 900/); }); -test('pester run is execution-only and validates the readiness receipt before dispatch', () => { +test('pester run is execution-only and validates context plus readiness receipts before dispatch', () => { const workflow = readRepoFile('.github/workflows/pester-run.yml'); assert.match(workflow, /name:\s+Pester run/); assert.match(workflow, /name:\s+Pester \(execution only\)/); - assert.match(workflow, /if:\s+\$\{\{\s*inputs\.readiness_status == 'ready'\s*\}\}/); + assert.match(workflow, /if:\s+\$\{\{\s*inputs\.context_status == 'ready' && inputs\.readiness_status == 'ready'\s*\}\}/); assert.match(workflow, /execution_status:/); assert.match(workflow, /execution_receipt_artifact_name:/); assert.match(workflow, /repository:\s+\$\{\{\s*inputs\.checkout_repository \|\| github\.repository\s*\}\}/); assert.match(workflow, /ref:\s+\$\{\{\s*inputs\.checkout_ref \|\| github\.sha\s*\}\}/); + assert.match(workflow, /Download context receipt artifact/); + assert.match(workflow, /Validate context receipt/); + assert.match(workflow, /pester-context\.json/); assert.match(workflow, /Download readiness receipt artifact/); assert.match(workflow, /Validate readiness receipt/); assert.match(workflow, /selfhosted-readiness\.json/); @@ -77,7 +99,7 @@ test('pester run is execution-only and validates the readiness receipt before di assert.doesNotMatch(workflow, /Invoke-DevDashboard\.ps1/); }); -test('pester evidence distinguishes readiness-blocked skips from seam defects', () => { +test('pester evidence distinguishes context-blocked and readiness-blocked skips from seam defects', () => { const workflow = readRepoFile('.github/workflows/pester-evidence.yml'); assert.match(workflow, /name:\s+Pester evidence/); @@ -91,7 +113,10 @@ test('pester evidence distinguishes readiness-blocked skips from seam defects', assert.match(workflow, /Ensure-SessionIndex\.ps1/); assert.match(workflow, /Invoke-DevDashboard\.ps1/); assert.match(workflow, /classification = 'seam-defect'/); + assert.match(workflow, /\$classification = 'context-blocked'/); assert.match(workflow, /\$classification = 'readiness-blocked'/); + assert.match(workflow, /\$contextStatus -ne 'ready'/); + assert.match(workflow, /\$executionReceiptStatus -eq 'context-blocked'/); assert.match(workflow, /\$readinessStatus -ne 'ready' -and \$executionJobResult -in @\('skipped','cancelled'\)/); assert.match(workflow, /raw-artifact-download=/); assert.match(workflow, /execution-receipt-seam-defect/); @@ -103,9 +128,11 @@ test('knowledgebase documents the additive service model and keeps the monolith const doc = readRepoFile('docs/knowledgebase/Pester-Service-Model.md'); assert.match(doc, /legacy Pester control plane couples four concerns into one self-hosted transaction/i); + assert.match(doc, /pester-context\.yml/); assert.match(doc, /selfhosted-readiness\.yml/); assert.match(doc, /pester-run\.yml/); assert.match(doc, /pester-evidence\.yml/); + assert.match(doc, /Context certifies repo\/control-plane assumptions/i); assert.match(doc, /readiness receipt/i); assert.match(doc, /execution receipt/i); assert.match(doc, /existing required gate remains in place/i); From ce4746117f99ef6d8addb921c0a8f8ecc8cb5453 Mon Sep 17 00:00:00 2001 From: svelderrainruiz Date: Tue, 31 Mar 2026 10:18:31 -0700 Subject: [PATCH 25/44] ci(pester): add hosted promotion evidence packet --- .../pester-service-model-quality.yml | 117 +++++++++++++++ .../pester-service-model-release-evidence.yml | 141 ++++++++++++++++++ ...-2078-pester-service-model-requirements.md | 38 +++++ .../pester-service-model-control-plane.md | 87 +++++++++++ docs/cm-plan-pester-service-model.md | 58 +++++++ docs/knowledgebase/Pester-Service-Model.md | 12 ++ ...ster-service-model-information-item-map.md | 29 ++++ docs/pester-service-model-quality-report.md | 34 +++++ .../release-procedure-pester-service-model.md | 35 +++++ docs/requirements-pester-service-model-srs.md | 53 +++++++ docs/rtm-pester-service-model.csv | 7 + .../testing/pester-service-model-test-plan.md | 51 +++++++ ...e-model-quality-workflow-contract.test.mjs | 27 ++++ ...elease-evidence-workflow-contract.test.mjs | 30 ++++ .../write-node-test-coverage-xml.test.mjs | 93 ++++++++++++ ...-pester-service-model-release-evidence.mjs | 139 +++++++++++++++++ ...pester-service-model-promotion-dossier.mjs | 115 ++++++++++++++ .../priority/write-node-test-coverage-xml.mjs | 126 ++++++++++++++++ 18 files changed, 1192 insertions(+) create mode 100644 .github/workflows/pester-service-model-quality.yml create mode 100644 .github/workflows/pester-service-model-release-evidence.yml create mode 100644 docs/architecture/ADR-2078-pester-service-model-requirements.md create mode 100644 docs/architecture/pester-service-model-control-plane.md create mode 100644 docs/cm-plan-pester-service-model.md create mode 100644 docs/pester-service-model-information-item-map.md create mode 100644 docs/pester-service-model-quality-report.md create mode 100644 docs/release-procedure-pester-service-model.md create mode 100644 docs/requirements-pester-service-model-srs.md create mode 100644 docs/rtm-pester-service-model.csv create mode 100644 docs/testing/pester-service-model-test-plan.md create mode 100644 tools/priority/__tests__/pester-service-model-quality-workflow-contract.test.mjs create mode 100644 tools/priority/__tests__/pester-service-model-release-evidence-workflow-contract.test.mjs create mode 100644 tools/priority/__tests__/write-node-test-coverage-xml.test.mjs create mode 100644 tools/priority/materialize-pester-service-model-release-evidence.mjs create mode 100644 tools/priority/render-pester-service-model-promotion-dossier.mjs create mode 100644 tools/priority/write-node-test-coverage-xml.mjs diff --git a/.github/workflows/pester-service-model-quality.yml b/.github/workflows/pester-service-model-quality.yml new file mode 100644 index 000000000..a08aa0795 --- /dev/null +++ b/.github/workflows/pester-service-model-quality.yml @@ -0,0 +1,117 @@ +name: Pester service-model quality + +on: + workflow_dispatch: + pull_request: + branches: + - develop + paths: + - '.github/workflows/pester-service-model-quality.yml' + - '.github/workflows/pester-service-model-release-evidence.yml' + - '.github/workflows/pester-context.yml' + - '.github/workflows/pester-gate.yml' + - '.github/workflows/pester-run.yml' + - '.github/workflows/pester-evidence.yml' + - '.github/workflows/pester-service-model-on-label.yml' + - '.github/workflows/selfhosted-readiness.yml' + - 'docs/knowledgebase/Pester-Service-Model.md' + - 'docs/architecture/pester-service-model-control-plane.md' + - 'docs/architecture/ADR-2078-pester-service-model-requirements.md' + - 'docs/cm-plan-pester-service-model.md' + - 'docs/release-procedure-pester-service-model.md' + - 'docs/requirements-pester-service-model-srs.md' + - 'docs/rtm-pester-service-model.csv' + - 'docs/testing/pester-service-model-test-plan.md' + - 'docs/pester-service-model-quality-report.md' + - 'docs/pester-service-model-information-item-map.md' + - 'tools/priority/write-node-test-coverage-xml.mjs' + - 'tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs' + - 'tools/priority/__tests__/pester-service-model-quality-workflow-contract.test.mjs' + - 'tools/priority/__tests__/write-node-test-coverage-xml.test.mjs' + push: + branches: + - integration/** + - codex/pester-service-model-* + - codex/integration-routing-observability + paths: + - '.github/workflows/pester-service-model-quality.yml' + - '.github/workflows/pester-context.yml' + - '.github/workflows/pester-gate.yml' + - '.github/workflows/pester-run.yml' + - '.github/workflows/pester-evidence.yml' + - '.github/workflows/pester-service-model-on-label.yml' + - '.github/workflows/selfhosted-readiness.yml' + - 'docs/knowledgebase/Pester-Service-Model.md' + - 'docs/architecture/pester-service-model-control-plane.md' + - 'docs/architecture/ADR-2078-pester-service-model-requirements.md' + - 'docs/cm-plan-pester-service-model.md' + - 'docs/release-procedure-pester-service-model.md' + - 'docs/requirements-pester-service-model-srs.md' + - 'docs/rtm-pester-service-model.csv' + - 'docs/testing/pester-service-model-test-plan.md' + - 'docs/pester-service-model-quality-report.md' + - 'docs/pester-service-model-information-item-map.md' + - 'tools/priority/write-node-test-coverage-xml.mjs' + - 'tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs' + - 'tools/priority/__tests__/pester-service-model-quality-workflow-contract.test.mjs' + - 'tools/priority/__tests__/write-node-test-coverage-xml.test.mjs' + +permissions: + contents: read + +jobs: + coverage: + name: PR Coverage Gate / coverage + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Install Node dependencies + shell: bash + run: node tools/npm/cli.mjs ci + + - name: Run packet tests with Node coverage + shell: bash + run: | + mkdir -p tests/results/_agent/pester-service-model + node --test --experimental-test-coverage \ + tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs \ + tools/priority/__tests__/pester-service-model-quality-workflow-contract.test.mjs \ + tools/priority/__tests__/write-node-test-coverage-xml.test.mjs \ + 2>&1 | tee tests/results/_agent/pester-service-model/node-test-coverage.log + + - name: Materialize coverage.xml + shell: bash + run: | + node tools/priority/write-node-test-coverage-xml.mjs \ + --input tests/results/_agent/pester-service-model/node-test-coverage.log \ + --output tests/results/_agent/pester-service-model/coverage.xml \ + --line-threshold 75 + + - name: Upload coverage artifact + uses: actions/upload-artifact@v7 + with: + name: pester-service-model-coverage + path: tests/results/_agent/pester-service-model/coverage.xml + if-no-files-found: error + + docs: + name: Docs link check / lychee + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Run Lychee on packet docs + uses: lycheeverse/lychee-action@v2 + with: + fail: true + args: >- + --no-progress + docs/knowledgebase/Pester-Service-Model.md + docs/architecture/pester-service-model-control-plane.md + docs/architecture/ADR-2078-pester-service-model-requirements.md + docs/cm-plan-pester-service-model.md + docs/release-procedure-pester-service-model.md + docs/requirements-pester-service-model-srs.md + docs/pester-service-model-quality-report.md + docs/pester-service-model-information-item-map.md diff --git a/.github/workflows/pester-service-model-release-evidence.yml b/.github/workflows/pester-service-model-release-evidence.yml new file mode 100644 index 000000000..f0f13ec67 --- /dev/null +++ b/.github/workflows/pester-service-model-release-evidence.yml @@ -0,0 +1,141 @@ +name: Pester service-model release evidence + +on: + workflow_dispatch: + pull_request: + branches: + - develop + paths: + - '.github/workflows/pester-service-model-release-evidence.yml' + - '.github/workflows/pester-service-model-quality.yml' + - '.github/workflows/pester-context.yml' + - '.github/workflows/pester-gate.yml' + - '.github/workflows/pester-run.yml' + - '.github/workflows/pester-evidence.yml' + - '.github/workflows/pester-service-model-on-label.yml' + - '.github/workflows/selfhosted-readiness.yml' + - 'docs/knowledgebase/Pester-Service-Model.md' + - 'docs/architecture/pester-service-model-control-plane.md' + - 'docs/architecture/ADR-2078-pester-service-model-requirements.md' + - 'docs/cm-plan-pester-service-model.md' + - 'docs/release-procedure-pester-service-model.md' + - 'docs/requirements-pester-service-model-srs.md' + - 'docs/rtm-pester-service-model.csv' + - 'docs/testing/pester-service-model-test-plan.md' + - 'docs/pester-service-model-quality-report.md' + - 'docs/pester-service-model-information-item-map.md' + - 'tools/priority/materialize-pester-service-model-release-evidence.mjs' + - 'tools/priority/render-pester-service-model-promotion-dossier.mjs' + - 'tools/priority/write-node-test-coverage-xml.mjs' + - 'tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs' + - 'tools/priority/__tests__/pester-service-model-quality-workflow-contract.test.mjs' + - 'tools/priority/__tests__/pester-service-model-release-evidence-workflow-contract.test.mjs' + - 'tools/priority/__tests__/write-node-test-coverage-xml.test.mjs' + push: + branches: + - integration/** + - codex/pester-service-model-* + - codex/integration-routing-observability + paths: + - '.github/workflows/pester-service-model-release-evidence.yml' + - '.github/workflows/pester-service-model-quality.yml' + - '.github/workflows/pester-context.yml' + - '.github/workflows/pester-gate.yml' + - '.github/workflows/pester-run.yml' + - '.github/workflows/pester-evidence.yml' + - '.github/workflows/pester-service-model-on-label.yml' + - '.github/workflows/selfhosted-readiness.yml' + - 'docs/knowledgebase/Pester-Service-Model.md' + - 'docs/architecture/pester-service-model-control-plane.md' + - 'docs/architecture/ADR-2078-pester-service-model-requirements.md' + - 'docs/cm-plan-pester-service-model.md' + - 'docs/release-procedure-pester-service-model.md' + - 'docs/requirements-pester-service-model-srs.md' + - 'docs/rtm-pester-service-model.csv' + - 'docs/testing/pester-service-model-test-plan.md' + - 'docs/pester-service-model-quality-report.md' + - 'docs/pester-service-model-information-item-map.md' + - 'tools/priority/materialize-pester-service-model-release-evidence.mjs' + - 'tools/priority/render-pester-service-model-promotion-dossier.mjs' + - 'tools/priority/write-node-test-coverage-xml.mjs' + - 'tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs' + - 'tools/priority/__tests__/pester-service-model-quality-workflow-contract.test.mjs' + - 'tools/priority/__tests__/pester-service-model-release-evidence-workflow-contract.test.mjs' + - 'tools/priority/__tests__/write-node-test-coverage-xml.test.mjs' + +permissions: + contents: read + +env: + PSM_UPSTREAM_ISSUE: '2069' + PSM_FORK_ISSUE: '2078' + PSM_FORK_BASIS_COMMIT: 'e188d43768a16286f79b20046e508a7c6e53e2b2' + PSM_FORK_BASIS_URL: 'https://github.com/LabVIEW-Community-CI-CD/compare-vi-cli-action/issues/2078#issuecomment-4164132415' + +jobs: + release-evidence: + name: Release evidence / pester-service-model + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Install Node dependencies + shell: bash + run: node tools/npm/cli.mjs ci + + - name: Run packet tests with Node coverage + shell: bash + run: | + mkdir -p tests/results/_agent/pester-service-model + node --test --experimental-test-coverage \ + tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs \ + tools/priority/__tests__/pester-service-model-quality-workflow-contract.test.mjs \ + tools/priority/__tests__/write-node-test-coverage-xml.test.mjs \ + 2>&1 | tee tests/results/_agent/pester-service-model/node-test-coverage.log + + - name: Materialize coverage.xml + shell: bash + run: | + node tools/priority/write-node-test-coverage-xml.mjs \ + --input tests/results/_agent/pester-service-model/node-test-coverage.log \ + --output tests/results/_agent/pester-service-model/coverage.xml \ + --line-threshold 75 + + - name: Docs link check / lychee + uses: lycheeverse/lychee-action@v2 + with: + fail: true + args: >- + --no-progress + --format json + --output tests/results/_agent/pester-service-model/docs-link-check.json + docs/knowledgebase/Pester-Service-Model.md + docs/architecture/pester-service-model-control-plane.md + docs/architecture/ADR-2078-pester-service-model-requirements.md + docs/cm-plan-pester-service-model.md + docs/release-procedure-pester-service-model.md + docs/requirements-pester-service-model-srs.md + docs/pester-service-model-quality-report.md + docs/pester-service-model-information-item-map.md + + - name: Materialize release evidence bundle + shell: bash + run: | + node tools/priority/materialize-pester-service-model-release-evidence.mjs \ + --version v0.1.0 \ + --upstream-issue "$PSM_UPSTREAM_ISSUE" \ + --fork-issue "$PSM_FORK_ISSUE" \ + --fork-basis-commit "$PSM_FORK_BASIS_COMMIT" \ + --fork-basis-url "$PSM_FORK_BASIS_URL" + node tools/priority/render-pester-service-model-promotion-dossier.mjs \ + --upstream-issue "$PSM_UPSTREAM_ISSUE" \ + --fork-issue "$PSM_FORK_ISSUE" \ + --fork-basis-commit "$PSM_FORK_BASIS_COMMIT" \ + --fork-basis-url "$PSM_FORK_BASIS_URL" + + - name: Upload release-evidence bundle + uses: actions/upload-artifact@v7 + with: + name: pester-service-model-release-evidence + path: tests/results/_agent/pester-service-model/release-evidence + if-no-files-found: error diff --git a/docs/architecture/ADR-2078-pester-service-model-requirements.md b/docs/architecture/ADR-2078-pester-service-model-requirements.md new file mode 100644 index 000000000..ccc3edeb1 --- /dev/null +++ b/docs/architecture/ADR-2078-pester-service-model-requirements.md @@ -0,0 +1,38 @@ +# ADR-2078-PSM-001: Specify The Pester Service Model As A Layered Subsystem + +## Status + +Accepted + +## Context + +The current Pester pilot already separates routing, context, readiness, +execution, and evidence, but those obligations still live mainly in workflow +files and workflow-contract tests. That makes the subsystem executable, but not +yet fully specified or traceable as a control plane. + +## Decision + +Create a dedicated assurance packet for the Pester service model with: +- an SRS +- an RTM +- an architecture packet +- a CM plan and release procedure +- a test plan +- an information-item map +- a retained fork dossier that can justify minimal upstream promotion slices + +## Rationale + +- Requirements should describe stable obligations, not just test behavior. +- Tests should verify the subsystem contract instead of acting as the contract. +- The fork is the correct place to specify and audit the subsystem before + deciding what to mount upstream, but upstream promotion should consume the + resulting dossier as an input rather than rerunning fork-only assurance. + +## Consequences + +- The service-model packet can now be audited locally as a fork design basis. +- Remaining gaps become explicit action items instead of ambient workflow debt. +- Promotion to upstream can be argued from requirements and receipts, not just + from observed workflow behavior. diff --git a/docs/architecture/pester-service-model-control-plane.md b/docs/architecture/pester-service-model-control-plane.md new file mode 100644 index 000000000..655dcd8fb --- /dev/null +++ b/docs/architecture/pester-service-model-control-plane.md @@ -0,0 +1,87 @@ +# Pester Service Model Control Plane + +## Overview + +- System: Pester service-model control plane +- Purpose: + Replace the monolithic self-hosted Pester transaction with explicit control + layers that can be proven, audited, and promoted intentionally. +- Scope: + Trusted routing, context, readiness, execution, evidence, and the additive + promotion boundary. + +## Stakeholders And Concerns + +| Stakeholder | Concern | Viewpoint | +| --- | --- | --- | +| Product | The Pester plane should be an engineered subsystem, not a debugging tangle. | Governance | +| Engineering | Each failure should localize to one layer with one receipt chain. | Execution | +| QA | Requirements, tests, and receipts need end-to-end traceability. | Verification | +| Operations | Self-hosted ingress must stay protected behind trusted routing and readiness contracts. | Runtime | + +## Context View + +- External actors: + Maintainers, trusted same-owner PR heads, `workflow_dispatch`, and the + self-hosted ingress runner. +- Upstream systems: + Standing-priority issue state, release workflows, and the legacy required + Pester gate. +- Downstream systems: + Execution receipts, evidence artifacts, dashboards, and promotion decisions. + +## Container View + +| Container | Responsibility | Technology | +| --- | --- | --- | +| Trusted router | Decide whether the pilot is allowed to run and which ref it should use | GitHub Actions YAML | +| Context layer | Certify repository slug and standing-priority control-plane assumptions | GitHub Actions + Node | +| Readiness layer | Certify self-hosted ingress runtime state and host dependencies | GitHub Actions + PowerShell | +| Execution layer | Run the selected Pester pack after validating upstream receipts | GitHub Actions + PowerShell | +| Evidence layer | Classify results, summarize them, and publish operator artifacts | GitHub Actions + PowerShell | + +## Component View + +| Component | Container | Responsibility | +| --- | --- | --- | +| `pester-service-model-on-label.yml` | Trusted router | Admission control for dispatch and same-owner labeled PRs | +| `pester-context.yml` | Context layer | Repository and standing-priority receipt | +| `selfhosted-readiness.yml` | Readiness layer | Runner labels, session lock, `.NET`, Docker, and LVCompare readiness | +| `pester-run.yml` | Execution layer | Receipt validation, dispatcher invocation, execution contract | +| `pester-evidence.yml` | Evidence layer | Classification, summary, session index, and dashboard publication | + +## Deployment View + +- Environments: + GitHub hosted Ubuntu, self-hosted Windows ingress, and the upstream + integration rail used to prove promotion slices. +- Nodes: + GitHub Actions runners, self-hosted ingress host, repo filesystem, and receipt + artifact storage. +- Runtime dependencies: + GitHub token, standing-priority cache, `dotnet`, Windows Docker runtime, + LVCompare, Session-Lock scripts, and `Invoke-PesterTests.ps1`. + +## Correspondence And Rationale + +- Requirement-to-component notes: + `REQ-PSM-001` maps to the trusted router. + `REQ-PSM-002` maps to context. + `REQ-PSM-003` maps to readiness. + `REQ-PSM-004` maps to execution. + `REQ-PSM-005` maps to evidence. + `REQ-PSM-006` maps to the additive promotion boundary. +- Decision rationale: + The service model exists to separate concerns and make failures classifiable by + layer instead of inferred from one coupled self-hosted run. +- Promotion rationale: + The upstream packet is promoted from a retained fork dossier, but the + upstream slice itself remains hosted-first until that evidence justifies a + broader change. +- Known tradeoffs: + The system gains more artifacts and receipts, but those are the mechanism that + makes the control plane auditable. + +## ADR Index + +- ADR-2078-PSM-001: Model the Pester plane as a specified layered subsystem. diff --git a/docs/cm-plan-pester-service-model.md b/docs/cm-plan-pester-service-model.md new file mode 100644 index 000000000..66730de14 --- /dev/null +++ b/docs/cm-plan-pester-service-model.md @@ -0,0 +1,58 @@ +# Pester Service Model Configuration Management Plan + +## Scope + +- Product or service: + Pester service-model control plane +- Managed baselines: + upstream issue `#2069`, the retained fork dossier for `#2078`, and any + intentional `integration/**` mounts used to prove the subsystem + +## Configuration Items + +| CI | Type | Owner | Baseline Rule | +| --- | --- | --- | --- | +| Service-model workflows | Code | Engineering | Versioned on branch baselines and mounted upstream intentionally | +| Workflow-contract tests | Test artifact | Engineering | Must track the declared layer responsibilities | +| Requirement and promotion packet | Document set | Engineering | Versioned with the promoted upstream slice and linked back to the retained fork dossier | +| Execution and evidence receipts | Artifact | Engineering | Retained as proof for service-model decisions | + +## Versioning + +- Scheme: `v0.1.0` for the service-model assurance packet +- Tag trigger: + A pilot baseline may be tagged when the additive model is stable enough to + compare directly against the monolith +- Release branch rule: + Promotion to upstream or release branches occurs only after explicit proof on + the integration rail + +## Change Control + +| Change Type | Approval | Timing | +| --- | --- | --- | +| Standard | Branch review plus issue log on `#2069` or `#2078` | Before push or mount | +| Urgent | Maintainer approval | Same day | +| Concession | Recorded in the issue ledger with rationale | Before merge or promotion | + +## Status Accounting + +- Record location: + `#2069`, the retained fork dossier for `#2078`, integration proof runs, and + the hosted release-evidence bundles +- Release record owner: + Pester service-model pilot program +- Audit trail: + Git history, integration proof runs, retained release-evidence bundles, and + the retained fork promotion dossier + +## Baseline And Release Evidence + +- Baseline decisions must reference a concrete branch or integration rail state. +- Release flow evidence is anchored by `.github/workflows/release.yml`. +- Packet-level retained evidence is anchored by + `.github/workflows/pester-service-model-release-evidence.yml`. +- The packet release bundle retains `coverage.xml`, `docs-link-check.json`, the + RTM, the quality report, and the generated promotion dossier. +- Promotion remains additive until the service model proves equivalent or better + behavior than the monolith. diff --git a/docs/knowledgebase/Pester-Service-Model.md b/docs/knowledgebase/Pester-Service-Model.md index 50bbb8706..f2d386e80 100644 --- a/docs/knowledgebase/Pester-Service-Model.md +++ b/docs/knowledgebase/Pester-Service-Model.md @@ -48,3 +48,15 @@ The pilot can replace the monolith only after: - execution runs the declared pack without host bootstrap - evidence produces deterministic classifications - PR/release comparisons show better failure localization and lower operator ambiguity + +## Promotion Packet + +The current upstream promotion packet for the pilot is hosted-first: + +- `docs/requirements-pester-service-model-srs.md` +- `docs/rtm-pester-service-model.csv` +- `.github/workflows/pester-service-model-quality.yml` +- `.github/workflows/pester-service-model-release-evidence.yml` + +That packet is derived from the retained fork dossier on `#2078` and is used to +justify the next minimal upstream slice on `#2069`. diff --git a/docs/pester-service-model-information-item-map.md b/docs/pester-service-model-information-item-map.md new file mode 100644 index 000000000..b0b3435b8 --- /dev/null +++ b/docs/pester-service-model-information-item-map.md @@ -0,0 +1,29 @@ +# Pester Service Model Information Item Map + +## Scope + +- Product or service: + Pester service-model control plane +- Repository: + `compare-vi-cli-action` +- Baseline: + Upstream pilot program `#2069` using the retained fork dossier from `#2078` +- Owner: + Engineering + +## Information Items + +| Item Type | Current Path | Owner | Trigger | Proving Evidence | +| --- | --- | --- | --- | --- | +| Plan | `docs/testing/pester-service-model-test-plan.md` | Engineering | Layer or receipt change | Workflow-contract and hosted release-evidence gates stay aligned | +| Specification | `docs/requirements-pester-service-model-srs.md` | Engineering | Contract change | REQ IDs remain linked to tests and workflow/code refs | +| Report | `docs/pester-service-model-quality-report.md` | Engineering | Assurance rerun | Report links to current assurance outputs | +| Procedure | `docs/release-procedure-pester-service-model.md` | Engineering | Mount or promotion decision | Procedure matches the additive promotion flow | +| Architecture | `docs/architecture/pester-service-model-control-plane.md` | Engineering | Layer boundary change | Architecture packet and ADR remain current | +| Traceability | `docs/rtm-pester-service-model.csv` | Engineering | Requirement or verification change | RTM remains current | + +## Notes + +- The service model is a subsystem within the repo, not merely a collection of workflows. +- The retained fork packet exists to justify the first upstream promotion + slices; hosted packet evidence then carries the upstream line forward. diff --git a/docs/pester-service-model-quality-report.md b/docs/pester-service-model-quality-report.md new file mode 100644 index 000000000..3bc737214 --- /dev/null +++ b/docs/pester-service-model-quality-report.md @@ -0,0 +1,34 @@ +# Pester Service Model Quality Report + +## Scope + +This report covers the layered Pester service-model control plane defined by the +trusted router, context, readiness, execution, and evidence workflows. + +## Current Evidence + +- Service-model knowledgebase: + `docs/knowledgebase/Pester-Service-Model.md` +- Requirement packet: + `docs/requirements-pester-service-model-srs.md` +- Traceability matrix: + `docs/rtm-pester-service-model.csv` +- Workflow-contract test: + `tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs` +- Packet quality workflow: + `.github/workflows/pester-service-model-quality.yml` +- Packet release-evidence workflow: + `.github/workflows/pester-service-model-release-evidence.yml` +- Fork promotion dossier basis: + `#2078` comment `4164132415` +- Promotion dossier: + `tests/results/_agent/pester-service-model/release-evidence/promotion-dossier.md` + +## Current Quality Position + +- The subsystem now has explicit requirements and traceability. +- The upstream slice now has dedicated hosted packet-quality and release-evidence gates. +- Coverage and docs integrity now have dedicated packet-level gates. +- Promotion remains blocked until additive proof against the monolith is + intentionally accepted and the retained evidence bundle is used to justify the + upstream slice. diff --git a/docs/release-procedure-pester-service-model.md b/docs/release-procedure-pester-service-model.md new file mode 100644 index 000000000..9c5ff90cf --- /dev/null +++ b/docs/release-procedure-pester-service-model.md @@ -0,0 +1,35 @@ +# Pester Service Model Release Procedure + +## Purpose + +Define how the Pester service-model subsystem moves from a retained fork design +dossier to an upstream-mounted proof baseline and, eventually, a promotable +release surface. + +## Procedure + +1. Review the retained fork promotion dossier and confirm that it still matches + the intended upstream slice. +2. Mount only the intended hosted workflow and packet changes onto the upstream + integration rail. +3. Run `.github/workflows/pester-service-model-quality.yml`. +4. Run `.github/workflows/pester-service-model-release-evidence.yml`. +5. Compare the additive service-model proof against the monolithic gate. +6. Only after positive proof, advance the next minimal promotion slice on + `#2069`. +7. Retain the upstream release-evidence bundle alongside the referenced fork + dossier before proposing any wider service-model promotion. + +## Baseline Rule + +- Fork packet baselines are local design baselines. +- Upstream integration mounts are proving baselines. +- Promotion to release truth requires explicit comparative proof. + +## Status Accounting + +- Status is recorded in the issues and the retained release-evidence outputs. +- The upstream baseline must retain both the requirements packet and the hosted + release-evidence bundle used to justify the move. +- The retained packet release-evidence bundle plus the referenced fork dossier + is the minimum promotion handoff. diff --git a/docs/requirements-pester-service-model-srs.md b/docs/requirements-pester-service-model-srs.md new file mode 100644 index 000000000..d8d7706ab --- /dev/null +++ b/docs/requirements-pester-service-model-srs.md @@ -0,0 +1,53 @@ +# Pester Service Model SRS + +## Document Control + +- System: Pester service-model control plane +- Version: `v0.1.0` +- Owner: `#2069` +- Basis: retained fork promotion dossier under `#2078` +- Status: Active + +## Scope + +- Purpose: + Specify the trusted Pester control plane that separates context, host + readiness, execution, and evidence into auditable workflow surfaces. +- In scope: + Trusted pilot routing, repo/control-plane context certification, self-hosted + readiness receipts, execution-only dispatcher runs, and evidence + classification. +- Out of scope: + Legacy monolithic `test-pester.yml` behavior except where it remains the + current baseline to compare against. + +## Stakeholders + +| Role | Need | Priority | +| --- | --- | --- | +| Product | Replace workflow debugging with specified control-plane behavior | High | +| Engineering | Isolate failures by layer instead of chasing one large self-hosted seam | High | +| QA | Trace requirements to workflow-contract tests and evidence artifacts | High | +| Operations | Keep self-hosted ingress access behind trusted routing and explicit receipts | High | + +## Requirements + +| ID | Requirement | Rationale | Fit Criterion | Verification | +| --- | --- | --- | --- | --- | +| REQ-PSM-001 | The trusted Pester pilot shall admit `workflow_dispatch` and same-owner PR heads carrying the `pester-service-model` label, and shall reject untrusted cross-owner fork heads before self-hosted execution begins. | Trust and admission are part of the subsystem boundary, not incidental workflow behavior. | `pester-service-model-on-label.yml` writes `should_run=true` only for dispatch or trusted same-owner heads, and emits `untrusted-cross-owner-fork` for disallowed heads. | `TEST-PSM-001` | +| REQ-PSM-002 | The context layer shall resolve repository and standing-priority state and emit a context receipt before readiness begins. | Repo/control-plane assumptions must be certified separately from host readiness. | `pester-context.yml` uploads `pester-context.json` with `schema=pester-context-receipt@v1` and a status of `ready`, `warning`, or `blocked`. | `TEST-PSM-002` | +| REQ-PSM-003 | The readiness layer shall certify runner labels, session-lock health, `.NET`, Windows Docker runtime, and LVCompare or idle LabVIEW state, and shall emit a bounded-freshness readiness receipt. | Self-hosted ingress debt must be observable as readiness debt, not hidden inside execution. | `selfhosted-readiness.yml` uploads `selfhosted-readiness.json` with `schema=pester-selfhosted-readiness-receipt@v1`, individual probe outcomes, and `freshnessWindowSeconds=900`. | `TEST-PSM-003` | +| REQ-PSM-004 | The execution layer shall validate context and readiness receipts before dispatch, run the declared Pester pack without bootstrapping Docker runtimes or core toolchains, and shall emit an execution receipt even when execution is skipped. | Execution should only execute; it must not absorb context or readiness responsibilities. | `pester-run.yml` refuses to start unless both receipts are ready, calls `Invoke-PesterTests.ps1`, uploads raw results when produced, and always uploads `pester-run-receipt.json`. | `TEST-PSM-004` | +| REQ-PSM-005 | The evidence layer shall classify `context-blocked`, `readiness-blocked`, `test-failures`, and `seam-defect` explicitly from execution receipts and raw artifacts. | Operators need precise failure classes instead of `missing-summary` ambiguity. | `pester-evidence.yml` reads the execution contract and emits the explicit classification when raw artifacts are missing or execution is skipped. | `TEST-PSM-005` | +| REQ-PSM-006 | The pilot shall remain additive until it proves equivalent or better behavior than the monolithic required gate. | Promotion must follow evidence, not preference. | The service-model knowledgebase and promotion rule state that the legacy required gate remains in place until the pilot is proven. | `TEST-PSM-006` | + +## Assumptions + +- The repo continues to use GitHub Actions for the Pester control plane. +- Trusted self-hosted ingress proof remains on the upstream environment. + +## Constraints + +- Cross-owner fork heads shall not drive self-hosted execution. +- Upstream promotion must follow the trusted integration rail and retain a + reference to the fork dossier that justified the slice. diff --git a/docs/rtm-pester-service-model.csv b/docs/rtm-pester-service-model.csv new file mode 100644 index 000000000..6f350d92f --- /dev/null +++ b/docs/rtm-pester-service-model.csv @@ -0,0 +1,7 @@ +ReqID,Requirement,Source,Priority,TestID,TestArtifact,CodeRef,Status +REQ-PSM-001,"Trusted router admits workflow_dispatch and same-owner labeled PR heads; rejects cross-owner heads","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-001,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs",".github/workflows/pester-service-model-on-label.yml",Implemented +REQ-PSM-002,"Context resolves repository and standing-priority state before readiness and emits a receipt","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-002,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs",".github/workflows/pester-context.yml;tools/priority/run-sync-standing-priority.mjs",Implemented +REQ-PSM-003,"Readiness certifies runner labels, session lock, dotnet, Docker runtime, and LVCompare state and emits a bounded-freshness receipt","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-003,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs",".github/workflows/selfhosted-readiness.yml;tools/Invoke-DockerRuntimeManager.ps1;tools/Session-Lock.ps1",Implemented +REQ-PSM-004,"Execution validates context and readiness receipts, runs Invoke-PesterTests without host bootstrap, and emits an execution receipt even when skipped","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-004,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs",".github/workflows/pester-run.yml;Invoke-PesterTests.ps1",Implemented +REQ-PSM-005,"Evidence classifies context-blocked, readiness-blocked, test-failures, and seam-defect explicitly from the execution contract","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-005,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs",".github/workflows/pester-evidence.yml",Implemented +REQ-PSM-006,"The pilot remains additive until it proves equivalent or better than the monolith","docs/requirements-pester-service-model-srs.md",Medium,TEST-PSM-006,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs","docs/knowledgebase/Pester-Service-Model.md;.github/workflows/pester-gate.yml",Implemented diff --git a/docs/testing/pester-service-model-test-plan.md b/docs/testing/pester-service-model-test-plan.md new file mode 100644 index 000000000..a463755c2 --- /dev/null +++ b/docs/testing/pester-service-model-test-plan.md @@ -0,0 +1,51 @@ +# Pester Service Model Test Plan + +## Overview + +- Release or baseline: + Pester service-model assurance packet `v0.1.0` +- Owner: + `#2069` with retained fork basis on `#2078` +- Scope: + Trusted routing, context receipts, readiness receipts, execution-only + behavior, and evidence classification for the Pester service model + +## Test Items + +| Item | Type | Risk | Notes | +| --- | --- | --- | --- | +| `pester-service-model-workflow-contract.test.mjs` | Integration | High | Verifies the workflow split and core receipt/evidence obligations | +| `pester-service-model-quality-workflow-contract.test.mjs` | Integration | Medium | Verifies the coverage gate and docs link-check control-plane workflow | +| `pester-gate.yml` + trusted pilot routing | Workflow | High | Verifies admission and orchestration across layers | +| `Invoke-PesterTests.ps1` execution contract | Execution | High | Verifies the dispatcher remains the execution engine only | + +## Entry Criteria + +- The service-model workflows and contract test are in sync with the declared requirements. +- The retained fork dossier still matches the mounted upstream promotion slice. + +## Exit Criteria + +- Workflow-contract tests pass. +- Hosted packet quality and release-evidence workflows complete. +- Any remaining action items are explicitly accepted before the slice widens + beyond hosted quality and evidence. + +## Coverage Targets + +| Metric | Target | Evidence | +| --- | --- | --- | +| Workflow contract coverage | All layer responsibilities represented | `tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs` | +| Receipt coverage | Context, readiness, execution, and evidence all emit auditable artifacts | assurance report + integration runs | +| Classification coverage | blocked and defect outcomes remain distinguishable | evidence workflow outputs | +| Packet coverage gate | Retained `coverage.xml` and named PR coverage gate | `.github/workflows/pester-service-model-quality.yml` | +| Promotion bundle retention | Hosted bundle retains the minimal promotion handoff | `.github/workflows/pester-service-model-release-evidence.yml` | + +## Reporting + +- CI artifacts: + upstream integration runs plus hosted release-evidence outputs +- Test report location: + `tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs` +- Defect tracking link: + `#2069` and `#2078` diff --git a/tools/priority/__tests__/pester-service-model-quality-workflow-contract.test.mjs b/tools/priority/__tests__/pester-service-model-quality-workflow-contract.test.mjs new file mode 100644 index 000000000..9b2269798 --- /dev/null +++ b/tools/priority/__tests__/pester-service-model-quality-workflow-contract.test.mjs @@ -0,0 +1,27 @@ +#!/usr/bin/env node + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { readFileSync } from 'node:fs'; + +const repoRoot = process.cwd(); + +function readRepoFile(relativePath) { + return readFileSync(path.join(repoRoot, relativePath), 'utf8'); +} + +test('pester service-model quality workflow publishes coverage evidence and docs link integrity', () => { + const workflow = readRepoFile('.github/workflows/pester-service-model-quality.yml'); + + assert.match(workflow, /name:\s+Pester service-model quality/); + assert.match(workflow, /pull_request:/); + assert.match(workflow, /integration\/\*\*/); + assert.match(workflow, /name:\s+PR Coverage Gate \/ coverage/); + assert.match(workflow, /node --test --experimental-test-coverage/); + assert.match(workflow, /write-node-test-coverage-xml\.mjs/); + assert.match(workflow, /coverage\.xml/); + assert.match(workflow, /upload-artifact@v7/); + assert.match(workflow, /name:\s+Docs link check \/ lychee/); + assert.match(workflow, /lycheeverse\/lychee-action@v2/); +}); diff --git a/tools/priority/__tests__/pester-service-model-release-evidence-workflow-contract.test.mjs b/tools/priority/__tests__/pester-service-model-release-evidence-workflow-contract.test.mjs new file mode 100644 index 000000000..47829c126 --- /dev/null +++ b/tools/priority/__tests__/pester-service-model-release-evidence-workflow-contract.test.mjs @@ -0,0 +1,30 @@ +#!/usr/bin/env node + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { readFileSync } from 'node:fs'; + +const repoRoot = process.cwd(); + +function readRepoFile(relativePath) { + return readFileSync(path.join(repoRoot, relativePath), 'utf8'); +} + +test('pester service-model release-evidence workflow retains hosted promotion evidence without fork-only runtime dependencies', () => { + const workflow = readRepoFile('.github/workflows/pester-service-model-release-evidence.yml'); + + assert.match(workflow, /name:\s+Pester service-model release evidence/); + assert.match(workflow, /workflow_dispatch:/); + assert.match(workflow, /pull_request:/); + assert.match(workflow, /PSM_FORK_BASIS_COMMIT/); + assert.match(workflow, /PSM_FORK_BASIS_URL/); + assert.match(workflow, /name:\s+Release evidence \/ pester-service-model/); + assert.match(workflow, /write-node-test-coverage-xml\.mjs/); + assert.match(workflow, /coverage\.xml/); + assert.match(workflow, /Docs link check \/ lychee/); + assert.doesNotMatch(workflow, /fork-lane-local-assurance-ci\.mjs/); + assert.match(workflow, /materialize-pester-service-model-release-evidence\.mjs/); + assert.match(workflow, /render-pester-service-model-promotion-dossier\.mjs/); + assert.match(workflow, /Upload release-evidence bundle/); +}); diff --git a/tools/priority/__tests__/write-node-test-coverage-xml.test.mjs b/tools/priority/__tests__/write-node-test-coverage-xml.test.mjs new file mode 100644 index 000000000..06a44c36a --- /dev/null +++ b/tools/priority/__tests__/write-node-test-coverage-xml.test.mjs @@ -0,0 +1,93 @@ +#!/usr/bin/env node + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { buildCoberturaXml, extractCoverageMetrics, materializeCoverageXml, parseArgs } from '../write-node-test-coverage-xml.mjs'; + +test('extractCoverageMetrics parses aggregate node coverage summary', () => { + const sample = ` +ℹ start of coverage report +ℹ ---------------------------------------------------------- +ℹ file | line % | branch % | funcs % | uncovered lines +ℹ ---------------------------------------------------------- +ℹ ---------------------------------------------------------- +ℹ all files | 100.00 | 87.50 | 92.30 | +ℹ ---------------------------------------------------------- +ℹ end of coverage report +`; + const metrics = extractCoverageMetrics(sample); + assert.deepEqual(metrics, { + lineRatePercent: 100, + branchRatePercent: 87.5, + functionRatePercent: 92.3 + }); +}); + +test('buildCoberturaXml writes aggregate line and branch rates', () => { + const xml = buildCoberturaXml({ + lineRatePercent: 100, + branchRatePercent: 87.5, + functionRatePercent: 92.3, + lineThreshold: 75 + }); + assert.match(xml, /line-rate="1.0000"/); + assert.match(xml, /branch-rate="0.8750"/); + assert.match(xml, /thresholds line="75"/); +}); + +test('parseArgs accepts explicit paths and threshold', () => { + const parsed = parseArgs(['--input', 'in.log', '--output', 'coverage.xml', '--line-threshold', '80']); + assert.deepEqual(parsed, { + input: 'in.log', + output: 'coverage.xml', + lineThreshold: 80 + }); +}); + +test('parseArgs returns help sentinel', () => { + assert.deepEqual(parseArgs(['--help']), { help: true }); +}); + +test('materializeCoverageXml writes output on passing threshold', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'coverage-xml-')); + const inputPath = path.join(tempDir, 'node-test.log'); + const outputPath = path.join(tempDir, 'coverage.xml'); + await fs.writeFile(inputPath, ` +ℹ start of coverage report +ℹ all files | 100.00 | 87.50 | 92.30 | +ℹ end of coverage report +`, 'utf8'); + + const result = await materializeCoverageXml({ + inputPath, + outputPath, + lineThreshold: 75 + }); + + const xml = await fs.readFile(outputPath, 'utf8'); + assert.equal(result.metrics.lineRatePercent, 100); + assert.match(xml, /coverage line-rate="1.0000"/); +}); + +test('materializeCoverageXml fails when threshold is not met', async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'coverage-xml-')); + const inputPath = path.join(tempDir, 'node-test.log'); + const outputPath = path.join(tempDir, 'coverage.xml'); + await fs.writeFile(inputPath, ` +ℹ start of coverage report +ℹ all files | 48.72 | 66.67 | 50.00 | +ℹ end of coverage report +`, 'utf8'); + + await assert.rejects( + materializeCoverageXml({ + inputPath, + outputPath, + lineThreshold: 75 + }), + /below threshold 75/ + ); +}); diff --git a/tools/priority/materialize-pester-service-model-release-evidence.mjs b/tools/priority/materialize-pester-service-model-release-evidence.mjs new file mode 100644 index 000000000..22c034a9c --- /dev/null +++ b/tools/priority/materialize-pester-service-model-release-evidence.mjs @@ -0,0 +1,139 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import process from 'node:process'; +import { spawnSync } from 'node:child_process'; + +const repoRoot = process.cwd(); +const defaultBase = path.join(repoRoot, 'tests', 'results', '_agent', 'pester-service-model'); +const defaultOutputDir = path.join(defaultBase, 'release-evidence'); + +function parseArgs(argv = process.argv.slice(2)) { + const options = { + version: 'v0.1.0', + outputDir: defaultOutputDir, + upstreamIssue: '2069', + forkIssue: '2078', + forkBasisCommit: '', + forkBasisUrl: '' + }; + + for (let i = 0; i < argv.length; i += 1) { + const token = argv[i]; + const next = argv[i + 1]; + if (token === '--version') { + options.version = next; + i += 1; + continue; + } + if (token === '--output-dir') { + options.outputDir = path.resolve(next); + i += 1; + continue; + } + if (token === '--upstream-issue') { + options.upstreamIssue = next; + i += 1; + continue; + } + if (token === '--fork-issue') { + options.forkIssue = next; + i += 1; + continue; + } + if (token === '--fork-basis-commit') { + options.forkBasisCommit = next; + i += 1; + continue; + } + if (token === '--fork-basis-url') { + options.forkBasisUrl = next; + i += 1; + continue; + } + throw new Error(`Unknown option: ${token}`); + } + + return options; +} + +function runGit(args) { + const result = spawnSync('git', args, { cwd: repoRoot, encoding: 'utf8' }); + if (result.status !== 0) { + throw new Error(result.stderr?.trim() || `git ${args.join(' ')} failed`); + } + return result.stdout.trim(); +} + +async function ensureFile(filePath) { + await fs.access(filePath); +} + +async function copyIntoBundle(source, bundleRoot, name = path.basename(source)) { + const destination = path.join(bundleRoot, name); + await fs.copyFile(source, destination); + return destination; +} + +async function main() { + const options = parseArgs(); + await fs.mkdir(options.outputDir, { recursive: true }); + + const coverageXml = path.join(defaultBase, 'coverage.xml'); + const docsLinkCheck = path.join(defaultBase, 'docs-link-check.json'); + const requirementsPath = path.join(repoRoot, 'docs', 'requirements-pester-service-model-srs.md'); + const rtmPath = path.join(repoRoot, 'docs', 'rtm-pester-service-model.csv'); + const qualityReportPath = path.join(repoRoot, 'docs', 'pester-service-model-quality-report.md'); + + await Promise.all([ + ensureFile(coverageXml), + ensureFile(docsLinkCheck), + ensureFile(requirementsPath), + ensureFile(rtmPath), + ensureFile(qualityReportPath) + ]); + + await Promise.all([ + copyIntoBundle(coverageXml, options.outputDir), + copyIntoBundle(docsLinkCheck, options.outputDir), + copyIntoBundle(requirementsPath, options.outputDir), + copyIntoBundle(rtmPath, options.outputDir), + copyIntoBundle(qualityReportPath, options.outputDir) + ]); + + const recordPath = path.join(options.outputDir, `release-record-${options.version}.md`); + const headSha = runGit(['rev-parse', 'HEAD']); + const branch = runGit(['rev-parse', '--abbrev-ref', 'HEAD']); + + const lines = [ + `# Pester Service Model Release Record ${options.version}`, + '', + `- Baseline: ${options.version}`, + `- Branch: ${branch}`, + `- Commit: ${headSha}`, + `- Upstream epic: #${options.upstreamIssue}`, + `- Fork basis issue: #${options.forkIssue}`, + `- Fork basis commit: ${options.forkBasisCommit || 'unspecified'}`, + `- Fork basis reference: ${options.forkBasisUrl || 'unspecified'}`, + '- Change control: retained fork dossier plus upstream-hosted promotion evidence', + '- Status accounting: retained through the bundle files below', + '', + '## Retained Evidence', + '', + '- `coverage.xml`', + '- `docs-link-check.json`', + '- `requirements-pester-service-model-srs.md`', + '- `rtm-pester-service-model.csv`', + '- `pester-service-model-quality-report.md`', + '' + ]; + await fs.writeFile(recordPath, `${lines.join('\n')}\n`, 'utf8'); + + console.log(`release_evidence_dir=${options.outputDir}`); +} + +main().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/tools/priority/render-pester-service-model-promotion-dossier.mjs b/tools/priority/render-pester-service-model-promotion-dossier.mjs new file mode 100644 index 000000000..56d8c8208 --- /dev/null +++ b/tools/priority/render-pester-service-model-promotion-dossier.mjs @@ -0,0 +1,115 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import process from 'node:process'; +import { spawnSync } from 'node:child_process'; + +const repoRoot = process.cwd(); +const baseDir = path.join(repoRoot, 'tests', 'results', '_agent', 'pester-service-model'); +const releaseEvidenceDir = path.join(baseDir, 'release-evidence'); +const outputPath = path.join(releaseEvidenceDir, 'promotion-dossier.md'); + +function parseArgs(argv = process.argv.slice(2)) { + const options = { + upstreamIssue: '2069', + forkIssue: '2078', + forkBasisCommit: '', + forkBasisUrl: '' + }; + + for (let i = 0; i < argv.length; i += 1) { + const token = argv[i]; + const next = argv[i + 1]; + if (token === '--upstream-issue') { + options.upstreamIssue = next; + i += 1; + continue; + } + if (token === '--fork-issue') { + options.forkIssue = next; + i += 1; + continue; + } + if (token === '--fork-basis-commit') { + options.forkBasisCommit = next; + i += 1; + continue; + } + if (token === '--fork-basis-url') { + options.forkBasisUrl = next; + i += 1; + continue; + } + throw new Error(`Unknown option: ${token}`); + } + + return options; +} + +function runGit(args) { + const result = spawnSync('git', args, { cwd: repoRoot, encoding: 'utf8' }); + if (result.status !== 0) { + throw new Error(result.stderr?.trim() || `git ${args.join(' ')} failed`); + } + return result.stdout.trim(); +} + +async function ensureFile(filePath) { + await fs.access(filePath); +} + +async function main() { + const options = parseArgs(); + const coveragePath = path.join(releaseEvidenceDir, 'coverage.xml'); + const docsPath = path.join(releaseEvidenceDir, 'docs-link-check.json'); + const rtm = await fs.readFile(path.join(repoRoot, 'docs', 'rtm-pester-service-model.csv'), 'utf8'); + + await fs.mkdir(releaseEvidenceDir, { recursive: true }); + await Promise.all([ensureFile(coveragePath), ensureFile(docsPath)]); + + const lines = [ + '# Pester Service Model Promotion Dossier', + '', + `- Upstream slice commit: ${runGit(['rev-parse', 'HEAD'])}`, + `- Upstream epic: \`#${options.upstreamIssue}\``, + `- Fork basis issue: \`#${options.forkIssue}\``, + `- Fork basis commit: ${options.forkBasisCommit || 'unspecified'}`, + `- Fork basis reference: ${options.forkBasisUrl || 'unspecified'}`, + '- Hosted promotion evidence retained: yes', + '', + '## Promotion Basis', + '', + '- This upstream slice is derived from the retained fork promotion dossier and requirement packet.', + '- It promotes only hosted quality and release-evidence surfaces.', + '- It does not change the self-hosted execution boundary or required gate ownership.', + '', + '## Requirement Traceability', + '', + '```csv', + rtm.trim(), + '```', + '', + '## Promotion Evidence', + '', + '- the service-model requirement packet is now present on the upstream slice', + '- hosted packet quality retains `coverage.xml`', + '- hosted docs integrity retains `docs-link-check.json`', + '- the release-evidence bundle retains the upstream quality report and RTM alongside the evidence artifacts', + '', + '## Minimal Upstream Slice', + '', + '1. Keep the slice hosted-only: quality, packet docs, and retained promotion evidence.', + '2. Re-prove the mounted slice on the upstream integration rail before widening self-hosted behavior.', + '3. Use this dossier to justify the next service-model promotion step under `#2069`.', + '' + ]; + + await fs.writeFile(outputPath, `${lines.join('\n')}\n`, 'utf8'); + console.log(`promotion_dossier=${outputPath}`); +} + +main().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/tools/priority/write-node-test-coverage-xml.mjs b/tools/priority/write-node-test-coverage-xml.mjs new file mode 100644 index 000000000..dadf8df5f --- /dev/null +++ b/tools/priority/write-node-test-coverage-xml.mjs @@ -0,0 +1,126 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +const HELP = [ + 'Usage: node tools/priority/write-node-test-coverage-xml.mjs --input --output [--line-threshold ]', + '', + 'Parses the console coverage summary emitted by `node --test --experimental-test-coverage`', + 'and writes a minimal Cobertura-style coverage.xml artifact.' +]; + +function parseArgs(argv = process.argv.slice(2)) { + const options = { + input: '', + output: '', + lineThreshold: 75 + }; + + for (let i = 0; i < argv.length; i += 1) { + const token = argv[i]; + const next = argv[i + 1]; + if (token === '--help' || token === '-h') { + return { help: true }; + } + if (token === '--input') { + options.input = next; + i += 1; + continue; + } + if (token === '--output') { + options.output = next; + i += 1; + continue; + } + if (token === '--line-threshold') { + options.lineThreshold = Number(next); + i += 1; + continue; + } + throw new Error(`Unknown option: ${token}`); + } + + if (!options.input || !options.output) { + throw new Error('Both --input and --output are required.'); + } + + return options; +} + +function extractCoverageMetrics(text) { + const allFilesMatch = text.match(/all files\s+\|\s+([0-9.]+)\s+\|\s+([0-9.]+)\s+\|\s+([0-9.]+)/i); + if (!allFilesMatch) { + throw new Error('Unable to locate aggregate coverage metrics in node test output.'); + } + return { + lineRatePercent: Number(allFilesMatch[1]), + branchRatePercent: Number(allFilesMatch[2]), + functionRatePercent: Number(allFilesMatch[3]) + }; +} + +function toRate(percent) { + return (percent / 100).toFixed(4); +} + +function buildCoberturaXml({ lineRatePercent, branchRatePercent, functionRatePercent, lineThreshold }) { + const lineRate = toRate(lineRatePercent); + const branchRate = toRate(branchRatePercent); + return ` + + + . + + + + + + + + + + + + + +`; +} + +async function materializeCoverageXml({ inputPath, outputPath, lineThreshold }) { + const inputText = await fs.readFile(inputPath, 'utf8'); + const metrics = extractCoverageMetrics(inputText); + const xml = buildCoberturaXml({ ...metrics, lineThreshold }); + await fs.mkdir(path.dirname(outputPath), { recursive: true }); + await fs.writeFile(outputPath, xml, 'utf8'); + if (metrics.lineRatePercent < lineThreshold) { + throw new Error(`Line coverage ${metrics.lineRatePercent}% is below threshold ${lineThreshold}%`); + } + return { metrics, outputPath }; +} + +export { parseArgs, extractCoverageMetrics, buildCoberturaXml, materializeCoverageXml }; + +async function main() { + const options = parseArgs(); + if (options.help) { + console.log(HELP.join('\n')); + return; + } + await materializeCoverageXml({ + inputPath: options.input, + outputPath: options.output, + lineThreshold: options.lineThreshold + }); + console.log(`coverage_xml=${options.output}`); +} + +const isEntrypoint = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (isEntrypoint) { + main().catch((error) => { + console.error(error.stack || error.message); + process.exit(1); + }); +} From 790ae420f27231f6e9a9a6da02cbaa51fef72469 Mon Sep 17 00:00:00 2001 From: svelderrainruiz Date: Tue, 31 Mar 2026 10:21:01 -0700 Subject: [PATCH 26/44] ci(pester): pin hosted packet checkouts to PR head --- .github/workflows/pester-service-model-quality.yml | 6 ++++++ .github/workflows/pester-service-model-release-evidence.yml | 3 +++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/pester-service-model-quality.yml b/.github/workflows/pester-service-model-quality.yml index a08aa0795..6cda6daa7 100644 --- a/.github/workflows/pester-service-model-quality.yml +++ b/.github/workflows/pester-service-model-quality.yml @@ -65,6 +65,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 + with: + repository: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - name: Install Node dependencies shell: bash @@ -100,6 +103,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 + with: + repository: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - name: Run Lychee on packet docs uses: lycheeverse/lychee-action@v2 diff --git a/.github/workflows/pester-service-model-release-evidence.yml b/.github/workflows/pester-service-model-release-evidence.yml index f0f13ec67..9d067533b 100644 --- a/.github/workflows/pester-service-model-release-evidence.yml +++ b/.github/workflows/pester-service-model-release-evidence.yml @@ -78,6 +78,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 + with: + repository: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - name: Install Node dependencies shell: bash From e71b8e055bb52edb56657cb8705f6764c2808c2e Mon Sep 17 00:00:00 2001 From: svelderrainruiz Date: Tue, 31 Mar 2026 10:34:52 -0700 Subject: [PATCH 27/44] ci(pester): split selection receipt from execution --- .github/actions/dispatcher-profile/action.yml | 28 +++ .github/workflows/pester-evidence.yml | 16 ++ .github/workflows/pester-gate.yml | 20 ++- .github/workflows/pester-run.yml | 150 ++++++++++------- .github/workflows/pester-selection.yml | 159 ++++++++++++++++++ .../pester-service-model-control-plane.md | 11 +- docs/knowledgebase/Pester-Service-Model.md | 10 +- docs/pester-service-model-quality-report.md | 3 +- docs/requirements-pester-service-model-srs.md | 11 +- docs/rtm-pester-service-model.csv | 7 +- .../testing/pester-service-model-test-plan.md | 8 +- ...r-service-model-workflow-contract.test.mjs | 43 ++++- 12 files changed, 375 insertions(+), 91 deletions(-) create mode 100644 .github/workflows/pester-selection.yml diff --git a/.github/actions/dispatcher-profile/action.yml b/.github/actions/dispatcher-profile/action.yml index 04bd29c30..68632fcc4 100644 --- a/.github/actions/dispatcher-profile/action.yml +++ b/.github/actions/dispatcher-profile/action.yml @@ -44,6 +44,27 @@ outputs: emit_failures_json_always: description: 'Emit failures json always (true/false)' value: ${{ steps.set.outputs.emit_failures_json_always }} + detect_leaks: + description: 'Detect lingering processes after execution (true/false)' + value: ${{ steps.set.outputs.detect_leaks }} + fail_on_leaks: + description: 'Fail the run on detected leaks (true/false)' + value: ${{ steps.set.outputs.fail_on_leaks }} + kill_leaks: + description: 'Attempt to kill leaks when detected (true/false)' + value: ${{ steps.set.outputs.kill_leaks }} + leak_grace_seconds: + description: 'Grace period before the final leak check' + value: ${{ steps.set.outputs.leak_grace_seconds }} + clean_labview_before: + description: 'Clean LabVIEW before execution (true/false)' + value: ${{ steps.set.outputs.clean_labview_before }} + clean_after: + description: 'Clean LabVIEW after execution (true/false)' + value: ${{ steps.set.outputs.clean_after }} + track_artifacts: + description: 'Track execution artifacts (true/false)' + value: ${{ steps.set.outputs.track_artifacts }} runs: using: composite steps: @@ -73,6 +94,13 @@ runs: # Outputs for dispatcher parameters "timeout_seconds=$to" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 "emit_failures_json_always=$emit" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "detect_leaks=$detect" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "fail_on_leaks=$fail" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "kill_leaks=$kill" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "leak_grace_seconds=$grace" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "clean_labview_before=$cleanBefore" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "clean_after=$cleanAfter" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "track_artifacts=$track" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 if ($env:GITHUB_STEP_SUMMARY) { $lines = @('### Dispatcher Profile','') diff --git a/.github/workflows/pester-evidence.yml b/.github/workflows/pester-evidence.yml index 40dcf7309..f496102e8 100644 --- a/.github/workflows/pester-evidence.yml +++ b/.github/workflows/pester-evidence.yml @@ -23,6 +23,10 @@ on: required: false type: string default: 'unknown' + selection_status: + required: false + type: string + default: 'unknown' execution_job_result: required: false type: string @@ -77,6 +81,11 @@ on: required: false default: 'unknown' type: string + selection_status: + description: 'Selection receipt status' + required: false + default: 'unknown' + type: string execution_job_result: description: 'Execution contract outcome' required: false @@ -280,6 +289,7 @@ jobs: $reasons = New-Object System.Collections.Generic.List[string] $contextStatus = '${{ inputs.context_status }}' $readinessStatus = '${{ inputs.readiness_status }}' + $selectionStatus = '${{ inputs.selection_status }}' $executionJobResult = '${{ inputs.execution_job_result }}' $executionReceiptPresent = '${{ steps.execution_receipt.outputs.present }}' $executionReceiptStatus = '${{ steps.execution_receipt.outputs.status }}' @@ -289,6 +299,9 @@ jobs: if ($readinessStatus -ne 'ready') { $reasons.Add(("readiness-status={0}" -f $readinessStatus)) | Out-Null } + if ($selectionStatus -ne 'ready') { + $reasons.Add(("selection-status={0}" -f $selectionStatus)) | Out-Null + } if ($executionJobResult -eq 'skipped') { $reasons.Add('execution-job-skipped') | Out-Null } elseif ($executionJobResult -eq 'cancelled') { @@ -309,6 +322,8 @@ jobs: $classification = 'context-blocked' } elseif ($readinessStatus -ne 'ready' -and $executionJobResult -in @('skipped','cancelled')) { $classification = 'readiness-blocked' + } elseif (($selectionStatus -ne 'ready' -and $executionJobResult -in @('skipped','cancelled')) -or $executionReceiptStatus -eq 'selection-blocked') { + $classification = 'selection-blocked' } elseif ($executionReceiptStatus -eq 'seam-defect') { $reasons.Add('execution-receipt-seam-defect') | Out-Null } elseif ($executionReceiptStatus -eq 'test-failures') { @@ -336,6 +351,7 @@ jobs: generatedAtUtc = [DateTime]::UtcNow.ToString('o') contextStatus = $contextStatus readinessStatus = $readinessStatus + selectionStatus = $selectionStatus executionJobResult = $executionJobResult rawArtifactDownload = if ('${{ steps.artifact_name.outputs.should_download }}' -eq 'true') { '${{ steps.download.outcome }}' } else { 'skipped' } dispatcherExitCode = [int]$dispatcherExitCode diff --git a/.github/workflows/pester-gate.yml b/.github/workflows/pester-gate.yml index a795c0397..4cfcb7c7b 100644 --- a/.github/workflows/pester-gate.yml +++ b/.github/workflows/pester-gate.yml @@ -119,23 +119,34 @@ jobs: checkout_repository: ${{ inputs.checkout_repository || github.repository }} checkout_ref: ${{ inputs.checkout_ref || github.sha }} + selection: + needs: context + if: ${{ always() && fromJSON(inputs.route_should_run || 'true') && needs.context.outputs.receipt_status == 'ready' }} + uses: ./.github/workflows/pester-selection.yml + with: + include_integration: ${{ inputs.include_integration || 'false' }} + include_patterns: ${{ inputs.include_patterns || '' }} + sample_id: ${{ inputs.sample_id || '' }} + checkout_repository: ${{ inputs.checkout_repository || github.repository }} + checkout_ref: ${{ inputs.checkout_ref || github.sha }} + pester-run: - needs: [context, readiness] + needs: [context, readiness, selection] if: ${{ always() && fromJSON(inputs.route_should_run || 'true') }} uses: ./.github/workflows/pester-run.yml with: - include_integration: ${{ inputs.include_integration || 'false' }} - include_patterns: ${{ inputs.include_patterns || '' }} sample_id: ${{ inputs.sample_id || '' }} context_status: ${{ needs.context.outputs.receipt_status || needs.context.result || 'unknown' }} context_artifact_name: ${{ needs.context.outputs.receipt_artifact_name || 'pester-context' }} readiness_status: ${{ needs.readiness.outputs.receipt_status || needs.readiness.result || 'unknown' }} readiness_artifact_name: ${{ needs.readiness.outputs.receipt_artifact_name || 'pester-readiness' }} + selection_status: ${{ needs.selection.outputs.receipt_status || needs.selection.result || 'unknown' }} + selection_artifact_name: ${{ needs.selection.outputs.receipt_artifact_name || 'pester-selection' }} checkout_repository: ${{ inputs.checkout_repository || github.repository }} checkout_ref: ${{ inputs.checkout_ref || github.sha }} pester-evidence: - needs: [context, readiness, pester-run] + needs: [context, readiness, selection, pester-run] if: ${{ always() && fromJSON(inputs.route_should_run || 'true') }} uses: ./.github/workflows/pester-evidence.yml with: @@ -144,4 +155,5 @@ jobs: dispatcher_exit_code: ${{ needs.pester-run.outputs.dispatcher_exit_code }} context_status: ${{ needs.context.outputs.receipt_status || needs.context.result || 'unknown' }} readiness_status: ${{ needs.readiness.outputs.receipt_status || needs.readiness.result || 'unknown' }} + selection_status: ${{ needs.selection.outputs.receipt_status || needs.selection.result || 'unknown' }} execution_job_result: ${{ needs.pester-run.outputs.execution_status || needs.pester-run.result }} diff --git a/.github/workflows/pester-run.yml b/.github/workflows/pester-run.yml index 8d1502f9c..24448737b 100644 --- a/.github/workflows/pester-run.yml +++ b/.github/workflows/pester-run.yml @@ -3,14 +3,6 @@ name: Pester run on: workflow_call: inputs: - include_integration: - required: false - type: string - default: 'false' - include_patterns: - required: false - type: string - default: '' context_status: required: false type: string @@ -30,6 +22,14 @@ on: required: false type: string default: 'pester-readiness' + selection_status: + required: false + type: string + default: 'unknown' + selection_artifact_name: + required: false + type: string + default: 'pester-selection' checkout_repository: required: false type: string @@ -51,17 +51,6 @@ on: value: ${{ jobs.finalize.outputs.execution_receipt_artifact_name }} workflow_dispatch: inputs: - include_integration: - description: "Include Integration-tagged tests in the execution pack" - required: false - default: 'false' - type: choice - options: ['false', 'true'] - include_patterns: - description: 'Optional repo-relative IncludePatterns selector' - required: false - default: '' - type: string context_status: description: 'Context receipt status' required: false @@ -87,6 +76,16 @@ on: required: false default: 'pester-readiness' type: string + selection_status: + description: 'Selection receipt status' + required: false + default: 'unknown' + type: string + selection_artifact_name: + description: 'Selection receipt artifact name' + required: false + default: 'pester-selection' + type: string checkout_repository: description: 'Repository to checkout for execution' required: false @@ -103,30 +102,10 @@ concurrency: cancel-in-progress: true jobs: - normalize: - runs-on: ubuntu-latest - outputs: - include_integration: ${{ steps.b.outputs.normalized }} - steps: - - uses: actions/checkout@v5 - with: - repository: ${{ inputs.checkout_repository || github.repository }} - ref: ${{ inputs.checkout_ref || github.sha }} - - name: Apply determinism profile - uses: ./.github/actions/determinism-profile - with: - strict: 'true' - - name: Normalize include_integration - id: b - uses: ./.github/actions/bool-normalize - with: - value: ${{ inputs.include_integration || 'false' }} - pester: name: Pester (execution only) - if: ${{ inputs.context_status == 'ready' && inputs.readiness_status == 'ready' }} + if: ${{ inputs.context_status == 'ready' && inputs.readiness_status == 'ready' && inputs.selection_status == 'ready' }} runs-on: [self-hosted, Windows, X64, comparevi, capability-ingress] - needs: normalize env: LV_SUPPRESS_UI: ${{ vars.LV_SUPPRESS_UI || '1' }} LV_NO_ACTIVATE: ${{ vars.LV_NO_ACTIVATE || '1' }} @@ -144,6 +123,9 @@ jobs: execution_receipt_status: ${{ steps.execution_receipt.outputs.status }} steps: - uses: actions/checkout@v5 + with: + repository: ${{ inputs.checkout_repository || github.repository }} + ref: ${{ inputs.checkout_ref || github.sha }} - name: Install Node dependencies shell: pwsh @@ -217,6 +199,43 @@ jobs: "generated_at_utc=$($generatedAtUtc.ToString('o'))" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 "freshness_window_seconds=$freshnessWindowSeconds" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + - name: Download selection receipt artifact + uses: actions/download-artifact@v5 + with: + name: ${{ inputs.selection_artifact_name }} + path: tests/selection + + - name: Validate selection receipt + id: selection_receipt + shell: pwsh + run: | + $receiptPath = 'tests/selection/pester-selection.json' + if (-not (Test-Path -LiteralPath $receiptPath)) { + throw "Selection receipt missing: $receiptPath" + } + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -ErrorAction Stop + if ($receipt.schema -ne 'pester-selection-receipt@v1') { + throw ("Unexpected selection receipt schema: {0}" -f $receipt.schema) + } + if ($receipt.status -ne 'ready') { + throw ("Selection receipt status is not ready: {0}" -f $receipt.status) + } + $patternsJson = @($receipt.selection.includePatterns) | ConvertTo-Json -Compress + if (-not $patternsJson) { $patternsJson = '[]' } + "path=$receiptPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "integration_mode=$($receipt.selection.integrationMode)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "fixture_required=$(([string]$receipt.selection.fixtureRequired).ToLowerInvariant())" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "include_patterns_json=$patternsJson" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "timeout_seconds=$($receipt.dispatcherProfile.timeoutSeconds)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "emit_failures_json_always=$($receipt.dispatcherProfile.emitFailuresJsonAlways)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "detect_leaks=$($receipt.dispatcherProfile.detectLeaks)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "fail_on_leaks=$($receipt.dispatcherProfile.failOnLeaks)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "kill_leaks=$($receipt.dispatcherProfile.killLeaks)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "leak_grace_seconds=$($receipt.dispatcherProfile.leakGraceSeconds)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "clean_labview_before=$($receipt.dispatcherProfile.cleanLabVIEWBefore)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "clean_after=$($receipt.dispatcherProfile.cleanAfter)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "track_artifacts=$($receipt.dispatcherProfile.trackArtifacts)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + - name: Acquire session lock shell: pwsh run: pwsh -NoLogo -NoProfile -File tools/Session-Lock.ps1 -Action Acquire -Group 'pester-selfhosted' -QueueWaitSeconds 15 -QueueMaxAttempts 40 -StaleSeconds 300 -HeartbeatSeconds 15 @@ -229,12 +248,12 @@ jobs: process-names: 'LVCompare,LabVIEW' - name: Prepare fixture copies (base/head) - if: ${{ needs.normalize.outputs.include_integration == 'true' }} + if: ${{ steps.selection_receipt.outputs.fixture_required == 'true' }} id: fixtures uses: ./.github/actions/prepare-fixtures - name: Export fixture env for tests - if: ${{ needs.normalize.outputs.include_integration == 'true' }} + if: ${{ steps.selection_receipt.outputs.fixture_required == 'true' }} shell: pwsh run: | if ('${{ steps.fixtures.outputs.base }}' -and '${{ steps.fixtures.outputs.head }}') { @@ -248,15 +267,15 @@ jobs: id: dprofile uses: ./.github/actions/dispatcher-profile with: - timeout-seconds: '0' - emit-failures-json-always: 'true' - detect-leaks: 'true' - fail-on-leaks: 'false' - kill-leaks: 'false' - leak-grace-seconds: '3' - clean-labview-before: 'false' - clean-after: 'false' - track-artifacts: 'true' + timeout-seconds: ${{ steps.selection_receipt.outputs.timeout_seconds }} + emit-failures-json-always: ${{ steps.selection_receipt.outputs.emit_failures_json_always }} + detect-leaks: ${{ steps.selection_receipt.outputs.detect_leaks }} + fail-on-leaks: ${{ steps.selection_receipt.outputs.fail_on_leaks }} + kill-leaks: ${{ steps.selection_receipt.outputs.kill_leaks }} + leak-grace-seconds: ${{ steps.selection_receipt.outputs.leak_grace_seconds }} + clean-labview-before: ${{ steps.selection_receipt.outputs.clean_labview_before }} + clean-after: ${{ steps.selection_receipt.outputs.clean_after }} + track-artifacts: ${{ steps.selection_receipt.outputs.track_artifacts }} - name: Wire Probe (T1) if: ${{ vars.WIRE_PROBES != '0' }} @@ -280,19 +299,16 @@ jobs: } $bound = [ordered]@{} $bound.TestsPath = 'tests' - $includeIntegration = '${{ needs.normalize.outputs.include_integration }}' - if ($includeIntegration) { - switch ($includeIntegration.ToString().ToLowerInvariant()) { - { $_ -in @('true','1','yes','y','on','include') } { $bound.IntegrationMode = 'include' } - { $_ -in @('false','0','no','n','off','exclude') } { $bound.IntegrationMode = 'exclude' } - 'auto' { $bound.IntegrationMode = 'auto' } - } + $integrationMode = '${{ steps.selection_receipt.outputs.integration_mode }}' + if ($integrationMode) { + $bound.IntegrationMode = $integrationMode } $bound.ResultsPath = 'tests/results' if ($env:DISPATCHER_LIVE_OUTPUT -ne '0') { $bound.LiveOutput = $true } if ('${{ steps.dprofile.outputs.emit_failures_json_always }}' -eq 'true') { $bound.EmitFailuresJsonAlways = $true } - if ('${{ inputs.include_patterns }}' -ne '') { - $bound.IncludePatterns = (Split-Path -Leaf '${{ inputs.include_patterns }}') + $patterns = '${{ steps.selection_receipt.outputs.include_patterns_json }}' | ConvertFrom-Json -ErrorAction Stop + if ($patterns -and @($patterns).Count -gt 0) { + $bound.IncludePatterns = @($patterns) } $timeoutSeconds = '${{ steps.dprofile.outputs.timeout_seconds }}' if ($timeoutSeconds -and $timeoutSeconds -match '^-?\d+(\.\d+)?$') { @@ -351,6 +367,7 @@ jobs: } $contextReceiptPresent = Test-Path -LiteralPath '${{ steps.context_receipt.outputs.path }}' $readinessReceiptPresent = Test-Path -LiteralPath '${{ steps.readiness_receipt.outputs.path }}' + $selectionReceiptPresent = Test-Path -LiteralPath '${{ steps.selection_receipt.outputs.path }}' if ((Test-Path -LiteralPath $summaryPath) -and $dispatcherExitCode -eq '0') { $status = 'completed' } elseif (Test-Path -LiteralPath $summaryPath) { @@ -369,6 +386,11 @@ jobs: readinessReceiptPresent = $readinessReceiptPresent readinessReceiptGeneratedAtUtc = '${{ steps.readiness_receipt.outputs.generated_at_utc }}' readinessReceiptFreshnessWindowSeconds = '${{ steps.readiness_receipt.outputs.freshness_window_seconds }}' + selectionStatus = '${{ inputs.selection_status }}' + selectionReceiptPath = '${{ steps.selection_receipt.outputs.path }}' + selectionReceiptPresent = $selectionReceiptPresent + selectionIntegrationMode = '${{ steps.selection_receipt.outputs.integration_mode }}' + selectionFixtureRequired = '${{ steps.selection_receipt.outputs.fixture_required }}' dispatcherExitCode = [int]$dispatcherExitCode summaryPresent = Test-Path -LiteralPath $summaryPath status = $status @@ -390,7 +412,7 @@ jobs: finalize: runs-on: ubuntu-latest - needs: [normalize, pester] + needs: [pester] if: always() outputs: dispatcher_exit_code: ${{ steps.emit.outputs.dispatcher_exit_code }} @@ -412,7 +434,10 @@ jobs: $receiptStatus = 'context-blocked' } elseif ('${{ inputs.readiness_status }}' -ne 'ready') { $executionStatus = 'skipped' - if ([string]::IsNullOrWhiteSpace($receiptStatus)) { $receiptStatus = 'skipped' } + if ([string]::IsNullOrWhiteSpace($receiptStatus)) { $receiptStatus = 'readiness-blocked' } + } elseif ('${{ inputs.selection_status }}' -ne 'ready') { + $executionStatus = 'skipped' + if ([string]::IsNullOrWhiteSpace($receiptStatus)) { $receiptStatus = 'selection-blocked' } } else { switch ('${{ needs.pester.result }}') { 'success' { @@ -452,6 +477,7 @@ jobs: generatedAtUtc = [DateTime]::UtcNow.ToString('o') contextStatus = '${{ inputs.context_status }}' readinessStatus = '${{ inputs.readiness_status }}' + selectionStatus = '${{ inputs.selection_status }}' executionJobResult = '${{ needs.pester.result }}' dispatcherExitCode = [int]$dispatcherExitCode status = $receiptStatus diff --git a/.github/workflows/pester-selection.yml b/.github/workflows/pester-selection.yml new file mode 100644 index 000000000..6d721d526 --- /dev/null +++ b/.github/workflows/pester-selection.yml @@ -0,0 +1,159 @@ +name: Pester selection + +on: + workflow_call: + inputs: + include_integration: + required: false + type: string + default: 'false' + include_patterns: + required: false + type: string + default: '' + sample_id: + required: false + type: string + checkout_repository: + required: false + type: string + checkout_ref: + required: false + type: string + outputs: + receipt_status: + description: 'Overall selection status for pack shaping and dispatcher profile resolution' + value: ${{ jobs.selection.outputs.receipt_status }} + receipt_artifact_name: + description: 'Artifact name containing the selection receipt bundle' + value: ${{ jobs.selection.outputs.receipt_artifact_name }} + workflow_dispatch: + inputs: + include_integration: + description: "Include Integration-tagged tests in the execution pack" + required: false + default: 'false' + type: choice + options: ['false', 'true'] + include_patterns: + description: 'Optional repo-relative IncludePatterns selector' + required: false + default: '' + type: string + sample_id: + description: 'Sampling correlation id (prevents cancels)' + required: false + default: '' + type: string + checkout_repository: + description: 'Repository to checkout for selection resolution' + required: false + default: '' + type: string + checkout_ref: + description: 'Git ref or SHA to checkout for selection resolution' + required: false + default: '' + type: string + +concurrency: + group: ${{ github.workflow }}-${{ github.event.inputs.sample_id || inputs.sample_id || github.ref }} + cancel-in-progress: true + +jobs: + selection: + runs-on: ubuntu-latest + outputs: + receipt_status: ${{ steps.receipt.outputs.status }} + receipt_artifact_name: ${{ steps.receipt.outputs.artifact_name }} + steps: + - uses: actions/checkout@v5 + with: + repository: ${{ inputs.checkout_repository || github.repository }} + ref: ${{ inputs.checkout_ref || github.sha }} + + - name: Normalize include_integration + id: include_integration + uses: ./.github/actions/bool-normalize + with: + value: ${{ inputs.include_integration || 'false' }} + + - name: Shape include patterns + id: include_patterns + shell: pwsh + env: + RAW_INCLUDE_PATTERNS: ${{ inputs.include_patterns || '' }} + run: | + $raw = $env:RAW_INCLUDE_PATTERNS + $tokens = New-Object System.Collections.Generic.List[string] + foreach ($candidate in ($raw -split "[`r`n,;]")) { + $token = $candidate.Trim() + if ([string]::IsNullOrWhiteSpace($token)) { continue } + $leaf = Split-Path -Leaf $token + if ([string]::IsNullOrWhiteSpace($leaf)) { continue } + if (-not $tokens.Contains($leaf)) { + $tokens.Add($leaf) | Out-Null + } + } + $patternsJson = @($tokens.ToArray()) | ConvertTo-Json -Compress + if (-not $patternsJson) { $patternsJson = '[]' } + "patterns_json=$patternsJson" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + + - name: Resolve dispatcher profile + id: dispatcher_profile + uses: ./.github/actions/dispatcher-profile + with: + timeout-seconds: '0' + emit-failures-json-always: 'true' + detect-leaks: 'true' + fail-on-leaks: 'false' + kill-leaks: 'false' + leak-grace-seconds: '3' + clean-labview-before: 'false' + clean-after: 'false' + track-artifacts: 'true' + + - name: Write selection receipt + id: receipt + shell: pwsh + run: | + $outDir = 'tests/results/pester-selection' + New-Item -ItemType Directory -Force -Path $outDir | Out-Null + $includeIntegration = '${{ steps.include_integration.outputs.normalized }}' + $integrationMode = if ($includeIntegration -eq 'true') { 'include' } else { 'exclude' } + $includePatterns = '${{ steps.include_patterns.outputs.patterns_json }}' | ConvertFrom-Json -ErrorAction Stop + $receipt = [ordered]@{ + schema = 'pester-selection-receipt@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + status = 'ready' + sampleId = '${{ inputs.sample_id || github.event.inputs.sample_id || '' }}' + selection = [ordered]@{ + includeIntegrationNormalized = $includeIntegration + integrationMode = $integrationMode + includePatterns = @($includePatterns) + fixtureRequired = ($includeIntegration -eq 'true') + } + dispatcherProfile = [ordered]@{ + timeoutSeconds = '${{ steps.dispatcher_profile.outputs.timeout_seconds }}' + emitFailuresJsonAlways = '${{ steps.dispatcher_profile.outputs.emit_failures_json_always }}' + detectLeaks = '${{ steps.dispatcher_profile.outputs.detect_leaks }}' + failOnLeaks = '${{ steps.dispatcher_profile.outputs.fail_on_leaks }}' + killLeaks = '${{ steps.dispatcher_profile.outputs.kill_leaks }}' + leakGraceSeconds = '${{ steps.dispatcher_profile.outputs.leak_grace_seconds }}' + cleanLabVIEWBefore = '${{ steps.dispatcher_profile.outputs.clean_labview_before }}' + cleanAfter = '${{ steps.dispatcher_profile.outputs.clean_after }}' + trackArtifacts = '${{ steps.dispatcher_profile.outputs.track_artifacts }}' + } + } + $receiptPath = Join-Path $outDir 'pester-selection.json' + $receipt | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $receiptPath -Encoding UTF8 + "status=ready" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "artifact_name=pester-selection" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + + - name: Upload selection receipt + if: always() + uses: actions/upload-artifact@v7 + with: + name: pester-selection + path: tests/results/pester-selection + if-no-files-found: error diff --git a/docs/architecture/pester-service-model-control-plane.md b/docs/architecture/pester-service-model-control-plane.md index 655dcd8fb..f22cfd40f 100644 --- a/docs/architecture/pester-service-model-control-plane.md +++ b/docs/architecture/pester-service-model-control-plane.md @@ -7,7 +7,7 @@ Replace the monolithic self-hosted Pester transaction with explicit control layers that can be proven, audited, and promoted intentionally. - Scope: - Trusted routing, context, readiness, execution, evidence, and the additive + Trusted routing, context, selection, readiness, execution, evidence, and the additive promotion boundary. ## Stakeholders And Concerns @@ -36,6 +36,7 @@ | --- | --- | --- | | Trusted router | Decide whether the pilot is allowed to run and which ref it should use | GitHub Actions YAML | | Context layer | Certify repository slug and standing-priority control-plane assumptions | GitHub Actions + Node | +| Selection layer | Resolve the declared pack and dispatcher profile into a receipt | GitHub Actions + PowerShell | | Readiness layer | Certify self-hosted ingress runtime state and host dependencies | GitHub Actions + PowerShell | | Execution layer | Run the selected Pester pack after validating upstream receipts | GitHub Actions + PowerShell | | Evidence layer | Classify results, summarize them, and publish operator artifacts | GitHub Actions + PowerShell | @@ -46,6 +47,7 @@ | --- | --- | --- | | `pester-service-model-on-label.yml` | Trusted router | Admission control for dispatch and same-owner labeled PRs | | `pester-context.yml` | Context layer | Repository and standing-priority receipt | +| `pester-selection.yml` | Selection layer | Integration mode, include pattern, and dispatcher profile receipt | | `selfhosted-readiness.yml` | Readiness layer | Runner labels, session lock, `.NET`, Docker, and LVCompare readiness | | `pester-run.yml` | Execution layer | Receipt validation, dispatcher invocation, execution contract | | `pester-evidence.yml` | Evidence layer | Classification, summary, session index, and dashboard publication | @@ -68,9 +70,10 @@ `REQ-PSM-001` maps to the trusted router. `REQ-PSM-002` maps to context. `REQ-PSM-003` maps to readiness. - `REQ-PSM-004` maps to execution. - `REQ-PSM-005` maps to evidence. - `REQ-PSM-006` maps to the additive promotion boundary. + `REQ-PSM-004` maps to selection. + `REQ-PSM-005` maps to execution. + `REQ-PSM-006` maps to evidence. + `REQ-PSM-007` maps to the additive promotion boundary. - Decision rationale: The service model exists to separate concerns and make failures classifiable by layer instead of inferred from one coupled self-hosted run. diff --git a/docs/knowledgebase/Pester-Service-Model.md b/docs/knowledgebase/Pester-Service-Model.md index f2d386e80..0b5570354 100644 --- a/docs/knowledgebase/Pester-Service-Model.md +++ b/docs/knowledgebase/Pester-Service-Model.md @@ -11,7 +11,7 @@ That coupling is what makes the monolithic self-hosted seam expensive to reprodu ## Pilot Split -The additive pilot introduces five workflow surfaces: +The additive pilot introduces seven workflow surfaces: - `.github/workflows/pester-context.yml` - repo/control-plane receipts for repository slug, token-backed standing-priority sync, and context classification @@ -21,6 +21,8 @@ The additive pilot introduces five workflow surfaces: - trusted PR/dispatch entrypoint for proving the pilot without exposing the self-hosted ingress plane to untrusted fork heads - `.github/workflows/selfhosted-readiness.yml` - host-plane readiness receipts for the self-hosted ingress surface +- `.github/workflows/pester-selection.yml` + - receipt-driven pack shaping for integration mode, include patterns, and dispatcher profile resolution - `.github/workflows/pester-run.yml` - receipt-driven Pester execution only - `.github/workflows/pester-evidence.yml` @@ -29,11 +31,12 @@ The additive pilot introduces five workflow surfaces: ## Design Rules - Context certifies repo/control-plane assumptions. It does not probe host readiness or execute tests. +- Selection resolves integration mode, include patterns, and dispatcher profile into a receipt before execution begins. +- Selection consumes context. It does not probe host readiness or invoke the dispatcher. - Readiness certifies the environment. It does not execute the test pack. - Readiness consumes context. It does not discover standing-priority state itself. -- Selection is still internal to execution in the current pilot baseline; pulling it into its own receipt is the next planned decomposition slice. - Readiness emits a bounded-freshness receipt artifact that execution must download and validate before dispatch. -- Execution consumes context and readiness. It does not bootstrap Docker runtimes, install core toolchains, or discover standing-priority state. +- Execution consumes context, selection, and readiness. It does not normalize pack inputs, choose the dispatcher profile, bootstrap Docker runtimes, install core toolchains, or discover standing-priority state. - Execution writes an execution receipt before uploading raw artifacts so evidence can classify the real seam outcome. - Execution must also emit a skip-safe execution contract from an always-on finalize path so reusable-workflow outputs do not collapse when the execution job never starts. - Evidence consumes raw execution output plus the execution receipt. It classifies `context-blocked`, `readiness-blocked`, and `seam-defect` explicitly instead of collapsing them into one execution symptom. @@ -45,6 +48,7 @@ The additive pilot introduces five workflow surfaces: The pilot can replace the monolith only after: - readiness receipts are stable on the ingress host +- selection receipts resolve the declared pack and dispatcher profile without execution-side drift - execution runs the declared pack without host bootstrap - evidence produces deterministic classifications - PR/release comparisons show better failure localization and lower operator ambiguity diff --git a/docs/pester-service-model-quality-report.md b/docs/pester-service-model-quality-report.md index 3bc737214..3a75013d7 100644 --- a/docs/pester-service-model-quality-report.md +++ b/docs/pester-service-model-quality-report.md @@ -3,7 +3,7 @@ ## Scope This report covers the layered Pester service-model control plane defined by the -trusted router, context, readiness, execution, and evidence workflows. +trusted router, context, selection, readiness, execution, and evidence workflows. ## Current Evidence @@ -28,6 +28,7 @@ trusted router, context, readiness, execution, and evidence workflows. - The subsystem now has explicit requirements and traceability. - The upstream slice now has dedicated hosted packet-quality and release-evidence gates. +- Selection is now a first-class packet layer instead of execution-internal shaping. - Coverage and docs integrity now have dedicated packet-level gates. - Promotion remains blocked until additive proof against the monolith is intentionally accepted and the retained evidence bundle is used to justify the diff --git a/docs/requirements-pester-service-model-srs.md b/docs/requirements-pester-service-model-srs.md index d8d7706ab..79653f176 100644 --- a/docs/requirements-pester-service-model-srs.md +++ b/docs/requirements-pester-service-model-srs.md @@ -12,10 +12,10 @@ - Purpose: Specify the trusted Pester control plane that separates context, host - readiness, execution, and evidence into auditable workflow surfaces. + readiness, selection, execution, and evidence into auditable workflow surfaces. - In scope: Trusted pilot routing, repo/control-plane context certification, self-hosted - readiness receipts, execution-only dispatcher runs, and evidence + readiness receipts, selection receipts, execution-only dispatcher runs, and evidence classification. - Out of scope: Legacy monolithic `test-pester.yml` behavior except where it remains the @@ -37,9 +37,10 @@ | REQ-PSM-001 | The trusted Pester pilot shall admit `workflow_dispatch` and same-owner PR heads carrying the `pester-service-model` label, and shall reject untrusted cross-owner fork heads before self-hosted execution begins. | Trust and admission are part of the subsystem boundary, not incidental workflow behavior. | `pester-service-model-on-label.yml` writes `should_run=true` only for dispatch or trusted same-owner heads, and emits `untrusted-cross-owner-fork` for disallowed heads. | `TEST-PSM-001` | | REQ-PSM-002 | The context layer shall resolve repository and standing-priority state and emit a context receipt before readiness begins. | Repo/control-plane assumptions must be certified separately from host readiness. | `pester-context.yml` uploads `pester-context.json` with `schema=pester-context-receipt@v1` and a status of `ready`, `warning`, or `blocked`. | `TEST-PSM-002` | | REQ-PSM-003 | The readiness layer shall certify runner labels, session-lock health, `.NET`, Windows Docker runtime, and LVCompare or idle LabVIEW state, and shall emit a bounded-freshness readiness receipt. | Self-hosted ingress debt must be observable as readiness debt, not hidden inside execution. | `selfhosted-readiness.yml` uploads `selfhosted-readiness.json` with `schema=pester-selfhosted-readiness-receipt@v1`, individual probe outcomes, and `freshnessWindowSeconds=900`. | `TEST-PSM-003` | -| REQ-PSM-004 | The execution layer shall validate context and readiness receipts before dispatch, run the declared Pester pack without bootstrapping Docker runtimes or core toolchains, and shall emit an execution receipt even when execution is skipped. | Execution should only execute; it must not absorb context or readiness responsibilities. | `pester-run.yml` refuses to start unless both receipts are ready, calls `Invoke-PesterTests.ps1`, uploads raw results when produced, and always uploads `pester-run-receipt.json`. | `TEST-PSM-004` | -| REQ-PSM-005 | The evidence layer shall classify `context-blocked`, `readiness-blocked`, `test-failures`, and `seam-defect` explicitly from execution receipts and raw artifacts. | Operators need precise failure classes instead of `missing-summary` ambiguity. | `pester-evidence.yml` reads the execution contract and emits the explicit classification when raw artifacts are missing or execution is skipped. | `TEST-PSM-005` | -| REQ-PSM-006 | The pilot shall remain additive until it proves equivalent or better behavior than the monolithic required gate. | Promotion must follow evidence, not preference. | The service-model knowledgebase and promotion rule state that the legacy required gate remains in place until the pilot is proven. | `TEST-PSM-006` | +| REQ-PSM-004 | The selection layer shall resolve integration mode, include patterns, and dispatcher profile into a selection receipt before self-hosted execution begins. | Pack shaping and dispatcher defaults are control-plane decisions, not execution-side improvisation. | `pester-selection.yml` uploads `pester-selection.json` with `schema=pester-selection-receipt@v1`, the normalized pack selector, and dispatcher profile values. | `TEST-PSM-004` | +| REQ-PSM-005 | The execution layer shall validate context, readiness, and selection receipts before dispatch, run the declared Pester pack without bootstrapping Docker runtimes or core toolchains, and shall emit an execution receipt even when execution is skipped. | Execution should only execute; it must not absorb context, selection, or readiness responsibilities. | `pester-run.yml` refuses to start unless all upstream receipts are ready, calls `Invoke-PesterTests.ps1`, uploads raw results when produced, and always uploads `pester-run-receipt.json`. | `TEST-PSM-005` | +| REQ-PSM-006 | The evidence layer shall classify `context-blocked`, `selection-blocked`, `readiness-blocked`, `test-failures`, and `seam-defect` explicitly from execution receipts and raw artifacts. | Operators need precise failure classes instead of `missing-summary` ambiguity. | `pester-evidence.yml` reads the execution contract and emits the explicit classification when raw artifacts are missing or execution is skipped. | `TEST-PSM-006` | +| REQ-PSM-007 | The pilot shall remain additive until it proves equivalent or better behavior than the monolithic required gate. | Promotion must follow evidence, not preference. | The service-model knowledgebase and promotion rule state that the legacy required gate remains in place until the pilot is proven. | `TEST-PSM-007` | ## Assumptions diff --git a/docs/rtm-pester-service-model.csv b/docs/rtm-pester-service-model.csv index 6f350d92f..e86cbd068 100644 --- a/docs/rtm-pester-service-model.csv +++ b/docs/rtm-pester-service-model.csv @@ -2,6 +2,7 @@ ReqID,Requirement,Source,Priority,TestID,TestArtifact,CodeRef,Status REQ-PSM-001,"Trusted router admits workflow_dispatch and same-owner labeled PR heads; rejects cross-owner heads","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-001,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs",".github/workflows/pester-service-model-on-label.yml",Implemented REQ-PSM-002,"Context resolves repository and standing-priority state before readiness and emits a receipt","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-002,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs",".github/workflows/pester-context.yml;tools/priority/run-sync-standing-priority.mjs",Implemented REQ-PSM-003,"Readiness certifies runner labels, session lock, dotnet, Docker runtime, and LVCompare state and emits a bounded-freshness receipt","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-003,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs",".github/workflows/selfhosted-readiness.yml;tools/Invoke-DockerRuntimeManager.ps1;tools/Session-Lock.ps1",Implemented -REQ-PSM-004,"Execution validates context and readiness receipts, runs Invoke-PesterTests without host bootstrap, and emits an execution receipt even when skipped","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-004,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs",".github/workflows/pester-run.yml;Invoke-PesterTests.ps1",Implemented -REQ-PSM-005,"Evidence classifies context-blocked, readiness-blocked, test-failures, and seam-defect explicitly from the execution contract","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-005,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs",".github/workflows/pester-evidence.yml",Implemented -REQ-PSM-006,"The pilot remains additive until it proves equivalent or better than the monolith","docs/requirements-pester-service-model-srs.md",Medium,TEST-PSM-006,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs","docs/knowledgebase/Pester-Service-Model.md;.github/workflows/pester-gate.yml",Implemented +REQ-PSM-004,"Selection resolves integration mode, include patterns, and dispatcher profile into a receipt before execution","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-004,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs",".github/workflows/pester-selection.yml;.github/actions/dispatcher-profile/action.yml",Implemented +REQ-PSM-005,"Execution validates context, readiness, and selection receipts, runs Invoke-PesterTests without host bootstrap, and emits an execution receipt even when skipped","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-005,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs",".github/workflows/pester-run.yml;Invoke-PesterTests.ps1",Implemented +REQ-PSM-006,"Evidence classifies context-blocked, selection-blocked, readiness-blocked, test-failures, and seam-defect explicitly from the execution contract","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-006,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs",".github/workflows/pester-evidence.yml",Implemented +REQ-PSM-007,"The pilot remains additive until it proves equivalent or better than the monolith","docs/requirements-pester-service-model-srs.md",Medium,TEST-PSM-007,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs","docs/knowledgebase/Pester-Service-Model.md;.github/workflows/pester-gate.yml",Implemented diff --git a/docs/testing/pester-service-model-test-plan.md b/docs/testing/pester-service-model-test-plan.md index a463755c2..7f5c7fda3 100644 --- a/docs/testing/pester-service-model-test-plan.md +++ b/docs/testing/pester-service-model-test-plan.md @@ -7,8 +7,9 @@ - Owner: `#2069` with retained fork basis on `#2078` - Scope: - Trusted routing, context receipts, readiness receipts, execution-only - behavior, and evidence classification for the Pester service model + Trusted routing, context receipts, selection receipts, readiness receipts, + execution-only behavior, and evidence classification for the Pester service + model ## Test Items @@ -17,6 +18,7 @@ | `pester-service-model-workflow-contract.test.mjs` | Integration | High | Verifies the workflow split and core receipt/evidence obligations | | `pester-service-model-quality-workflow-contract.test.mjs` | Integration | Medium | Verifies the coverage gate and docs link-check control-plane workflow | | `pester-gate.yml` + trusted pilot routing | Workflow | High | Verifies admission and orchestration across layers | +| `pester-selection.yml` selection contract | Workflow | High | Verifies pack shaping and dispatcher-profile resolution leave execution clean | | `Invoke-PesterTests.ps1` execution contract | Execution | High | Verifies the dispatcher remains the execution engine only | ## Entry Criteria @@ -36,7 +38,7 @@ | Metric | Target | Evidence | | --- | --- | --- | | Workflow contract coverage | All layer responsibilities represented | `tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs` | -| Receipt coverage | Context, readiness, execution, and evidence all emit auditable artifacts | assurance report + integration runs | +| Receipt coverage | Context, selection, readiness, execution, and evidence all emit auditable artifacts | assurance report + integration runs | | Classification coverage | blocked and defect outcomes remain distinguishable | evidence workflow outputs | | Packet coverage gate | Retained `coverage.xml` and named PR coverage gate | `.github/workflows/pester-service-model-quality.yml` | | Promotion bundle retention | Hosted bundle retains the minimal promotion handoff | `.github/workflows/pester-service-model-release-evidence.yml` | diff --git a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs index 4a84abfa6..81e07f807 100644 --- a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs +++ b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs @@ -11,7 +11,7 @@ function readRepoFile(relativePath) { return readFileSync(path.join(repoRoot, relativePath), 'utf8'); } -test('pester gate pilot routes context, readiness, execution, and evidence through separate reusable workflows', () => { +test('pester gate pilot routes context, selection, readiness, execution, and evidence through separate reusable workflows', () => { const workflow = readRepoFile('.github/workflows/pester-gate.yml'); assert.match(workflow, /name:\s+Pester gate \(service model pilot\)/); @@ -24,15 +24,19 @@ test('pester gate pilot routes context, readiness, execution, and evidence throu assert.match(workflow, /jobs:\s*\n\s*skipped:\s*\n\s+if:\s+\$\{\{\s*!fromJSON\(inputs\.route_should_run \|\| 'true'\)\s*\}\}/); assert.match(workflow, /\n\s*context:\s*\n\s+if:\s+\$\{\{\s*fromJSON\(inputs\.route_should_run \|\| 'true'\)\s*\}\}\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/pester-context\.yml/); assert.match(workflow, /\n\s*readiness:\s*\n\s+needs:\s+context\s*\n\s+if:\s+\$\{\{\s*always\(\) && fromJSON\(inputs\.route_should_run \|\| 'true'\) && needs\.context\.outputs\.receipt_status == 'ready'\s*\}\}\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/selfhosted-readiness\.yml/); - assert.match(workflow, /\n\s*pester-run:\s*\n\s+needs:\s+\[context, readiness\]\s*\n\s+if:\s+\$\{\{\s*always\(\) && fromJSON\(inputs\.route_should_run \|\| 'true'\)\s*\}\}\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/pester-run\.yml/); + assert.match(workflow, /\n\s*selection:\s*\n\s+needs:\s+context\s*\n\s+if:\s+\$\{\{\s*always\(\) && fromJSON\(inputs\.route_should_run \|\| 'true'\) && needs\.context\.outputs\.receipt_status == 'ready'\s*\}\}\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/pester-selection\.yml/); + assert.match(workflow, /\n\s*pester-run:\s*\n\s+needs:\s+\[context, readiness, selection\]\s*\n\s+if:\s+\$\{\{\s*always\(\) && fromJSON\(inputs\.route_should_run \|\| 'true'\)\s*\}\}\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/pester-run\.yml/); assert.match(workflow, /context_status:\s+\$\{\{\s*needs\.context\.outputs\.receipt_status\s*\|\|\s*needs\.context\.result/); assert.match(workflow, /context_artifact_name:\s+\$\{\{\s*needs\.context\.outputs\.receipt_artifact_name\s*\|\|\s*'pester-context'/); assert.match(workflow, /readiness_artifact_name:\s+\$\{\{\s*needs\.readiness\.outputs\.receipt_artifact_name\s*\|\|\s*'pester-readiness'/); assert.match(workflow, /readiness_status:\s+\$\{\{\s*needs\.readiness\.outputs\.receipt_status\s*\|\|\s*needs\.readiness\.result/); + assert.match(workflow, /selection_artifact_name:\s+\$\{\{\s*needs\.selection\.outputs\.receipt_artifact_name\s*\|\|\s*'pester-selection'/); + assert.match(workflow, /selection_status:\s+\$\{\{\s*needs\.selection\.outputs\.receipt_status\s*\|\|\s*needs\.selection\.result/); assert.match(workflow, /checkout_repository:\s+\$\{\{\s*inputs\.checkout_repository \|\| github\.repository\s*\}\}/); assert.match(workflow, /checkout_ref:\s+\$\{\{\s*inputs\.checkout_ref \|\| github\.sha\s*\}\}/); - assert.match(workflow, /\n\s*pester-evidence:\s*\n\s+needs:\s+\[context, readiness, pester-run\]\s*\n\s+if:\s+\$\{\{\s*always\(\) && fromJSON\(inputs\.route_should_run \|\| 'true'\)\s*\}\}\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/pester-evidence\.yml/); + assert.match(workflow, /\n\s*pester-evidence:\s*\n\s+needs:\s+\[context, readiness, selection, pester-run\]\s*\n\s+if:\s+\$\{\{\s*always\(\) && fromJSON\(inputs\.route_should_run \|\| 'true'\)\s*\}\}\s*\n\s+uses:\s+\.\s*\/\.github\/workflows\/pester-evidence\.yml/); assert.match(workflow, /context_status:\s+\$\{\{\s*needs\.context\.outputs\.receipt_status\s*\|\|\s*needs\.context\.result/); + assert.match(workflow, /selection_status:\s+\$\{\{\s*needs\.selection\.outputs\.receipt_status\s*\|\|\s*needs\.selection\.result/); assert.match(workflow, /execution_job_result:\s+\$\{\{\s*needs\.pester-run\.outputs\.execution_status\s*\|\|\s*needs\.pester-run\.result/); assert.match(workflow, /execution_receipt_artifact_name:\s+\$\{\{\s*needs\.pester-run\.outputs\.execution_receipt_artifact_name/); assert.match(workflow, /### Pester gate \(service model pilot\)/); @@ -72,12 +76,28 @@ test('selfhosted readiness owns host-plane certification and emits a receipt art assert.match(workflow, /freshnessWindowSeconds = 900/); }); -test('pester run is execution-only and validates context plus readiness receipts before dispatch', () => { +test('pester selection owns pack shaping and dispatcher profile resolution before execution begins', () => { + const workflow = readRepoFile('.github/workflows/pester-selection.yml'); + + assert.match(workflow, /name:\s+Pester selection/); + assert.match(workflow, /workflow_call:/); + assert.match(workflow, /receipt_status:/); + assert.match(workflow, /receipt_artifact_name:/); + assert.match(workflow, /Normalize include_integration/); + assert.match(workflow, /Shape include patterns/); + assert.match(workflow, /Resolve dispatcher profile/); + assert.match(workflow, /pester-selection-receipt@v1/); + assert.match(workflow, /integrationMode/); + assert.match(workflow, /fixtureRequired/); + assert.match(workflow, /Upload selection receipt/); +}); + +test('pester run is execution-only and validates context, readiness, and selection receipts before dispatch', () => { const workflow = readRepoFile('.github/workflows/pester-run.yml'); assert.match(workflow, /name:\s+Pester run/); assert.match(workflow, /name:\s+Pester \(execution only\)/); - assert.match(workflow, /if:\s+\$\{\{\s*inputs\.context_status == 'ready' && inputs\.readiness_status == 'ready'\s*\}\}/); + assert.match(workflow, /if:\s+\$\{\{\s*inputs\.context_status == 'ready' && inputs\.readiness_status == 'ready' && inputs\.selection_status == 'ready'\s*\}\}/); assert.match(workflow, /execution_status:/); assert.match(workflow, /execution_receipt_artifact_name:/); assert.match(workflow, /repository:\s+\$\{\{\s*inputs\.checkout_repository \|\| github\.repository\s*\}\}/); @@ -88,18 +108,24 @@ test('pester run is execution-only and validates context plus readiness receipts assert.match(workflow, /Download readiness receipt artifact/); assert.match(workflow, /Validate readiness receipt/); assert.match(workflow, /selfhosted-readiness\.json/); + assert.match(workflow, /Download selection receipt artifact/); + assert.match(workflow, /Validate selection receipt/); + assert.match(workflow, /pester-selection\.json/); + assert.match(workflow, /selection-blocked/); assert.match(workflow, /Run Pester tests via local dispatcher/); assert.match(workflow, /pester-run-receipt\.json/); assert.match(workflow, /Upload raw Pester execution artifact/); assert.match(workflow, /Emit execution contract/); assert.match(workflow, /Upload execution contract artifact/); + assert.doesNotMatch(workflow, /Normalize include_integration/); + assert.doesNotMatch(workflow, /needs\.normalize/); assert.doesNotMatch(workflow, /Install Pester/); assert.doesNotMatch(workflow, /Invoke-DockerRuntimeManager\.ps1/); assert.doesNotMatch(workflow, /Write-PesterSummaryToStepSummary\.ps1/); assert.doesNotMatch(workflow, /Invoke-DevDashboard\.ps1/); }); -test('pester evidence distinguishes context-blocked and readiness-blocked skips from seam defects', () => { +test('pester evidence distinguishes context-blocked, selection-blocked, and readiness-blocked skips from seam defects', () => { const workflow = readRepoFile('.github/workflows/pester-evidence.yml'); assert.match(workflow, /name:\s+Pester evidence/); @@ -114,8 +140,11 @@ test('pester evidence distinguishes context-blocked and readiness-blocked skips assert.match(workflow, /Invoke-DevDashboard\.ps1/); assert.match(workflow, /classification = 'seam-defect'/); assert.match(workflow, /\$classification = 'context-blocked'/); + assert.match(workflow, /\$classification = 'selection-blocked'/); assert.match(workflow, /\$classification = 'readiness-blocked'/); assert.match(workflow, /\$contextStatus -ne 'ready'/); + assert.match(workflow, /\$selectionStatus -ne 'ready'/); + assert.match(workflow, /\$executionReceiptStatus -eq 'selection-blocked'/); assert.match(workflow, /\$executionReceiptStatus -eq 'context-blocked'/); assert.match(workflow, /\$readinessStatus -ne 'ready' -and \$executionJobResult -in @\('skipped','cancelled'\)/); assert.match(workflow, /raw-artifact-download=/); @@ -129,10 +158,12 @@ test('knowledgebase documents the additive service model and keeps the monolith assert.match(doc, /legacy Pester control plane couples four concerns into one self-hosted transaction/i); assert.match(doc, /pester-context\.yml/); + assert.match(doc, /pester-selection\.yml/); assert.match(doc, /selfhosted-readiness\.yml/); assert.match(doc, /pester-run\.yml/); assert.match(doc, /pester-evidence\.yml/); assert.match(doc, /Context certifies repo\/control-plane assumptions/i); + assert.match(doc, /Selection resolves integration mode, include patterns, and dispatcher profile into a receipt/i); assert.match(doc, /readiness receipt/i); assert.match(doc, /execution receipt/i); assert.match(doc, /existing required gate remains in place/i); From e4a103e547f8585e77d681257998808c38701ae4 Mon Sep 17 00:00:00 2001 From: svelderrainruiz Date: Tue, 31 Mar 2026 10:41:13 -0700 Subject: [PATCH 28/44] ci(pester): grant issues read to trusted router --- .github/workflows/pester-service-model-on-label.yml | 1 + .../__tests__/pester-service-model-workflow-contract.test.mjs | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/pester-service-model-on-label.yml b/.github/workflows/pester-service-model-on-label.yml index 2e16eb7e1..f2f0a16fe 100644 --- a/.github/workflows/pester-service-model-on-label.yml +++ b/.github/workflows/pester-service-model-on-label.yml @@ -28,6 +28,7 @@ concurrency: permissions: contents: read + issues: read pull-requests: read jobs: diff --git a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs index 81e07f807..bb6c75469 100644 --- a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs +++ b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs @@ -174,6 +174,7 @@ test('trusted PR pilot router only runs self-hosted service-model proof for work assert.match(workflow, /name:\s+Pester service-model pilot on trusted PR label/); assert.match(workflow, /pull_request_target:/); + assert.match(workflow, /permissions:\s*\n\s+contents:\s+read\s*\n\s+issues:\s+read\s*\n\s+pull-requests:\s+read/); assert.match(workflow, /types:\s*\[labeled, reopened, synchronize\]/); assert.doesNotMatch(workflow, /paths-ignore:/); assert.match(workflow, /group:\s+trusted-pilot-router-\$\{\{\s*github\.event\.pull_request\.number \|\| github\.event\.inputs\.sample_id \|\| github\.ref\s*\}\}/); From 4f59f848a9666fdd0bdc49df7574019123ea7222 Mon Sep 17 00:00:00 2001 From: svelderrainruiz Date: Tue, 31 Mar 2026 10:46:33 -0700 Subject: [PATCH 29/44] ci(pester): isolate service-model concurrency groups --- .github/workflows/pester-context.yml | 2 +- .github/workflows/pester-run.yml | 2 +- .github/workflows/pester-selection.yml | 2 +- .github/workflows/selfhosted-readiness.yml | 2 +- .../__tests__/pester-service-model-workflow-contract.test.mjs | 4 ++++ 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pester-context.yml b/.github/workflows/pester-context.yml index a7f4bc57b..c9caba965 100644 --- a/.github/workflows/pester-context.yml +++ b/.github/workflows/pester-context.yml @@ -47,7 +47,7 @@ on: type: string concurrency: - group: ${{ github.workflow }}-${{ github.event.inputs.sample_id || inputs.sample_id || github.ref }} + group: pester-context-${{ github.event.inputs.sample_id || inputs.sample_id || github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/pester-run.yml b/.github/workflows/pester-run.yml index 24448737b..0545ad029 100644 --- a/.github/workflows/pester-run.yml +++ b/.github/workflows/pester-run.yml @@ -98,7 +98,7 @@ on: type: string concurrency: - group: ${{ github.workflow }}-pester-run-${{ github.event.inputs.sample_id || inputs.sample_id || github.ref }} + group: pester-run-${{ github.event.inputs.sample_id || inputs.sample_id || github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/pester-selection.yml b/.github/workflows/pester-selection.yml index 6d721d526..bedacc057 100644 --- a/.github/workflows/pester-selection.yml +++ b/.github/workflows/pester-selection.yml @@ -57,7 +57,7 @@ on: type: string concurrency: - group: ${{ github.workflow }}-${{ github.event.inputs.sample_id || inputs.sample_id || github.ref }} + group: pester-selection-${{ github.event.inputs.sample_id || inputs.sample_id || github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/selfhosted-readiness.yml b/.github/workflows/selfhosted-readiness.yml index a9a15c716..bbaa61733 100644 --- a/.github/workflows/selfhosted-readiness.yml +++ b/.github/workflows/selfhosted-readiness.yml @@ -38,7 +38,7 @@ on: type: string concurrency: - group: ${{ github.workflow }}-${{ github.event.inputs.sample_id || inputs.sample_id || github.ref }} + group: selfhosted-readiness-${{ github.event.inputs.sample_id || inputs.sample_id || github.ref }} cancel-in-progress: true jobs: diff --git a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs index bb6c75469..4833072d5 100644 --- a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs +++ b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs @@ -47,6 +47,7 @@ test('pester context owns repo/control-plane receipts before host readiness begi assert.match(workflow, /name:\s+Pester context/); assert.match(workflow, /workflow_call:/); + assert.match(workflow, /group:\s+pester-context-\$\{\{\s*github\.event\.inputs\.sample_id \|\| inputs\.sample_id \|\| github\.ref\s*\}\}/); assert.match(workflow, /receipt_status:/); assert.match(workflow, /standing_priority_issue:/); assert.match(workflow, /standing_priority_reason:/); @@ -62,6 +63,7 @@ test('selfhosted readiness owns host-plane certification and emits a receipt art assert.match(workflow, /name:\s+Self-hosted readiness/); assert.match(workflow, /workflow_call:/); + assert.match(workflow, /group:\s+selfhosted-readiness-\$\{\{\s*github\.event\.inputs\.sample_id \|\| inputs\.sample_id \|\| github\.ref\s*\}\}/); assert.match(workflow, /receipt_status:/); assert.match(workflow, /runs-on:\s*\[self-hosted, Windows, X64, comparevi, capability-ingress\]/); assert.match(workflow, /repository:\s+\$\{\{\s*inputs\.checkout_repository \|\| github\.repository\s*\}\}/); @@ -81,6 +83,7 @@ test('pester selection owns pack shaping and dispatcher profile resolution befor assert.match(workflow, /name:\s+Pester selection/); assert.match(workflow, /workflow_call:/); + assert.match(workflow, /group:\s+pester-selection-\$\{\{\s*github\.event\.inputs\.sample_id \|\| inputs\.sample_id \|\| github\.ref\s*\}\}/); assert.match(workflow, /receipt_status:/); assert.match(workflow, /receipt_artifact_name:/); assert.match(workflow, /Normalize include_integration/); @@ -97,6 +100,7 @@ test('pester run is execution-only and validates context, readiness, and selecti assert.match(workflow, /name:\s+Pester run/); assert.match(workflow, /name:\s+Pester \(execution only\)/); + assert.match(workflow, /group:\s+pester-run-\$\{\{\s*github\.event\.inputs\.sample_id \|\| inputs\.sample_id \|\| github\.ref\s*\}\}/); assert.match(workflow, /if:\s+\$\{\{\s*inputs\.context_status == 'ready' && inputs\.readiness_status == 'ready' && inputs\.selection_status == 'ready'\s*\}\}/); assert.match(workflow, /execution_status:/); assert.match(workflow, /execution_receipt_artifact_name:/); From 68e53ea0c43d9ff0e1cb295b862897bd17f7e165 Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Tue, 31 Mar 2026 12:03:51 -0700 Subject: [PATCH 30/44] ci(auto): restore gh-based automerge on integration rail (#2084) Co-authored-by: svelderrainruiz --- .github/workflows/pr-automerge.yml | 10 ++++---- .../pr-automerge-workflow-contract.test.mjs | 23 +++++++++++++++++++ 2 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 tools/priority/__tests__/pr-automerge-workflow-contract.test.mjs diff --git a/.github/workflows/pr-automerge.yml b/.github/workflows/pr-automerge.yml index 67d6db0f5..ea1b3c94f 100644 --- a/.github/workflows/pr-automerge.yml +++ b/.github/workflows/pr-automerge.yml @@ -14,9 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Enable auto-merge (merge method) - uses: peter-evans/enable-pull-request-automerge@v3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - pull-request-number: ${{ github.event.pull_request.number }} - merge-method: MERGE - + env: + GH_TOKEN: ${{ secrets.GH_TOKEN || secrets.GITHUB_TOKEN }} + run: | + gh pr merge -R "${{ github.repository }}" --auto "${{ github.event.pull_request.number }}" diff --git a/tools/priority/__tests__/pr-automerge-workflow-contract.test.mjs b/tools/priority/__tests__/pr-automerge-workflow-contract.test.mjs new file mode 100644 index 000000000..5014a81af --- /dev/null +++ b/tools/priority/__tests__/pr-automerge-workflow-contract.test.mjs @@ -0,0 +1,23 @@ +#!/usr/bin/env node + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { readFileSync } from 'node:fs'; + +const repoRoot = process.cwd(); + +function readRepoFile(relativePath) { + return readFileSync(path.join(repoRoot, relativePath), 'utf8'); +} + +test('pr-automerge workflow uses gh CLI with GH_TOKEN fallback', () => { + const workflow = readRepoFile('.github/workflows/pr-automerge.yml'); + + assert.match(workflow, /name:\s*PR Auto-merge \(on label\)/); + assert.match(workflow, /pull_request_target:\s*\r?\n\s+types:\s*\[labeled, synchronize, reopened\]/); + assert.match(workflow, /if:\s*contains\(github\.event\.pull_request\.labels\.\*\.name,\s*'automerge'\)/); + assert.match(workflow, /GH_TOKEN:\s*\$\{\{\s*secrets\.GH_TOKEN \|\| secrets\.GITHUB_TOKEN\s*\}\}/); + assert.match(workflow, /gh pr merge -R "\$\{\{\s*github\.repository\s*\}\}" --auto "\$\{\{\s*github\.event\.pull_request\.number\s*\}\}"/); + assert.doesNotMatch(workflow, /enable-pull-request-automerge@v3/); +}); From 7b6a7647b36c7f2585c719987090df3fe4b33577 Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Tue, 31 Mar 2026 12:06:05 -0700 Subject: [PATCH 31/44] ci(auto): require squash auto-merge on integration rail (#2085) Co-authored-by: svelderrainruiz --- .github/workflows/pr-automerge.yml | 2 +- .../priority/__tests__/pr-automerge-workflow-contract.test.mjs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-automerge.yml b/.github/workflows/pr-automerge.yml index ea1b3c94f..9fc2cce95 100644 --- a/.github/workflows/pr-automerge.yml +++ b/.github/workflows/pr-automerge.yml @@ -17,4 +17,4 @@ jobs: env: GH_TOKEN: ${{ secrets.GH_TOKEN || secrets.GITHUB_TOKEN }} run: | - gh pr merge -R "${{ github.repository }}" --auto "${{ github.event.pull_request.number }}" + gh pr merge -R "${{ github.repository }}" --auto --squash "${{ github.event.pull_request.number }}" diff --git a/tools/priority/__tests__/pr-automerge-workflow-contract.test.mjs b/tools/priority/__tests__/pr-automerge-workflow-contract.test.mjs index 5014a81af..f4e71e9a7 100644 --- a/tools/priority/__tests__/pr-automerge-workflow-contract.test.mjs +++ b/tools/priority/__tests__/pr-automerge-workflow-contract.test.mjs @@ -18,6 +18,6 @@ test('pr-automerge workflow uses gh CLI with GH_TOKEN fallback', () => { assert.match(workflow, /pull_request_target:\s*\r?\n\s+types:\s*\[labeled, synchronize, reopened\]/); assert.match(workflow, /if:\s*contains\(github\.event\.pull_request\.labels\.\*\.name,\s*'automerge'\)/); assert.match(workflow, /GH_TOKEN:\s*\$\{\{\s*secrets\.GH_TOKEN \|\| secrets\.GITHUB_TOKEN\s*\}\}/); - assert.match(workflow, /gh pr merge -R "\$\{\{\s*github\.repository\s*\}\}" --auto "\$\{\{\s*github\.event\.pull_request\.number\s*\}\}"/); + assert.match(workflow, /gh pr merge -R "\$\{\{\s*github\.repository\s*\}\}" --auto --squash "\$\{\{\s*github\.event\.pull_request\.number\s*\}\}"/); assert.doesNotMatch(workflow, /enable-pull-request-automerge@v3/); }); From d78a552c07aabfb03e5432c261a12b8609d89bf4 Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Tue, 31 Mar 2026 12:07:46 -0700 Subject: [PATCH 32/44] ci(pester): split execution postprocess from dispatch (#2083) * ci(pester): split execution postprocess from dispatch * ci(auto): restore gh-based automerge on integration rail --------- Co-authored-by: svelderrainruiz --- .github/workflows/pester-evidence.yml | 23 +- .github/workflows/pester-run.yml | 52 +- CONTRIBUTING.md | 3 + Invoke-PesterTests.ps1 | 61 +- docs/knowledgebase/Pester-Service-Model.md | 9 + docs/requirements-pester-service-model-srs.md | 4 +- docs/rtm-pester-service-model.csv | 4 +- .../testing/pester-service-model-test-plan.md | 8 + package.json | 1 + tests/Get-PesterResultXmlSummary.Tests.ps1 | 60 ++ ...nvoke-PesterExecutionPostprocess.Tests.ps1 | 120 ++++ tools/Get-PesterResultXmlSummary.ps1 | 157 +++++ tools/Invoke-PesterExecutionPostprocess.ps1 | 153 +++++ tools/Run-PesterExecutionOnly.Local.ps1 | 604 ++++++++++++++++++ ...vice-model-local-harness-contract.test.mjs | 63 ++ ...r-service-model-workflow-contract.test.mjs | 19 + 16 files changed, 1315 insertions(+), 26 deletions(-) create mode 100644 tests/Get-PesterResultXmlSummary.Tests.ps1 create mode 100644 tests/Invoke-PesterExecutionPostprocess.Tests.ps1 create mode 100644 tools/Get-PesterResultXmlSummary.ps1 create mode 100644 tools/Invoke-PesterExecutionPostprocess.ps1 create mode 100644 tools/Run-PesterExecutionOnly.Local.ps1 create mode 100644 tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs diff --git a/.github/workflows/pester-evidence.yml b/.github/workflows/pester-evidence.yml index f496102e8..5043e8817 100644 --- a/.github/workflows/pester-evidence.yml +++ b/.github/workflows/pester-evidence.yml @@ -306,6 +306,12 @@ jobs: $reasons.Add('execution-job-skipped') | Out-Null } elseif ($executionJobResult -eq 'cancelled') { $reasons.Add('execution-job-cancelled') | Out-Null + } elseif ($executionJobResult -eq 'results-xml-truncated') { + $reasons.Add('execution-job-results-xml-truncated') | Out-Null + } elseif ($executionJobResult -eq 'invalid-results-xml') { + $reasons.Add('execution-job-invalid-results-xml') | Out-Null + } elseif ($executionJobResult -eq 'missing-results-xml') { + $reasons.Add('execution-job-missing-results-xml') | Out-Null } elseif ($executionJobResult -eq 'seam-defect') { $reasons.Add('execution-job-seam-defect') | Out-Null } elseif ($executionJobResult -eq 'unknown') { @@ -324,6 +330,15 @@ jobs: $classification = 'readiness-blocked' } elseif (($selectionStatus -ne 'ready' -and $executionJobResult -in @('skipped','cancelled')) -or $executionReceiptStatus -eq 'selection-blocked') { $classification = 'selection-blocked' + } elseif ($executionReceiptStatus -eq 'results-xml-truncated') { + $classification = 'results-xml-truncated' + $reasons.Add('execution-receipt-results-xml-truncated') | Out-Null + } elseif ($executionReceiptStatus -eq 'invalid-results-xml') { + $classification = 'invalid-results-xml' + $reasons.Add('execution-receipt-invalid-results-xml') | Out-Null + } elseif ($executionReceiptStatus -eq 'missing-results-xml') { + $classification = 'missing-results-xml' + $reasons.Add('execution-receipt-missing-results-xml') | Out-Null } elseif ($executionReceiptStatus -eq 'seam-defect') { $reasons.Add('execution-receipt-seam-defect') | Out-Null } elseif ($executionReceiptStatus -eq 'test-failures') { @@ -334,7 +349,13 @@ jobs: if ('${{ steps.execution_receipt.outputs.dispatcher_exit_code }}' -and '${{ steps.execution_receipt.outputs.dispatcher_exit_code }}' -ne $dispatcherExitCode) { $reasons.Add('dispatcher-exit-mismatch') | Out-Null } - if (($summary.failed + $summary.errors) -gt 0 -or $dispatcherExitCode -ne '0') { + if (($summary.PSObject.Properties.Name -contains 'resultsXmlStatus') -and [string]$summary.resultsXmlStatus -like 'truncated*') { + $classification = 'results-xml-truncated' + $reasons.Add(("results-xml-status={0}" -f [string]$summary.resultsXmlStatus)) | Out-Null + } elseif (($summary.PSObject.Properties.Name -contains 'resultsXmlStatus') -and [string]$summary.resultsXmlStatus -like 'invalid*') { + $classification = 'invalid-results-xml' + $reasons.Add(("results-xml-status={0}" -f [string]$summary.resultsXmlStatus)) | Out-Null + } elseif (($summary.failed + $summary.errors) -gt 0 -or $dispatcherExitCode -ne '0') { $classification = 'test-failures' } else { $classification = 'ok' diff --git a/.github/workflows/pester-run.yml b/.github/workflows/pester-run.yml index 0545ad029..b318cb6ae 100644 --- a/.github/workflows/pester-run.yml +++ b/.github/workflows/pester-run.yml @@ -131,15 +131,6 @@ jobs: shell: pwsh run: node tools/npm/cli.mjs ci - - name: Export workflow token for priority sync - shell: pwsh - env: - WORKFLOW_TOKEN: ${{ github.token }} - run: | - if (-not $env:WORKFLOW_TOKEN) { throw 'github.token is empty' } - "GH_TOKEN=$env:WORKFLOW_TOKEN" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - "GITHUB_TOKEN=$env:WORKFLOW_TOKEN" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - - name: Download context receipt artifact uses: actions/download-artifact@v5 with: @@ -289,6 +280,8 @@ jobs: continue-on-error: true shell: pwsh run: | + $previousErrorActionPreference = $ErrorActionPreference + $ErrorActionPreference = 'Continue' $logPath = 'tests/results/pester-dispatcher.log' $logDir = Split-Path -Parent $logPath if (-not (Test-Path $logDir)) { New-Item -ItemType Directory -Force -Path $logDir | Out-Null } @@ -326,8 +319,12 @@ jobs: $exitCode = 0 try { & $dispatcherPath @bound 2>&1 | Tee-Object -FilePath $logPath - $exitCode = $LASTEXITCODE + $exitCode = if ($null -ne $LASTEXITCODE) { [int]$LASTEXITCODE } else { 0 } + } catch { + $_ | Out-String | Tee-Object -FilePath $logPath -Append | Write-Host + $exitCode = if ($null -ne $LASTEXITCODE -and [int]$LASTEXITCODE -ne 0) { [int]$LASTEXITCODE } else { 1 } } finally { + $ErrorActionPreference = $previousErrorActionPreference if ($heartbeatJob) { Stop-Job -Id $heartbeatJob.Id | Out-Null Remove-Job -Id $heartbeatJob.Id -Force | Out-Null @@ -343,8 +340,22 @@ jobs: $env:GITHUB_OUTPUT = [System.IO.Path]::GetFullPath($fallbackOutput) } "exit_code=$exitCode" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + if ($exitCode -ne 0) { + Write-Warning ("Dispatcher exited with code {0}" -f $exitCode) + } $global:LASTEXITCODE = 0 + - name: Postprocess execution results + id: postprocess + if: always() + shell: pwsh + run: | + $toolPath = Join-Path (Get-Location) 'tools/Invoke-PesterExecutionPostprocess.ps1' + if (-not (Test-Path -LiteralPath $toolPath -PathType Leaf)) { + throw "Postprocess tool not found at $toolPath" + } + pwsh -NoLogo -NoProfile -File $toolPath -ResultsDir 'tests/results' + - name: Release session lock if: always() shell: pwsh @@ -360,15 +371,29 @@ jobs: New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null } $summaryPath = Join-Path $resultsDir 'pester-summary.json' + $postprocessPath = Join-Path $resultsDir 'pester-execution-postprocess.json' $dispatcherExitCode = '${{ steps.dispatcher.outputs.exit_code }}' $status = 'seam-defect' + $postprocessStatus = '' + $resultsXmlStatus = '' if ($dispatcherExitCode -eq '') { $dispatcherExitCode = '-1' } $contextReceiptPresent = Test-Path -LiteralPath '${{ steps.context_receipt.outputs.path }}' $readinessReceiptPresent = Test-Path -LiteralPath '${{ steps.readiness_receipt.outputs.path }}' $selectionReceiptPresent = Test-Path -LiteralPath '${{ steps.selection_receipt.outputs.path }}' - if ((Test-Path -LiteralPath $summaryPath) -and $dispatcherExitCode -eq '0') { + if (Test-Path -LiteralPath $postprocessPath) { + try { + $postprocess = Get-Content -LiteralPath $postprocessPath -Raw | ConvertFrom-Json -ErrorAction Stop + $postprocessStatus = [string]$postprocess.status + $resultsXmlStatus = [string]$postprocess.resultsXmlStatus + } catch { + Write-Warning ("Failed to parse postprocess report: {0}" -f $_.Exception.Message) + } + } + if ($postprocessStatus -in @('results-xml-truncated', 'invalid-results-xml', 'missing-results-xml')) { + $status = $postprocessStatus + } elseif ((Test-Path -LiteralPath $summaryPath) -and $dispatcherExitCode -eq '0') { $status = 'completed' } elseif (Test-Path -LiteralPath $summaryPath) { $status = 'test-failures' @@ -392,6 +417,8 @@ jobs: selectionIntegrationMode = '${{ steps.selection_receipt.outputs.integration_mode }}' selectionFixtureRequired = '${{ steps.selection_receipt.outputs.fixture_required }}' dispatcherExitCode = [int]$dispatcherExitCode + postprocessStatus = $postprocessStatus + resultsXmlStatus = $resultsXmlStatus summaryPresent = Test-Path -LiteralPath $summaryPath status = $status rawArtifactName = 'pester-run-raw' @@ -438,6 +465,9 @@ jobs: } elseif ('${{ inputs.selection_status }}' -ne 'ready') { $executionStatus = 'skipped' if ([string]::IsNullOrWhiteSpace($receiptStatus)) { $receiptStatus = 'selection-blocked' } + } elseif ($receiptStatus -in @('results-xml-truncated', 'invalid-results-xml', 'missing-results-xml')) { + $executionStatus = $receiptStatus + $rawArtifactName = 'pester-run-raw' } else { switch ('${{ needs.pester.result }}') { 'success' { diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 43b4c74cf..7817921fe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,6 +96,9 @@ Quick commands: # Unit tests only ./Invoke-PesterTests.ps1 +# Local execution-slice harness (lock + LV guard + fixture prep + dispatcher) +pwsh -NoLogo -NoProfile -File tools/Run-PesterExecutionOnly.Local.ps1 + # Integration tests (requires LVCompare) $env:LV_BASE_VI = 'VI1.vi' $env:LV_HEAD_VI = 'VI2.vi' diff --git a/Invoke-PesterTests.ps1 b/Invoke-PesterTests.ps1 index 365068857..7f1491e9e 100644 --- a/Invoke-PesterTests.ps1 +++ b/Invoke-PesterTests.ps1 @@ -2662,21 +2662,54 @@ if (-not (Test-Path -LiteralPath $xmlPath -PathType Leaf)) { # Parse NUnit XML results Write-Host "Parsing test results..." -ForegroundColor Yellow +$executionPostprocessStatus = 'seam-defect' +$resultXmlStatus = 'missing' +$resultXmlSummarySource = $null +$resultXmlCloseTagPresent = $false +$resultXmlParseError = $null +$resultXmlSizeBytes = 0 try { - [xml]$doc = Get-Content -LiteralPath $xmlPath -Raw -ErrorAction Stop - $rootNode = $doc.'test-results' - - if (-not $rootNode) { - Write-Error "Invalid NUnit XML format in results file" + $postprocessToolPath = Join-Path $PSScriptRoot 'tools/Invoke-PesterExecutionPostprocess.ps1' + if (-not (Test-Path -LiteralPath $postprocessToolPath -PathType Leaf)) { + throw "Invoke-PesterExecutionPostprocess.ps1 not found: $postprocessToolPath" + } + & $postprocessToolPath -ResultsDir $resultsDir -JsonSummaryPath $JsonSummaryPath -XmlStabilizationTimeoutSeconds 3 -XmlPollIntervalMilliseconds 200 | Out-Host + if ($LASTEXITCODE -ne 0) { + throw "Invoke-PesterExecutionPostprocess.ps1 failed with exit code $LASTEXITCODE." + } + $postprocessReportPath = Join-Path $resultsDir 'pester-execution-postprocess.json' + if (-not (Test-Path -LiteralPath $postprocessReportPath -PathType Leaf)) { + throw "Execution postprocess report not found: $postprocessReportPath" + } + $postprocessReport = Get-Content -LiteralPath $postprocessReportPath -Raw | ConvertFrom-Json -ErrorAction Stop + + $executionPostprocessStatus = [string]$postprocessReport.status + $resultXmlStatus = [string]$postprocessReport.resultsXmlStatus + $resultXmlSummarySource = [string]$postprocessReport.resultsXmlSummarySource + $resultXmlCloseTagPresent = [bool]$postprocessReport.resultsXmlCloseTagPresent + $resultXmlParseError = [string]$postprocessReport.parseError + $resultXmlSizeBytes = [int64]$postprocessReport.resultsXmlSizeBytes + + if ($executionPostprocessStatus -in @('missing-results-xml', 'seam-defect')) { + $detail = if ([string]::IsNullOrWhiteSpace($resultXmlParseError)) { $executionPostprocessStatus } else { "{0}: {1}" -f $executionPostprocessStatus, $resultXmlParseError } + Write-Error ("Failed to parse test results: {0}" -f $detail) exit 1 } - - [int]$total = $rootNode.total - [int]$failed = $rootNode.failures - [int]$errors = $rootNode.errors - [int]$skipped = $rootNode.'not-run' + + [int]$total = [int]$postprocessReport.total + [int]$failed = [int]$postprocessReport.failed + [int]$errors = [int]$postprocessReport.errors + [int]$skipped = [int]$postprocessReport.skipped $passed = $total - $failed - $errors + if ($executionPostprocessStatus -ne 'complete') { + Write-Warning ("Pester execution postprocess status={0}; resultsXmlStatus={1}; summarySource={2}; closeTagPresent={3}; parseError={4}" -f $executionPostprocessStatus, $resultXmlStatus, $resultXmlSummarySource, $resultXmlCloseTagPresent, $resultXmlParseError) + if (($failed + $errors) -eq 0) { + Write-Error 'Incomplete Pester results XML reported zero failures and zero errors; refusing to certify a pass from partial XML.' + exit 1 + } + } + # Discovery failure adjustment: if discovery failures detected and no existing failures/errors recorded, promote to errors if ($discoveryFailureCount -gt 0 -and $failed -eq 0 -and $errors -eq 0) { Write-Host "Discovery failures detected ($discoveryFailureCount) with zero test failures; elevating to error state." -ForegroundColor Red @@ -3122,6 +3155,14 @@ try { schemaVersion = $SchemaSummaryVersion timedOut = $script:timedOut discoveryFailures = $discoveryFailureCount + executionPostprocessStatus = $executionPostprocessStatus + resultsXmlStatus = $resultXmlStatus + resultsXmlSummarySource = $resultXmlSummarySource + resultsXmlCloseTagPresent = [bool]$resultXmlCloseTagPresent + resultsXmlSizeBytes = $resultXmlSizeBytes + } + if (-not [string]::IsNullOrWhiteSpace($resultXmlParseError)) { + Add-Member -InputObject $jsonObj -Name resultsXmlParseError -MemberType NoteProperty -Value $resultXmlParseError } if ($labviewPidTrackerLoaded) { diff --git a/docs/knowledgebase/Pester-Service-Model.md b/docs/knowledgebase/Pester-Service-Model.md index 0b5570354..f48bd4fa1 100644 --- a/docs/knowledgebase/Pester-Service-Model.md +++ b/docs/knowledgebase/Pester-Service-Model.md @@ -27,6 +27,11 @@ The additive pilot introduces seven workflow surfaces: - receipt-driven Pester execution only - `.github/workflows/pester-evidence.yml` - summary, classification, session-index, dashboard, and artifact publication +- `tools/Run-PesterExecutionOnly.Local.ps1` + - local harness for the execution slice without the workflow shell: lock, LV guard, + fixture prep, dispatcher profile, dispatch, execution postprocess, and local execution receipt +- `tools/Invoke-PesterExecutionPostprocess.ps1` + - execution-post contract for XML integrity classification and machine-readable summary repair ## Design Rules @@ -39,6 +44,10 @@ The additive pilot introduces seven workflow surfaces: - Execution consumes context, selection, and readiness. It does not normalize pack inputs, choose the dispatcher profile, bootstrap Docker runtimes, install core toolchains, or discover standing-priority state. - Execution writes an execution receipt before uploading raw artifacts so evidence can classify the real seam outcome. - Execution must also emit a skip-safe execution contract from an always-on finalize path so reusable-workflow outputs do not collapse when the execution job never starts. +- Execution-post shall classify `results-xml-truncated`, `invalid-results-xml`, and `missing-results-xml` explicitly instead of collapsing XML integrity debt into generic `seam-defect`. +- Local iteration must not depend on the workflow shell. The local harness should + make the execution slice runnable on its own while keeping the same preflight + and receipt boundaries. - Evidence consumes raw execution output plus the execution receipt. It classifies `context-blocked`, `readiness-blocked`, and `seam-defect` explicitly instead of collapsing them into one execution symptom. - The existing required gate remains in place until the pilot proves equivalent or better behavior. - Trusted PR proving must stay on `pull_request_target` with same-owner gating. Cross-owner fork heads are not allowed to drive self-hosted execution. diff --git a/docs/requirements-pester-service-model-srs.md b/docs/requirements-pester-service-model-srs.md index 79653f176..12c22f64f 100644 --- a/docs/requirements-pester-service-model-srs.md +++ b/docs/requirements-pester-service-model-srs.md @@ -38,8 +38,8 @@ | REQ-PSM-002 | The context layer shall resolve repository and standing-priority state and emit a context receipt before readiness begins. | Repo/control-plane assumptions must be certified separately from host readiness. | `pester-context.yml` uploads `pester-context.json` with `schema=pester-context-receipt@v1` and a status of `ready`, `warning`, or `blocked`. | `TEST-PSM-002` | | REQ-PSM-003 | The readiness layer shall certify runner labels, session-lock health, `.NET`, Windows Docker runtime, and LVCompare or idle LabVIEW state, and shall emit a bounded-freshness readiness receipt. | Self-hosted ingress debt must be observable as readiness debt, not hidden inside execution. | `selfhosted-readiness.yml` uploads `selfhosted-readiness.json` with `schema=pester-selfhosted-readiness-receipt@v1`, individual probe outcomes, and `freshnessWindowSeconds=900`. | `TEST-PSM-003` | | REQ-PSM-004 | The selection layer shall resolve integration mode, include patterns, and dispatcher profile into a selection receipt before self-hosted execution begins. | Pack shaping and dispatcher defaults are control-plane decisions, not execution-side improvisation. | `pester-selection.yml` uploads `pester-selection.json` with `schema=pester-selection-receipt@v1`, the normalized pack selector, and dispatcher profile values. | `TEST-PSM-004` | -| REQ-PSM-005 | The execution layer shall validate context, readiness, and selection receipts before dispatch, run the declared Pester pack without bootstrapping Docker runtimes or core toolchains, and shall emit an execution receipt even when execution is skipped. | Execution should only execute; it must not absorb context, selection, or readiness responsibilities. | `pester-run.yml` refuses to start unless all upstream receipts are ready, calls `Invoke-PesterTests.ps1`, uploads raw results when produced, and always uploads `pester-run-receipt.json`. | `TEST-PSM-005` | -| REQ-PSM-006 | The evidence layer shall classify `context-blocked`, `selection-blocked`, `readiness-blocked`, `test-failures`, and `seam-defect` explicitly from execution receipts and raw artifacts. | Operators need precise failure classes instead of `missing-summary` ambiguity. | `pester-evidence.yml` reads the execution contract and emits the explicit classification when raw artifacts are missing or execution is skipped. | `TEST-PSM-006` | +| REQ-PSM-005 | The execution layer shall validate context, readiness, and selection receipts before dispatch, run the declared Pester pack without bootstrapping Docker runtimes or core toolchains, and shall emit an execution receipt even when execution is skipped. Execution-post shall run as a separate contract that can repair machine-readable summaries from raw results without re-entering dispatch. | Execution should only execute; it must not absorb context, selection, or readiness responsibilities, and XML/result postprocess debt must be classifiable separately from dispatch. | `pester-run.yml` refuses to start unless all upstream receipts are ready, calls `Invoke-PesterTests.ps1`, runs `Invoke-PesterExecutionPostprocess.ps1`, uploads raw results when produced, and always uploads `pester-run-receipt.json`; `tools/Run-PesterExecutionOnly.Local.ps1` mirrors that slice locally without the workflow shell. | `TEST-PSM-005` | +| REQ-PSM-006 | The evidence layer shall classify `context-blocked`, `selection-blocked`, `readiness-blocked`, `results-xml-truncated`, `invalid-results-xml`, `missing-results-xml`, `test-failures`, and `seam-defect` explicitly from execution receipts and raw artifacts. | Operators need precise failure classes instead of `missing-summary` ambiguity. | `pester-evidence.yml` reads the execution contract and emits the explicit classification when raw artifacts are missing, execution is skipped, or result XML is truncated, invalid, or missing. | `TEST-PSM-006` | | REQ-PSM-007 | The pilot shall remain additive until it proves equivalent or better behavior than the monolithic required gate. | Promotion must follow evidence, not preference. | The service-model knowledgebase and promotion rule state that the legacy required gate remains in place until the pilot is proven. | `TEST-PSM-007` | ## Assumptions diff --git a/docs/rtm-pester-service-model.csv b/docs/rtm-pester-service-model.csv index e86cbd068..3ed84be37 100644 --- a/docs/rtm-pester-service-model.csv +++ b/docs/rtm-pester-service-model.csv @@ -3,6 +3,6 @@ REQ-PSM-001,"Trusted router admits workflow_dispatch and same-owner labeled PR h REQ-PSM-002,"Context resolves repository and standing-priority state before readiness and emits a receipt","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-002,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs",".github/workflows/pester-context.yml;tools/priority/run-sync-standing-priority.mjs",Implemented REQ-PSM-003,"Readiness certifies runner labels, session lock, dotnet, Docker runtime, and LVCompare state and emits a bounded-freshness receipt","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-003,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs",".github/workflows/selfhosted-readiness.yml;tools/Invoke-DockerRuntimeManager.ps1;tools/Session-Lock.ps1",Implemented REQ-PSM-004,"Selection resolves integration mode, include patterns, and dispatcher profile into a receipt before execution","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-004,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs",".github/workflows/pester-selection.yml;.github/actions/dispatcher-profile/action.yml",Implemented -REQ-PSM-005,"Execution validates context, readiness, and selection receipts, runs Invoke-PesterTests without host bootstrap, and emits an execution receipt even when skipped","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-005,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs",".github/workflows/pester-run.yml;Invoke-PesterTests.ps1",Implemented -REQ-PSM-006,"Evidence classifies context-blocked, selection-blocked, readiness-blocked, test-failures, and seam-defect explicitly from the execution contract","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-006,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs",".github/workflows/pester-evidence.yml",Implemented +REQ-PSM-005,"Execution validates context, readiness, and selection receipts, runs Invoke-PesterTests without host bootstrap, invokes execution-post for XML/result postprocess, and emits an execution receipt even when skipped","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-005,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs;tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs;tests/Invoke-PesterExecutionPostprocess.Tests.ps1",".github/workflows/pester-run.yml;Invoke-PesterTests.ps1;tools/Invoke-PesterExecutionPostprocess.ps1;tools/Run-PesterExecutionOnly.Local.ps1",Implemented +REQ-PSM-006,"Evidence classifies context-blocked, selection-blocked, readiness-blocked, results-xml-truncated, invalid-results-xml, missing-results-xml, test-failures, and seam-defect explicitly from the execution contract","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-006,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs;tests/Invoke-PesterExecutionPostprocess.Tests.ps1",".github/workflows/pester-evidence.yml;tools/Invoke-PesterExecutionPostprocess.ps1",Implemented REQ-PSM-007,"The pilot remains additive until it proves equivalent or better than the monolith","docs/requirements-pester-service-model-srs.md",Medium,TEST-PSM-007,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs","docs/knowledgebase/Pester-Service-Model.md;.github/workflows/pester-gate.yml",Implemented diff --git a/docs/testing/pester-service-model-test-plan.md b/docs/testing/pester-service-model-test-plan.md index 7f5c7fda3..0e91b2485 100644 --- a/docs/testing/pester-service-model-test-plan.md +++ b/docs/testing/pester-service-model-test-plan.md @@ -16,19 +16,25 @@ | Item | Type | Risk | Notes | | --- | --- | --- | --- | | `pester-service-model-workflow-contract.test.mjs` | Integration | High | Verifies the workflow split and core receipt/evidence obligations | +| `pester-service-model-local-harness-contract.test.mjs` | Integration | High | Verifies the local execution harness stays aligned to the execution-layer boundary without the workflow shell | +| `Get-PesterResultXmlSummary.Tests.ps1` + `Invoke-PesterExecutionPostprocess.Tests.ps1` | Unit | High | Verifies execution-post can classify truncated, invalid, and missing XML integrity states and repair machine-readable summaries without re-entering dispatch | | `pester-service-model-quality-workflow-contract.test.mjs` | Integration | Medium | Verifies the coverage gate and docs link-check control-plane workflow | | `pester-gate.yml` + trusted pilot routing | Workflow | High | Verifies admission and orchestration across layers | | `pester-selection.yml` selection contract | Workflow | High | Verifies pack shaping and dispatcher-profile resolution leave execution clean | | `Invoke-PesterTests.ps1` execution contract | Execution | High | Verifies the dispatcher remains the execution engine only | +| `Invoke-PesterExecutionPostprocess.ps1` execution-post contract | Execution | High | Verifies XML integrity classification and machine-readable summary repair occur after dispatch | +| `Run-PesterExecutionOnly.Local.ps1` local harness | Execution | High | Mirrors lock, LV guard, fixture prep, dispatcher profile, dispatch, execution-post, and local execution receipt without the workflow shell | ## Entry Criteria - The service-model workflows and contract test are in sync with the declared requirements. +- The local execution harness remains traced to the execution-layer requirement and contract tests. - The retained fork dossier still matches the mounted upstream promotion slice. ## Exit Criteria - Workflow-contract tests pass. +- Local harness contract tests pass. - Hosted packet quality and release-evidence workflows complete. - Any remaining action items are explicitly accepted before the slice widens beyond hosted quality and evidence. @@ -38,6 +44,7 @@ | Metric | Target | Evidence | | --- | --- | --- | | Workflow contract coverage | All layer responsibilities represented | `tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs` | +| Local harness contract coverage | Local execution slice stays aligned with workflow execution boundaries | `tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs` | | Receipt coverage | Context, selection, readiness, execution, and evidence all emit auditable artifacts | assurance report + integration runs | | Classification coverage | blocked and defect outcomes remain distinguishable | evidence workflow outputs | | Packet coverage gate | Retained `coverage.xml` and named PR coverage gate | `.github/workflows/pester-service-model-quality.yml` | @@ -49,5 +56,6 @@ upstream integration runs plus hosted release-evidence outputs - Test report location: `tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs` + `tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs` - Defect tracking link: `#2069` and `#2078` diff --git a/package.json b/package.json index 42d318920..f1a56ea14 100644 --- a/package.json +++ b/package.json @@ -253,6 +253,7 @@ "session:teststand:validate": "node tools/npm/run-script.mjs schema:validate -- --schema docs/schema/generated/teststand-compare-session.schema.json --data fixtures/teststand-session/session-index.json --data fixtures/teststand-session/session-index.dual-plane-parity.json --data tests/results/teststand-session/session-index.json --optional", "smoke:vi-history": "pwsh -NoLogo -NoProfile -File tools/Test-PRVIHistorySmoke.ps1", "smoke:vi-stage": "pwsh -NoLogo -NoProfile -File tools/Test-PRVIStagingSmoke.ps1", + "tests:execution:local": "pwsh -NoLogo -NoProfile -File tools/Run-PesterExecutionOnly.Local.ps1", "tests:comparevi": "pwsh -NoLogo -NoProfile -File Invoke-PesterTests.ps1 -IncludePatterns CompareVI*", "tests:comparevi:int": "pwsh -NoLogo -NoProfile -File Invoke-PesterTests.ps1 -IncludePatterns CompareVI* -IntegrationMode include", "tests:comparevi:single-int": "pwsh -NoLogo -NoProfile -Command \"$env:LV_BASE_VI=(Resolve-Path 'VI1.vi').Path; $env:LV_HEAD_VI=(Resolve-Path 'VI2.vi').Path; pwsh -NoLogo -NoProfile -File Invoke-PesterTests.ps1 -TestsPath tests/CompareVI.RealCli.SingleRun.Integration.Tests.ps1 -IntegrationMode include\"", diff --git a/tests/Get-PesterResultXmlSummary.Tests.ps1 b/tests/Get-PesterResultXmlSummary.Tests.ps1 new file mode 100644 index 000000000..aba61f00a --- /dev/null +++ b/tests/Get-PesterResultXmlSummary.Tests.ps1 @@ -0,0 +1,60 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Describe 'Get-PesterResultXmlSummary.ps1' -Tag 'Unit' { + BeforeAll { + $script:repoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path + $script:summaryTool = Join-Path $script:repoRoot 'tools/Get-PesterResultXmlSummary.ps1' + } + + It 'parses complete NUnit XML through the DOM path' { + $xmlPath = Join-Path $TestDrive 'complete.xml' + $xml = @' + + + + + +'@ + Set-Content -LiteralPath $xmlPath -Value $xml -Encoding UTF8 + + $result = & $script:summaryTool -XmlPath $xmlPath -StabilizationTimeoutSeconds 0 + + $result.schema | Should -Be 'pester-result-xml-summary@v1' + $result.status | Should -Be 'complete' + $result.summarySource | Should -Be 'xml-dom' + $result.closeTagPresent | Should -BeTrue + $result.total | Should -Be 4 + $result.failed | Should -Be 1 + $result.errors | Should -Be 0 + $result.skipped | Should -Be 1 + $result.parseError | Should -BeNullOrEmpty + } + + It 'falls back to root attributes when the XML is truncated but totals are still recoverable' { + $xmlPath = Join-Path $TestDrive 'truncated.xml' + $xml = @' + + + + + + + + + Expected 0, but got 1. +'@ + Set-Content -LiteralPath $xmlPath -Value $xml -Encoding UTF8 + + $result = & $script:summaryTool -XmlPath $xmlPath -StabilizationTimeoutSeconds 0 + + $result.status | Should -Be 'truncated-root' + $result.summarySource | Should -Be 'root-attributes' + $result.closeTagPresent | Should -BeFalse + $result.total | Should -Be 1033 + $result.failed | Should -Be 156 + $result.errors | Should -Be 0 + $result.skipped | Should -Be 0 + $result.parseError | Should -Match 'Unexpected end of file' + } +} diff --git a/tests/Invoke-PesterExecutionPostprocess.Tests.ps1 b/tests/Invoke-PesterExecutionPostprocess.Tests.ps1 new file mode 100644 index 000000000..906edf106 --- /dev/null +++ b/tests/Invoke-PesterExecutionPostprocess.Tests.ps1 @@ -0,0 +1,120 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Describe 'Invoke-PesterExecutionPostprocess.ps1' -Tag 'Unit' { + BeforeAll { + $script:repoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path + $script:postprocessTool = Join-Path $script:repoRoot 'tools/Invoke-PesterExecutionPostprocess.ps1' + } + + It 'preserves an existing summary and marks complete XML as complete' { + $resultsDir = Join-Path $TestDrive 'complete' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + $xmlPath = Join-Path $resultsDir 'pester-results.xml' + $summaryPath = Join-Path $resultsDir 'pester-summary.json' + @' + + + + + +'@ | Set-Content -LiteralPath $xmlPath -Encoding UTF8 + @{ + duration_s = 1.25 + pesterVersion = '5.7.1' + includeIntegration = $false + } | ConvertTo-Json | Set-Content -LiteralPath $summaryPath -Encoding UTF8 + + & $script:postprocessTool -ResultsDir $resultsDir | Out-Null + + $report = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-execution-postprocess.json') -Raw | ConvertFrom-Json + $summary = Get-Content -LiteralPath $summaryPath -Raw | ConvertFrom-Json + + $report.status | Should -Be 'complete' + $report.summaryWritten | Should -BeTrue + $summary.total | Should -Be 4 + $summary.failed | Should -Be 1 + $summary.errors | Should -Be 0 + $summary.passed | Should -Be 3 + $summary.resultsXmlStatus | Should -Be 'complete' + $summary.duration_s | Should -Be 1.25 + $summary.pesterVersion | Should -Be '5.7.1' + } + + It 'writes a repaired machine-readable summary when XML is truncated' { + $resultsDir = Join-Path $TestDrive 'truncated' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + $xmlPath = Join-Path $resultsDir 'pester-results.xml' + @' + + + + + + + + + Expected 0, but got 1. +'@ | Set-Content -LiteralPath $xmlPath -Encoding UTF8 + + & $script:postprocessTool -ResultsDir $resultsDir | Out-Null + + $report = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-execution-postprocess.json') -Raw | ConvertFrom-Json + $summary = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-summary.json') -Raw | ConvertFrom-Json + + $report.status | Should -Be 'results-xml-truncated' + $report.resultsXmlStatus | Should -Be 'truncated-root' + $report.summaryWritten | Should -BeTrue + $summary.executionPostprocessStatus | Should -Be 'results-xml-truncated' + $summary.resultsXmlStatus | Should -Be 'truncated-root' + $summary.total | Should -Be 1033 + $summary.failed | Should -Be 156 + $summary.errors | Should -Be 0 + $summary.passed | Should -Be 877 + } + + It 'classifies malformed closed XML with recoverable root attributes as invalid-results-xml' { + $resultsDir = Join-Path $TestDrive 'invalid' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + $xmlPath = Join-Path $resultsDir 'pester-results.xml' + @' + + + + + + +'@ | Set-Content -LiteralPath $xmlPath -Encoding UTF8 + + & $script:postprocessTool -ResultsDir $resultsDir | Out-Null + + $report = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-execution-postprocess.json') -Raw | ConvertFrom-Json + $summary = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-summary.json') -Raw | ConvertFrom-Json + + $report.status | Should -Be 'invalid-results-xml' + $report.resultsXmlStatus | Should -Be 'invalid-root-attributes' + $report.summaryWritten | Should -BeTrue + $summary.executionPostprocessStatus | Should -Be 'invalid-results-xml' + $summary.resultsXmlStatus | Should -Be 'invalid-root-attributes' + $summary.total | Should -Be 8 + $summary.failed | Should -Be 2 + $summary.errors | Should -Be 1 + $summary.passed | Should -Be 5 + } + + It 'classifies missing XML as missing-results-xml without writing a summary' { + $resultsDir = Join-Path $TestDrive 'missing' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + + & $script:postprocessTool -ResultsDir $resultsDir | Out-Null + + $reportPath = Join-Path $resultsDir 'pester-execution-postprocess.json' + $summaryPath = Join-Path $resultsDir 'pester-summary.json' + $report = Get-Content -LiteralPath $reportPath -Raw | ConvertFrom-Json + + $report.status | Should -Be 'missing-results-xml' + $report.resultsXmlStatus | Should -Be 'missing' + $report.summaryWritten | Should -BeFalse + (Test-Path -LiteralPath $summaryPath) | Should -BeFalse + } +} diff --git a/tools/Get-PesterResultXmlSummary.ps1 b/tools/Get-PesterResultXmlSummary.ps1 new file mode 100644 index 000000000..e2bafce56 --- /dev/null +++ b/tools/Get-PesterResultXmlSummary.ps1 @@ -0,0 +1,157 @@ +param( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string]$XmlPath, + + [Parameter(Mandatory = $false)] + [ValidateRange(0, 30)] + [int]$StabilizationTimeoutSeconds = 3, + + [Parameter(Mandatory = $false)] + [ValidateRange(25, 5000)] + [int]$PollIntervalMilliseconds = 200 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Get-ResultXmlRootAttributes { + param( + [Parameter(Mandatory = $true)] + [string]$XmlText + ) + + $match = [regex]::Match( + $XmlText, + '[^>]*)>', + [System.Text.RegularExpressions.RegexOptions]::IgnoreCase ` + -bor [System.Text.RegularExpressions.RegexOptions]::Singleline + ) + if (-not $match.Success) { + return $null + } + + $attrs = $match.Groups['attrs'].Value + $values = [ordered]@{} + foreach ($name in @('total', 'errors', 'failures', 'not-run')) { + $attrMatch = [regex]::Match( + $attrs, + ('\b{0}="(?\d+)"' -f [regex]::Escape($name)), + [System.Text.RegularExpressions.RegexOptions]::IgnoreCase + ) + if (-not $attrMatch.Success) { + return $null + } + $values[$name] = [int]$attrMatch.Groups['value'].Value + } + + return [pscustomobject]@{ + Total = [int]$values['total'] + Errors = [int]$values['errors'] + Failures = [int]$values['failures'] + Skipped = [int]$values['not-run'] + } +} + +function Test-ResultXmlCloseTag { + param( + [Parameter(Mandatory = $true)] + [string]$XmlText + ) + + return [regex]::IsMatch( + $XmlText, + '\s*$', + [System.Text.RegularExpressions.RegexOptions]::IgnoreCase + ) +} + +$resolvedXmlPath = [System.IO.Path]::GetFullPath($XmlPath) +$deadline = (Get-Date).AddSeconds($StabilizationTimeoutSeconds) +$lastSize = -1L +$stablePolls = 0 +$rawText = $null +$closeTagPresent = $false +$sizeBytes = 0L + +while ((Get-Date) -le $deadline) { + if (Test-Path -LiteralPath $resolvedXmlPath -PathType Leaf) { + $rawText = Get-Content -LiteralPath $resolvedXmlPath -Raw -ErrorAction Stop + $sizeBytes = (Get-Item -LiteralPath $resolvedXmlPath -ErrorAction Stop).Length + $closeTagPresent = Test-ResultXmlCloseTag -XmlText $rawText + if ($closeTagPresent) { + break + } + + if ($sizeBytes -eq $lastSize -and $sizeBytes -gt 0) { + $stablePolls++ + } else { + $stablePolls = 0 + $lastSize = $sizeBytes + } + + if ($stablePolls -ge 2) { + break + } + } + + Start-Sleep -Milliseconds $PollIntervalMilliseconds +} + +if (-not $rawText -and (Test-Path -LiteralPath $resolvedXmlPath -PathType Leaf)) { + $rawText = Get-Content -LiteralPath $resolvedXmlPath -Raw -ErrorAction Stop + $sizeBytes = (Get-Item -LiteralPath $resolvedXmlPath -ErrorAction Stop).Length + $closeTagPresent = Test-ResultXmlCloseTag -XmlText $rawText +} + +$status = 'missing' +$summarySource = $null +$parseError = $null +$total = $null +$failed = $null +$errors = $null +$skipped = $null + +if (-not [string]::IsNullOrWhiteSpace($rawText)) { + try { + [xml]$document = $rawText + $rootNode = $document.'test-results' + if (-not $rootNode) { + throw 'Missing test-results root node.' + } + + $status = 'complete' + $summarySource = 'xml-dom' + $total = [int]$rootNode.total + $failed = [int]$rootNode.failures + $errors = [int]$rootNode.errors + $skipped = [int]$rootNode.'not-run' + } catch { + $parseError = $_.Exception.Message + $rootAttributes = Get-ResultXmlRootAttributes -XmlText $rawText + if ($rootAttributes) { + $status = if ($closeTagPresent) { 'invalid-root-attributes' } else { 'truncated-root' } + $summarySource = 'root-attributes' + $total = [int]$rootAttributes.Total + $failed = [int]$rootAttributes.Failures + $errors = [int]$rootAttributes.Errors + $skipped = [int]$rootAttributes.Skipped + } else { + $status = if ($closeTagPresent) { 'invalid' } else { 'truncated' } + } + } +} + +[pscustomobject]@{ + schema = 'pester-result-xml-summary@v1' + path = $resolvedXmlPath + status = $status + summarySource = $summarySource + closeTagPresent = [bool]$closeTagPresent + sizeBytes = [int64]$sizeBytes + total = $total + failed = $failed + errors = $errors + skipped = $skipped + parseError = $parseError +} diff --git a/tools/Invoke-PesterExecutionPostprocess.ps1 b/tools/Invoke-PesterExecutionPostprocess.ps1 new file mode 100644 index 000000000..a647197b1 --- /dev/null +++ b/tools/Invoke-PesterExecutionPostprocess.ps1 @@ -0,0 +1,153 @@ +param( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string]$ResultsDir, + + [Parameter(Mandatory = $false)] + [ValidateNotNullOrEmpty()] + [string]$JsonSummaryPath = 'pester-summary.json', + + [Parameter(Mandatory = $false)] + [ValidateNotNullOrEmpty()] + [string]$PostprocessReportPath = 'pester-execution-postprocess.json', + + [Parameter(Mandatory = $false)] + [ValidateRange(0, 30)] + [int]$XmlStabilizationTimeoutSeconds = 3, + + [Parameter(Mandatory = $false)] + [ValidateRange(25, 5000)] + [int]$XmlPollIntervalMilliseconds = 200 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Set-ObjectProperty { + param( + [Parameter(Mandatory = $true)]$InputObject, + [Parameter(Mandatory = $true)][string]$Name, + $Value + ) + + $property = $InputObject.PSObject.Properties[$Name] + if ($property) { + $property.Value = $Value + } else { + Add-Member -InputObject $InputObject -Name $Name -MemberType NoteProperty -Value $Value + } +} + +function Read-JsonObject { + param([Parameter(Mandatory = $true)][string]$PathValue) + + if (-not (Test-Path -LiteralPath $PathValue -PathType Leaf)) { + return $null + } + + try { + return (Get-Content -LiteralPath $PathValue -Raw | ConvertFrom-Json -ErrorAction Stop) + } catch { + return $null + } +} + +$resolvedResultsDir = [System.IO.Path]::GetFullPath($ResultsDir) +if (-not (Test-Path -LiteralPath $resolvedResultsDir -PathType Container)) { + New-Item -ItemType Directory -Path $resolvedResultsDir -Force | Out-Null +} + +$summaryPath = Join-Path $resolvedResultsDir $JsonSummaryPath +$reportPath = Join-Path $resolvedResultsDir $PostprocessReportPath +$xmlPath = Join-Path $resolvedResultsDir 'pester-results.xml' +$xmlSummaryToolPath = Join-Path $PSScriptRoot 'Get-PesterResultXmlSummary.ps1' +if (-not (Test-Path -LiteralPath $xmlSummaryToolPath -PathType Leaf)) { + throw "XML summary tool not found: $xmlSummaryToolPath" +} + +$existingSummary = Read-JsonObject -PathValue $summaryPath +$summaryPresentBefore = [bool]$existingSummary +$xmlSummary = & $xmlSummaryToolPath -XmlPath $xmlPath -StabilizationTimeoutSeconds $XmlStabilizationTimeoutSeconds -PollIntervalMilliseconds $XmlPollIntervalMilliseconds +if (-not $xmlSummary) { + throw 'Get-PesterResultXmlSummary.ps1 returned no result.' +} + +$postprocessStatus = switch ([string]$xmlSummary.status) { + 'complete' { 'complete'; break } + 'truncated-root' { 'results-xml-truncated'; break } + 'truncated' { 'results-xml-truncated'; break } + 'invalid-root-attributes' { 'invalid-results-xml'; break } + 'invalid' { 'invalid-results-xml'; break } + 'missing' { 'missing-results-xml'; break } + default { 'seam-defect'; break } +} + +$summaryWritten = $false +$summaryPayload = if ($existingSummary) { $existingSummary } else { [pscustomobject]@{} } +$passed = $null +if ($null -ne $xmlSummary.total -and $null -ne $xmlSummary.failed -and $null -ne $xmlSummary.errors) { + $passed = [Math]::Max(0, ([int]$xmlSummary.total - [int]$xmlSummary.failed - [int]$xmlSummary.errors)) +} + +$canWriteSummary = $postprocessStatus -in @('complete', 'results-xml-truncated', 'invalid-results-xml') +if ($canWriteSummary) { + Set-ObjectProperty -InputObject $summaryPayload -Name 'total' -Value $xmlSummary.total + Set-ObjectProperty -InputObject $summaryPayload -Name 'passed' -Value $passed + Set-ObjectProperty -InputObject $summaryPayload -Name 'failed' -Value $xmlSummary.failed + Set-ObjectProperty -InputObject $summaryPayload -Name 'errors' -Value $xmlSummary.errors + Set-ObjectProperty -InputObject $summaryPayload -Name 'skipped' -Value $xmlSummary.skipped + if (-not $summaryPayload.PSObject.Properties['timestamp']) { + Set-ObjectProperty -InputObject $summaryPayload -Name 'timestamp' -Value ([DateTime]::UtcNow.ToString('o')) + } + Set-ObjectProperty -InputObject $summaryPayload -Name 'resultsXmlStatus' -Value ([string]$xmlSummary.status) + Set-ObjectProperty -InputObject $summaryPayload -Name 'resultsXmlSummarySource' -Value $xmlSummary.summarySource + Set-ObjectProperty -InputObject $summaryPayload -Name 'resultsXmlCloseTagPresent' -Value ([bool]$xmlSummary.closeTagPresent) + Set-ObjectProperty -InputObject $summaryPayload -Name 'resultsXmlSizeBytes' -Value ([int64]$xmlSummary.sizeBytes) + if (-not [string]::IsNullOrWhiteSpace([string]$xmlSummary.parseError)) { + Set-ObjectProperty -InputObject $summaryPayload -Name 'resultsXmlParseError' -Value ([string]$xmlSummary.parseError) + } + Set-ObjectProperty -InputObject $summaryPayload -Name 'executionPostprocessStatus' -Value $postprocessStatus + Set-ObjectProperty -InputObject $summaryPayload -Name 'executionPostprocessSchema' -Value 'pester-execution-postprocess@v1' + $summaryPayload | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $summaryPath -Encoding UTF8 + $summaryWritten = $true +} + +$report = [ordered]@{ + schema = 'pester-execution-postprocess@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + resultsDir = $resolvedResultsDir + xmlPath = $xmlPath + summaryPath = $summaryPath + summaryPresentBefore = $summaryPresentBefore + summaryWritten = $summaryWritten + status = $postprocessStatus + resultsXmlStatus = [string]$xmlSummary.status + resultsXmlSummarySource = $xmlSummary.summarySource + resultsXmlCloseTagPresent = [bool]$xmlSummary.closeTagPresent + resultsXmlSizeBytes = [int64]$xmlSummary.sizeBytes + total = $xmlSummary.total + passed = $passed + failed = $xmlSummary.failed + errors = $xmlSummary.errors + skipped = $xmlSummary.skipped + parseError = [string]$xmlSummary.parseError +} +$report | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $reportPath -Encoding UTF8 + +if ($env:GITHUB_OUTPUT) { + "status=$postprocessStatus" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "results_xml_status=$($xmlSummary.status)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "summary_written=$summaryWritten" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "report_path=$reportPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 +} + +Write-Host '### Pester execution postprocess' -ForegroundColor Cyan +Write-Host ("status : {0}" -f $postprocessStatus) +Write-Host ("xmlStatus : {0}" -f $xmlSummary.status) +Write-Host ("summaryWrite : {0}" -f $summaryWritten) +Write-Host ("report : {0}" -f $reportPath) +if ($summaryWritten) { + Write-Host ("summary : {0}" -f $summaryPath) +} + +exit 0 diff --git a/tools/Run-PesterExecutionOnly.Local.ps1 b/tools/Run-PesterExecutionOnly.Local.ps1 new file mode 100644 index 000000000..8cb2bca14 --- /dev/null +++ b/tools/Run-PesterExecutionOnly.Local.ps1 @@ -0,0 +1,604 @@ +#Requires -Version 7.0 +[CmdletBinding()] +param( + [string]$TestsPath = 'tests', + [string]$ResultsPath = 'tests/results', + [string]$ContextReceiptPath, + [string]$ReadinessReceiptPath, + [string]$SelectionReceiptPath, + [ValidateSet('auto', 'include', 'exclude')] + [string]$IntegrationMode = 'exclude', + [string[]]$IncludePatterns, + [string]$BasePath, + [string]$HeadPath, + [switch]$EmitFailuresJsonAlways, + [double]$TimeoutSeconds = 0, + [switch]$DetectLeaks, + [switch]$FailOnLeaks, + [switch]$KillLeaks, + [double]$LeakGraceSeconds = 3, + [switch]$CleanLabVIEWBefore, + [switch]$CleanAfter, + [switch]$TrackArtifacts, + [string]$SessionLockGroup = 'pester-selfhosted', + [string]$SessionLockRoot, + [int]$SessionLockQueueWaitSeconds = 15, + [int]$SessionLockQueueMaxAttempts = 40, + [int]$SessionLockStaleSeconds = 300, + [int]$SessionHeartbeatSeconds = 15 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function ConvertTo-PortablePath { + param([string]$PathValue) + if ([string]::IsNullOrWhiteSpace($PathValue)) { + return $null + } + return ($PathValue -replace '\\', '/') +} + +function Resolve-RepoRoot { + return (Resolve-Path (Join-Path $PSScriptRoot '..')).Path +} + +function Resolve-OutputPath { + param( + [string]$RepoRoot, + [string]$PathValue + ) + if ([string]::IsNullOrWhiteSpace($PathValue)) { + return $null + } + if ([System.IO.Path]::IsPathRooted($PathValue)) { + return [System.IO.Path]::GetFullPath($PathValue) + } + return [System.IO.Path]::GetFullPath((Join-Path $RepoRoot $PathValue)) +} + +function Read-JsonFile { + param([Parameter(Mandatory = $true)][string]$PathValue) + if (-not (Test-Path -LiteralPath $PathValue -PathType Leaf)) { + throw "JSON file not found: $PathValue" + } + return (Get-Content -LiteralPath $PathValue -Raw | ConvertFrom-Json -ErrorAction Stop) +} + +function ConvertTo-Bool { + param($Value) + if ($Value -is [bool]) { + return $Value + } + if ($null -eq $Value) { + return $false + } + $text = [string]$Value + return $text.Trim().ToLowerInvariant() -in @('1', 'true', 'yes', 'on') +} + +function ConvertTo-NullableDouble { + param($Value) + if ($null -eq $Value -or $Value -eq '') { + return $null + } + $parsed = 0.0 + if ([double]::TryParse([string]$Value, [ref]$parsed)) { + return $parsed + } + return $null +} + +function Resolve-RepositorySlug { + param([string]$RepoRoot) + try { + $remoteUrl = git -C $RepoRoot remote get-url origin 2>$null + if ($remoteUrl -match 'github\.com[:/](?[^/]+/[^/.]+)') { + return $matches.slug + } + } catch {} + return Split-Path -Leaf $RepoRoot +} + +function Validate-ContextReceipt { + param([string]$ReceiptPath) + if ([string]::IsNullOrWhiteSpace($ReceiptPath)) { + return $null + } + $receipt = Read-JsonFile -PathValue $ReceiptPath + if ($receipt.schema -ne 'pester-context-receipt@v1') { + throw "Unexpected context receipt schema: $($receipt.schema)" + } + if ($receipt.status -ne 'ready') { + throw "Context receipt status is not ready: $($receipt.status)" + } + return $receipt +} + +function Validate-ReadinessReceipt { + param([string]$ReceiptPath) + if ([string]::IsNullOrWhiteSpace($ReceiptPath)) { + return $null + } + $receipt = Read-JsonFile -PathValue $ReceiptPath + if ($receipt.schema -ne 'pester-selfhosted-readiness-receipt@v1') { + throw "Unexpected readiness receipt schema: $($receipt.schema)" + } + if ($receipt.status -ne 'ready') { + throw "Readiness receipt status is not ready: $($receipt.status)" + } + $freshnessWindowSeconds = 900 + if ($receipt.PSObject.Properties.Name -contains 'freshnessWindowSeconds') { + $freshnessWindowSeconds = [int]$receipt.freshnessWindowSeconds + } + $generatedAtUtc = [DateTime]::Parse($receipt.generatedAtUtc).ToUniversalTime() + $ageSeconds = [math]::Floor(([DateTime]::UtcNow - $generatedAtUtc).TotalSeconds) + if ($ageSeconds -gt $freshnessWindowSeconds) { + throw "Readiness receipt stale: age ${ageSeconds}s exceeds freshness window ${freshnessWindowSeconds}s" + } + return $receipt +} + +function Validate-SelectionReceipt { + param([string]$ReceiptPath) + if ([string]::IsNullOrWhiteSpace($ReceiptPath)) { + return $null + } + $receipt = Read-JsonFile -PathValue $ReceiptPath + if ($receipt.schema -ne 'pester-selection-receipt@v1') { + throw "Unexpected selection receipt schema: $($receipt.schema)" + } + if ($receipt.status -ne 'ready') { + throw "Selection receipt status is not ready: $($receipt.status)" + } + return $receipt +} + +function Invoke-RunnerUnblockGuardLocal { + param( + [Parameter(Mandatory = $true)][string]$SnapshotPath, + [bool]$Cleanup = $false, + [string[]]$ProcessNames = @('LabVIEW', 'LVCompare') + ) + $snapshotDir = Split-Path -Parent $SnapshotPath + if ($snapshotDir -and -not (Test-Path -LiteralPath $snapshotDir -PathType Container)) { + New-Item -ItemType Directory -Path $snapshotDir -Force | Out-Null + } + $procs = @( + Get-Process -ErrorAction SilentlyContinue | + Where-Object { $_.ProcessName -in $ProcessNames } | + Select-Object ProcessName, Id, SessionId, StartTime + ) + $jobs = @(Get-Job -ErrorAction SilentlyContinue | Select-Object Id, Name, State, HasMoreData) + $snapshot = [ordered]@{ + processes = $procs + jobs = $jobs + cleanupPerformed = $false + } + if ($Cleanup) { + $stopped = @() + foreach ($name in $ProcessNames) { + Get-Process -Name $name -ErrorAction SilentlyContinue | ForEach-Object { + Stop-Process -Id $_.Id -Force -ErrorAction SilentlyContinue + $stopped += [ordered]@{ + name = $name + id = $_.Id + } + } + } + $snapshot.cleanupPerformed = $true + $snapshot.cleanup = $stopped + } + $snapshot | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $SnapshotPath -Encoding UTF8 +} + +function Invoke-PrepareFixturesLocal { + param( + [Parameter(Mandatory = $true)][string]$RepoRoot, + [string]$BaseSourcePath, + [string]$HeadSourcePath + ) + + $resolvedBase = if ([string]::IsNullOrWhiteSpace($BaseSourcePath)) { + if ($env:LV_BASE_VI) { $env:LV_BASE_VI } else { Join-Path $RepoRoot 'VI1.vi' } + } else { + $BaseSourcePath + } + $resolvedHead = if ([string]::IsNullOrWhiteSpace($HeadSourcePath)) { + if ($env:LV_HEAD_VI) { $env:LV_HEAD_VI } else { Join-Path $RepoRoot 'VI2.vi' } + } else { + $HeadSourcePath + } + + if (-not (Test-Path -LiteralPath $resolvedBase -PathType Leaf)) { + throw "Base VI not found: $resolvedBase" + } + if (-not (Test-Path -LiteralPath $resolvedHead -PathType Leaf)) { + throw "Head VI not found: $resolvedHead" + } + + $tmpBase = if ($env:RUNNER_TEMP) { $env:RUNNER_TEMP } else { [System.IO.Path]::GetTempPath() } + $tmpDir = Join-Path $tmpBase ("fixtures-" + [guid]::NewGuid().ToString()) + New-Item -ItemType Directory -Path $tmpDir -Force | Out-Null + $baseCopy = Join-Path $tmpDir 'base.vi' + $headCopy = Join-Path $tmpDir 'head.vi' + Copy-Item -LiteralPath $resolvedBase -Destination $baseCopy -Force + Copy-Item -LiteralPath $resolvedHead -Destination $headCopy -Force + + return [ordered]@{ + tempDir = $tmpDir + base = $baseCopy + head = $headCopy + sourceBase = $resolvedBase + sourceHead = $resolvedHead + } +} + +function Resolve-LVComparePath { + if ($env:LVCOMPARE_PATH -and (Test-Path -LiteralPath $env:LVCOMPARE_PATH -PathType Leaf)) { + return (Resolve-Path -LiteralPath $env:LVCOMPARE_PATH).Path + } + $canonical = 'C:\Program Files\National Instruments\Shared\LabVIEW Compare\LVCompare.exe' + if (Test-Path -LiteralPath $canonical -PathType Leaf) { + return $canonical + } + return $null +} + +function Write-JsonFile { + param( + [Parameter(Mandatory = $true)][string]$PathValue, + [Parameter(Mandatory = $true)]$Payload + ) + $dir = Split-Path -Parent $PathValue + if ($dir -and -not (Test-Path -LiteralPath $dir -PathType Container)) { + New-Item -ItemType Directory -Path $dir -Force | Out-Null + } + $Payload | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $PathValue -Encoding UTF8 +} + +$repoRoot = Resolve-RepoRoot +$resolvedResultsPath = Resolve-OutputPath -RepoRoot $repoRoot -PathValue $ResultsPath +$resolvedContextReceiptPath = Resolve-OutputPath -RepoRoot $repoRoot -PathValue $ContextReceiptPath +$resolvedReadinessReceiptPath = Resolve-OutputPath -RepoRoot $repoRoot -PathValue $ReadinessReceiptPath +$resolvedSelectionReceiptPath = Resolve-OutputPath -RepoRoot $repoRoot -PathValue $SelectionReceiptPath +$resolvedSessionLockRoot = if ([string]::IsNullOrWhiteSpace($SessionLockRoot)) { + Join-Path $resolvedResultsPath '_session_lock' +} else { + Resolve-OutputPath -RepoRoot $repoRoot -PathValue $SessionLockRoot +} + +$contextReceipt = $null +$readinessReceipt = $null +$selectionReceipt = $null +$preparedFixtures = $null +$dispatcherExitCode = -1 +$postprocessStatus = 'seam-defect' +$postprocessReportPath = $null +$resultsXmlStatus = $null +$executionStatus = 'seam-defect' +$executionJobResult = 'failure' +$heartbeatJob = $null +$lvComparePath = $null +$dotnetReady = $false +$sessionLockPath = $null +$sessionLockId = $null + +Push-Location $repoRoot +try { + New-Item -ItemType Directory -Path $resolvedResultsPath -Force | Out-Null + New-Item -ItemType Directory -Path $resolvedSessionLockRoot -Force | Out-Null + + $contextReceipt = Validate-ContextReceipt -ReceiptPath $resolvedContextReceiptPath + $readinessReceipt = Validate-ReadinessReceipt -ReceiptPath $resolvedReadinessReceiptPath + $selectionReceipt = Validate-SelectionReceipt -ReceiptPath $resolvedSelectionReceiptPath + + $repository = if ($contextReceipt) { [string]$contextReceipt.repository } else { Resolve-RepositorySlug -RepoRoot $repoRoot } + $standingPriorityIssue = if ($contextReceipt) { $contextReceipt.standingPriority.issueNumber } else { $null } + + $effectiveIntegrationMode = if ($selectionReceipt -and -not $PSBoundParameters.ContainsKey('IntegrationMode')) { + [string]$selectionReceipt.selection.integrationMode + } else { + $IntegrationMode + } + $effectiveIncludePatterns = if ($selectionReceipt -and -not $PSBoundParameters.ContainsKey('IncludePatterns')) { + @($selectionReceipt.selection.includePatterns) + } else { + @($IncludePatterns) + } + $fixtureRequired = if ($selectionReceipt) { + ConvertTo-Bool $selectionReceipt.selection.fixtureRequired + } else { + $false + } + if ($PSBoundParameters.ContainsKey('BasePath') -or $PSBoundParameters.ContainsKey('HeadPath') -or $env:LV_BASE_VI -or $env:LV_HEAD_VI) { + $fixtureRequired = $true + } + + $effectiveTimeoutSeconds = if ($selectionReceipt -and -not $PSBoundParameters.ContainsKey('TimeoutSeconds')) { + ConvertTo-NullableDouble $selectionReceipt.dispatcherProfile.timeoutSeconds + } else { + $TimeoutSeconds + } + if ($null -eq $effectiveTimeoutSeconds) { + $effectiveTimeoutSeconds = 0 + } + + $effectiveEmitFailuresJsonAlways = if ($selectionReceipt -and -not $PSBoundParameters.ContainsKey('EmitFailuresJsonAlways')) { + ConvertTo-Bool $selectionReceipt.dispatcherProfile.emitFailuresJsonAlways + } else { + $EmitFailuresJsonAlways.IsPresent + } + $effectiveDetectLeaks = if ($selectionReceipt -and -not $PSBoundParameters.ContainsKey('DetectLeaks')) { + ConvertTo-Bool $selectionReceipt.dispatcherProfile.detectLeaks + } else { + $DetectLeaks.IsPresent + } + $effectiveFailOnLeaks = if ($selectionReceipt -and -not $PSBoundParameters.ContainsKey('FailOnLeaks')) { + ConvertTo-Bool $selectionReceipt.dispatcherProfile.failOnLeaks + } else { + $FailOnLeaks.IsPresent + } + $effectiveKillLeaks = if ($selectionReceipt -and -not $PSBoundParameters.ContainsKey('KillLeaks')) { + ConvertTo-Bool $selectionReceipt.dispatcherProfile.killLeaks + } else { + $KillLeaks.IsPresent + } + $effectiveLeakGraceSeconds = if ($selectionReceipt -and -not $PSBoundParameters.ContainsKey('LeakGraceSeconds')) { + ConvertTo-NullableDouble $selectionReceipt.dispatcherProfile.leakGraceSeconds + } else { + $LeakGraceSeconds + } + if ($null -eq $effectiveLeakGraceSeconds) { + $effectiveLeakGraceSeconds = 3 + } + $effectiveCleanBefore = if ($selectionReceipt -and -not $PSBoundParameters.ContainsKey('CleanLabVIEWBefore')) { + ConvertTo-Bool $selectionReceipt.dispatcherProfile.cleanLabVIEWBefore + } else { + $CleanLabVIEWBefore.IsPresent + } + $effectiveCleanAfter = if ($selectionReceipt -and -not $PSBoundParameters.ContainsKey('CleanAfter')) { + ConvertTo-Bool $selectionReceipt.dispatcherProfile.cleanAfter + } else { + $CleanAfter.IsPresent + } + $effectiveTrackArtifacts = if ($selectionReceipt -and -not $PSBoundParameters.ContainsKey('TrackArtifacts')) { + ConvertTo-Bool $selectionReceipt.dispatcherProfile.trackArtifacts + } else { + $TrackArtifacts.IsPresent + } + + if (-not (Get-Command dotnet -ErrorAction SilentlyContinue)) { + throw '.NET host toolchain is not available. Install dotnet before running the local execution harness.' + } + $dotnetReady = $true + + $lvComparePath = Resolve-LVComparePath + $requiresLabVIEWRuntime = $fixtureRequired -or $effectiveIntegrationMode -eq 'include' + if ($requiresLabVIEWRuntime -and -not $lvComparePath) { + throw 'LVCompare is not available at LVCOMPARE_PATH or the canonical install path.' + } + if ($lvComparePath) { + $env:LVCOMPARE_PATH = $lvComparePath + } + + $preSnapshotPath = Join-Path $resolvedResultsPath 'runner-unblock-snapshot-pre.json' + Invoke-RunnerUnblockGuardLocal -SnapshotPath $preSnapshotPath -Cleanup:$effectiveCleanBefore + + if ($fixtureRequired) { + $preparedFixtures = Invoke-PrepareFixturesLocal -RepoRoot $repoRoot -BaseSourcePath $BasePath -HeadSourcePath $HeadPath + $env:LV_BASE_VI = $preparedFixtures.base + $env:LV_HEAD_VI = $preparedFixtures.head + } + + $env:LOCAL_DISPATCHER = '1' + $env:DISABLE_STEP_SUMMARY = '1' + if ($effectiveCleanBefore) { $env:CLEAN_LABVIEW = '1' } + if ($effectiveCleanAfter) { $env:CLEAN_AFTER = '1' } + if ($effectiveTrackArtifacts) { $env:SCAN_ARTIFACTS = '1' } + if ($effectiveDetectLeaks) { $env:DETECT_LEAKS = '1' } + if ($effectiveFailOnLeaks) { $env:FAIL_ON_LEAKS = '1' } + if ($effectiveKillLeaks) { $env:KILL_LEAKS = '1' } + $env:LEAK_GRACE_SECONDS = [string]$effectiveLeakGraceSeconds + + $lockScript = Join-Path $repoRoot 'tools/Session-Lock.ps1' + pwsh -NoLogo -NoProfile -File $lockScript -Action Acquire -Group $SessionLockGroup -LockRoot $resolvedSessionLockRoot -QueueWaitSeconds $SessionLockQueueWaitSeconds -QueueMaxAttempts $SessionLockQueueMaxAttempts -StaleSeconds $SessionLockStaleSeconds -HeartbeatSeconds $SessionHeartbeatSeconds | Out-Null + if ($LASTEXITCODE -ne 0) { + throw "Failed to acquire session lock for group '$SessionLockGroup' under '$resolvedSessionLockRoot' (exit $LASTEXITCODE)." + } + $sessionLockPath = Join-Path (Join-Path $resolvedSessionLockRoot $SessionLockGroup) 'lock.json' + if (-not (Test-Path -LiteralPath $sessionLockPath -PathType Leaf)) { + throw "Session lock file not found after acquire: $sessionLockPath" + } + $sessionLockRecord = Read-JsonFile -PathValue $sessionLockPath + if (-not $sessionLockRecord.lockId) { + throw "Session lock record missing lockId: $sessionLockPath" + } + $sessionLockId = [string]$sessionLockRecord.lockId + $env:SESSION_LOCK_ID = $sessionLockId + $env:SESSION_LOCK_GROUP = $SessionLockGroup + $env:SESSION_LOCK_PATH = $sessionLockPath + $env:SESSION_LOCK_ROOT = $resolvedSessionLockRoot + $env:SESSION_HEARTBEAT_SECONDS = [string]$SessionHeartbeatSeconds + + $heartbeatJob = Start-ThreadJob -ScriptBlock { + param($ScriptPath, $Seconds, $LockGroup, $LockRootPath, $LockId) + $env:SESSION_LOCK_ID = $LockId + $env:SESSION_LOCK_GROUP = $LockGroup + $env:SESSION_LOCK_ROOT = $LockRootPath + while ($true) { + pwsh -NoLogo -NoProfile -File $ScriptPath -Action Heartbeat -Group $LockGroup -LockRoot $LockRootPath | Out-Null + Start-Sleep -Seconds $Seconds + } + } -ArgumentList $lockScript, $SessionHeartbeatSeconds, $SessionLockGroup, $resolvedSessionLockRoot, $sessionLockId + + $invokeParams = @{ + TestsPath = $TestsPath + ResultsPath = $resolvedResultsPath + IntegrationMode = $effectiveIntegrationMode + } + $effectiveIncludePatternsList = @($effectiveIncludePatterns | Where-Object { -not [string]::IsNullOrWhiteSpace([string]$_) }) + if ($effectiveIncludePatternsList.Count -gt 0) { + $invokeParams.IncludePatterns = $effectiveIncludePatternsList + } + if ($effectiveEmitFailuresJsonAlways) { $invokeParams.EmitFailuresJsonAlways = $true } + if ($effectiveTimeoutSeconds -gt 0) { $invokeParams.TimeoutSeconds = $effectiveTimeoutSeconds } + if ($effectiveDetectLeaks) { $invokeParams.DetectLeaks = $true } + if ($effectiveFailOnLeaks) { $invokeParams.FailOnLeaks = $true } + if ($effectiveKillLeaks) { $invokeParams.KillLeaks = $true } + if ($effectiveLeakGraceSeconds -gt 0) { $invokeParams.LeakGraceSeconds = $effectiveLeakGraceSeconds } + if ($effectiveCleanBefore) { $invokeParams.CleanLabVIEW = $true } + if ($effectiveCleanAfter) { $invokeParams.CleanAfter = $true } + if ($effectiveTrackArtifacts) { $invokeParams.TrackArtifacts = $true } + + $dispatcherPath = Join-Path $repoRoot 'Invoke-PesterTests.ps1' + $logPath = Join-Path $resolvedResultsPath 'pester-dispatcher.log' + if (Test-Path -LiteralPath $logPath) { + Remove-Item -LiteralPath $logPath -Force + } + + try { + $previousErrorActionPreference = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + & $dispatcherPath @invokeParams 2>&1 | Tee-Object -FilePath $logPath + $dispatcherExitCode = if ($null -ne $LASTEXITCODE) { [int]$LASTEXITCODE } else { 0 } + } catch { + $_ | Out-String | Tee-Object -FilePath $logPath -Append | Write-Host + $dispatcherExitCode = if ($null -ne $LASTEXITCODE -and [int]$LASTEXITCODE -ne 0) { [int]$LASTEXITCODE } else { 1 } + } finally { + $ErrorActionPreference = $previousErrorActionPreference + if ($heartbeatJob) { + Stop-Job -Id $heartbeatJob.Id -ErrorAction SilentlyContinue | Out-Null + Remove-Job -Id $heartbeatJob.Id -Force -ErrorAction SilentlyContinue | Out-Null + } + if ($sessionLockId) { + $env:SESSION_LOCK_ID = $sessionLockId + $env:SESSION_LOCK_GROUP = $SessionLockGroup + $env:SESSION_LOCK_ROOT = $resolvedSessionLockRoot + pwsh -NoLogo -NoProfile -File $lockScript -Action Heartbeat -Group $SessionLockGroup -LockRoot $resolvedSessionLockRoot | Out-Null + pwsh -NoLogo -NoProfile -File $lockScript -Action Release -Group $SessionLockGroup -LockRoot $resolvedSessionLockRoot | Out-Null + } + } + + $postprocessToolPath = Join-Path $repoRoot 'tools/Invoke-PesterExecutionPostprocess.ps1' + if (-not (Test-Path -LiteralPath $postprocessToolPath -PathType Leaf)) { + throw "Postprocess tool not found: $postprocessToolPath" + } + & $postprocessToolPath -ResultsDir $resolvedResultsPath | Out-Host + if ($LASTEXITCODE -ne 0) { + throw "Pester execution postprocess failed with exit code $LASTEXITCODE." + } + $postprocessReportPath = Join-Path $resolvedResultsPath 'pester-execution-postprocess.json' + if (Test-Path -LiteralPath $postprocessReportPath -PathType Leaf) { + $postprocessReport = Read-JsonFile -PathValue $postprocessReportPath + $postprocessStatus = [string]$postprocessReport.status + $resultsXmlStatus = [string]$postprocessReport.resultsXmlStatus + } + + $summaryPath = Join-Path $resolvedResultsPath 'pester-summary.json' + $summaryPresent = Test-Path -LiteralPath $summaryPath + if ($postprocessStatus -in @('results-xml-truncated', 'invalid-results-xml', 'missing-results-xml')) { + $executionStatus = $postprocessStatus + $executionJobResult = 'failure' + } elseif ($summaryPresent -and $dispatcherExitCode -eq 0) { + $executionStatus = 'completed' + $executionJobResult = 'success' + } elseif ($summaryPresent) { + $executionStatus = 'test-failures' + $executionJobResult = 'failure' + } elseif ($dispatcherExitCode -ne 0) { + $executionStatus = 'seam-defect' + $executionJobResult = 'failure' + } + + $receipt = [ordered]@{ + schema = 'pester-execution-receipt@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + source = 'local-harness' + repository = $repository + contextStatus = if ($contextReceipt) { [string]$contextReceipt.status } else { 'local-ready' } + contextReceiptPath = ConvertTo-PortablePath $resolvedContextReceiptPath + contextReceiptPresent = [bool]$contextReceipt + standingPriorityIssue = $standingPriorityIssue + readinessStatus = if ($readinessReceipt) { [string]$readinessReceipt.status } else { 'local-ready' } + readinessReceiptPath = ConvertTo-PortablePath $resolvedReadinessReceiptPath + readinessReceiptPresent = [bool]$readinessReceipt + selectionStatus = if ($selectionReceipt) { [string]$selectionReceipt.status } else { 'local-ready' } + selectionReceiptPath = ConvertTo-PortablePath $resolvedSelectionReceiptPath + selectionReceiptPresent = [bool]$selectionReceipt + selectionIntegrationMode = $effectiveIntegrationMode + selectionFixtureRequired = $fixtureRequired + includePatterns = @($effectiveIncludePatternsList) + dispatcherExitCode = $dispatcherExitCode + postprocessStatus = $postprocessStatus + resultsXmlStatus = $resultsXmlStatus + executionJobResult = $executionJobResult + summaryPresent = $summaryPresent + status = $executionStatus + rawArtifactName = 'pester-run-raw-local' + localHarness = [ordered]@{ + dotnetReady = $dotnetReady + lvComparePath = ConvertTo-PortablePath $lvComparePath + fixtureBase = if ($preparedFixtures) { ConvertTo-PortablePath $preparedFixtures.base } else { $null } + fixtureHead = if ($preparedFixtures) { ConvertTo-PortablePath $preparedFixtures.head } else { $null } + timeoutSeconds = $effectiveTimeoutSeconds + emitFailuresJsonAlways = $effectiveEmitFailuresJsonAlways + detectLeaks = $effectiveDetectLeaks + failOnLeaks = $effectiveFailOnLeaks + killLeaks = $effectiveKillLeaks + leakGraceSeconds = $effectiveLeakGraceSeconds + cleanLabVIEWBefore = $effectiveCleanBefore + cleanAfter = $effectiveCleanAfter + trackArtifacts = $effectiveTrackArtifacts + sessionLockGroup = $SessionLockGroup + sessionLockRoot = ConvertTo-PortablePath $resolvedSessionLockRoot + sessionLockPath = ConvertTo-PortablePath $sessionLockPath + postprocessReportPath = ConvertTo-PortablePath $postprocessReportPath + preSnapshotPath = ConvertTo-PortablePath $preSnapshotPath + dispatcherLogPath = ConvertTo-PortablePath $logPath + } + } + + $receiptPath = Join-Path $resolvedResultsPath 'pester-run-receipt.json' + $contractReceiptPath = Join-Path $resolvedResultsPath 'pester-execution-contract' 'pester-run-receipt.json' + Write-JsonFile -PathValue $receiptPath -Payload $receipt + Write-JsonFile -PathValue $contractReceiptPath -Payload $receipt + + Write-Host '### Local Pester execution harness' -ForegroundColor Cyan + Write-Host ("status : {0}" -f $executionStatus) + Write-Host ("exitCode : {0}" -f $dispatcherExitCode) + Write-Host ("summary : {0}" -f $summaryPresent) + Write-Host ("receipt : {0}" -f $receiptPath) + Write-Host ("contract : {0}" -f $contractReceiptPath) +} +finally { + if ($preparedFixtures -and $preparedFixtures.tempDir -and (Test-Path -LiteralPath $preparedFixtures.tempDir -PathType Container)) { + Remove-Item -LiteralPath $preparedFixtures.tempDir -Recurse -Force -ErrorAction SilentlyContinue + } + foreach ($envName in @( + 'LOCAL_DISPATCHER', + 'DISABLE_STEP_SUMMARY', + 'CLEAN_LABVIEW', + 'CLEAN_AFTER', + 'SCAN_ARTIFACTS', + 'DETECT_LEAKS', + 'FAIL_ON_LEAKS', + 'KILL_LEAKS', + 'LEAK_GRACE_SECONDS', + 'LV_BASE_VI', + 'LV_HEAD_VI', + 'SESSION_LOCK_ID', + 'SESSION_LOCK_GROUP', + 'SESSION_LOCK_PATH', + 'SESSION_LOCK_ROOT', + 'SESSION_HEARTBEAT_SECONDS' + )) { + Remove-Item "Env:$envName" -ErrorAction SilentlyContinue + } + Pop-Location +} + +if ($executionStatus -ne 'completed') { + exit ([Math]::Max($dispatcherExitCode, 1)) +} +exit 0 diff --git a/tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs b/tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs new file mode 100644 index 000000000..4f71a2154 --- /dev/null +++ b/tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs @@ -0,0 +1,63 @@ +#!/usr/bin/env node + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { readFileSync } from 'node:fs'; + +const repoRoot = process.cwd(); + +function readRepoFile(relativePath) { + return readFileSync(path.join(repoRoot, relativePath), 'utf8'); +} + +test('package.json exposes the local execution harness as a first-class entrypoint', () => { + const packageJson = JSON.parse(readRepoFile('package.json')); + assert.equal( + packageJson.scripts['tests:execution:local'], + 'pwsh -NoLogo -NoProfile -File tools/Run-PesterExecutionOnly.Local.ps1' + ); +}); + +test('local execution harness owns lock lifecycle, preflight, dispatch, and receipt generation', () => { + const harness = readRepoFile('tools/Run-PesterExecutionOnly.Local.ps1'); + + assert.match(harness, /\[string\]\$SessionLockRoot/); + assert.match(harness, /\$resolvedSessionLockRoot = if \(\[string\]::IsNullOrWhiteSpace\(\$SessionLockRoot\)\)/); + assert.match(harness, /-Action Acquire -Group \$SessionLockGroup -LockRoot \$resolvedSessionLockRoot/); + assert.match(harness, /-Action Release/); + assert.match(harness, /SESSION_LOCK_ROOT = \$resolvedSessionLockRoot/); + assert.match(harness, /Invoke-RunnerUnblockGuardLocal/); + assert.match(harness, /Invoke-PrepareFixturesLocal/); + assert.match(harness, /Get-Command dotnet/); + assert.match(harness, /Resolve-LVComparePath/); + assert.match(harness, /Invoke-PesterTests\.ps1/); + assert.match(harness, /Invoke-PesterExecutionPostprocess\.ps1/); + assert.match(harness, /pester-execution-receipt@v1/); + assert.match(harness, /pester-execution-contract/); + assert.match(harness, /source = 'local-harness'/); + assert.match(harness, /results-xml-truncated/); + assert.match(harness, /summaryPresent/); + assert.match(harness, /sessionLockRoot = ConvertTo-PortablePath \$resolvedSessionLockRoot/); +}); + +test('knowledgebase documents the local harness as the workflow-shell-free execution entrypoint', () => { + const doc = readRepoFile('docs/knowledgebase/Pester-Service-Model.md'); + + assert.match(doc, /Run-PesterExecutionOnly\.Local\.ps1/); + assert.match(doc, /without the workflow shell/i); + assert.match(doc, /lock,\s+LV guard,\s+fixture prep,\s+dispatcher profile,\s+dispatch,\s+execution postprocess,\s+and local execution receipt/i); +}); + +test('execution-layer assurance packet traces the local harness in the SRS, RTM, and test plan', () => { + const srs = readRepoFile('docs/requirements-pester-service-model-srs.md'); + const rtm = readRepoFile('docs/rtm-pester-service-model.csv'); + const plan = readRepoFile('docs/testing/pester-service-model-test-plan.md'); + + assert.match(srs, /Run-PesterExecutionOnly\.Local\.ps1/); + assert.match(srs, /mirrors that slice locally without the workflow shell/i); + assert.match(rtm, /tools\/Run-PesterExecutionOnly\.Local\.ps1/); + assert.match(rtm, /pester-service-model-local-harness-contract\.test\.mjs/); + assert.match(plan, /Run-PesterExecutionOnly\.Local\.ps1/); + assert.match(plan, /Local harness contract tests pass/i); +}); diff --git a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs index 4833072d5..796d74dfb 100644 --- a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs +++ b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs @@ -117,10 +117,19 @@ test('pester run is execution-only and validates context, readiness, and selecti assert.match(workflow, /pester-selection\.json/); assert.match(workflow, /selection-blocked/); assert.match(workflow, /Run Pester tests via local dispatcher/); + assert.match(workflow, /Postprocess execution results/); + assert.match(workflow, /Invoke-PesterExecutionPostprocess\.ps1/); + assert.match(workflow, /\$ErrorActionPreference = 'Continue'/); + assert.match(workflow, /"exit_code=\$exitCode"/); + assert.match(workflow, /Write-Warning \("Dispatcher exited with code \{0\}" -f \$exitCode\)/); + assert.match(workflow, /results-xml-truncated/); + assert.match(workflow, /invalid-results-xml/); + assert.match(workflow, /missing-results-xml/); assert.match(workflow, /pester-run-receipt\.json/); assert.match(workflow, /Upload raw Pester execution artifact/); assert.match(workflow, /Emit execution contract/); assert.match(workflow, /Upload execution contract artifact/); + assert.doesNotMatch(workflow, /Export workflow token for priority sync/); assert.doesNotMatch(workflow, /Normalize include_integration/); assert.doesNotMatch(workflow, /needs\.normalize/); assert.doesNotMatch(workflow, /Install Pester/); @@ -146,10 +155,20 @@ test('pester evidence distinguishes context-blocked, selection-blocked, and read assert.match(workflow, /\$classification = 'context-blocked'/); assert.match(workflow, /\$classification = 'selection-blocked'/); assert.match(workflow, /\$classification = 'readiness-blocked'/); + assert.match(workflow, /\$classification = 'results-xml-truncated'/); + assert.match(workflow, /\$classification = 'invalid-results-xml'/); + assert.match(workflow, /\$classification = 'missing-results-xml'/); + assert.match(workflow, /execution-receipt-results-xml-truncated/); + assert.match(workflow, /execution-receipt-invalid-results-xml/); + assert.match(workflow, /execution-receipt-missing-results-xml/); + assert.match(workflow, /results-xml-status=/); assert.match(workflow, /\$contextStatus -ne 'ready'/); assert.match(workflow, /\$selectionStatus -ne 'ready'/); assert.match(workflow, /\$executionReceiptStatus -eq 'selection-blocked'/); assert.match(workflow, /\$executionReceiptStatus -eq 'context-blocked'/); + assert.match(workflow, /\$executionReceiptStatus -eq 'results-xml-truncated'/); + assert.match(workflow, /\$executionReceiptStatus -eq 'invalid-results-xml'/); + assert.match(workflow, /\$executionReceiptStatus -eq 'missing-results-xml'/); assert.match(workflow, /\$readinessStatus -ne 'ready' -and \$executionJobResult -in @\('skipped','cancelled'\)/); assert.match(workflow, /raw-artifact-download=/); assert.match(workflow, /execution-receipt-seam-defect/); From 077ceaa05453b505b1fd6a722d488cdcd9fc781b Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Tue, 31 Mar 2026 13:54:48 -0700 Subject: [PATCH 33/44] ci(pester): split execution finalize side effects (#2086) Co-authored-by: svelderrainruiz --- Invoke-PesterTests.ps1 | 467 ++++------- scripts/Pester-Invoker.psm1 | 2 +- .../Invoke-PesterExecutionFinalize.Tests.ps1 | 116 +++ tools/Invoke-PesterExecutionFinalize.ps1 | 728 ++++++++++++++++++ ...vice-model-local-harness-contract.test.mjs | 18 + 5 files changed, 1008 insertions(+), 323 deletions(-) create mode 100644 tests/Invoke-PesterExecutionFinalize.Tests.ps1 create mode 100644 tools/Invoke-PesterExecutionFinalize.ps1 diff --git a/Invoke-PesterTests.ps1 b/Invoke-PesterTests.ps1 index 7f1491e9e..658290888 100644 --- a/Invoke-PesterTests.ps1 +++ b/Invoke-PesterTests.ps1 @@ -1645,6 +1645,73 @@ function Write-SessionIndex { } catch { Write-Warning "Failed to write session index: $_" } } +function Invoke-ExecutionFinalizeHelper { + [CmdletBinding(DefaultParameterSetName = 'Build')] + param( + [Parameter(ParameterSetName = 'Build')] + [AllowEmptyString()] + [string]$SummaryText, + [Parameter(ParameterSetName = 'Build')] + $SummaryPayload, + [Parameter(ParameterSetName = 'Build')] + $ArtifactTrail, + [Parameter(ParameterSetName = 'Reuse')] + [switch]$ReuseExistingContext + ) + + try { + $finalizeToolPath = Join-Path $PSScriptRoot 'tools/Invoke-PesterExecutionFinalize.ps1' + if (-not (Test-Path -LiteralPath $finalizeToolPath -PathType Leaf)) { + throw "Invoke-PesterExecutionFinalize.ps1 not found: $finalizeToolPath" + } + + $contextPath = $null + if ($PSCmdlet.ParameterSetName -eq 'Reuse') { + if ([string]::IsNullOrWhiteSpace($script:executionFinalizeContextPath) -or -not (Test-Path -LiteralPath $script:executionFinalizeContextPath -PathType Leaf)) { + return $false + } + $contextPath = $script:executionFinalizeContextPath + } else { + $finalizeContext = [ordered]@{ + schema = 'pester-execution-finalize-context@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + repoRoot = $root + resultsDir = $resultsDir + jsonSummaryPath = $JsonSummaryPath + includeIntegration = [bool]$includeIntegrationBool + integrationMode = $script:integrationModeResolved + integrationSource = $script:integrationModeReason + summarySchemaVersion = $SchemaSummaryVersion + manifestVersion = $SchemaManifestVersion + failuresSchemaVersion = $SchemaFailuresVersion + leakReportSchemaVersion = ${SchemaLeakReportVersion} + diagnosticsSchemaVersion = ${SchemaDiagnosticsVersion} + } + if ($PSBoundParameters.ContainsKey('SummaryText')) { + $finalizeContext['summaryText'] = $SummaryText + } + if ($null -ne $SummaryPayload) { + $finalizeContext['summaryPayload'] = $SummaryPayload + } + if ($null -ne $ArtifactTrail) { + $finalizeContext['artifactTrail'] = $ArtifactTrail + } + $contextPath = Join-Path $resultsDir 'pester-execution-finalize-context.json' + $finalizeContext | ConvertTo-Json -Depth 12 | Out-File -FilePath $contextPath -Encoding utf8 -ErrorAction Stop + $script:executionFinalizeContextPath = $contextPath + } + + & $finalizeToolPath -ContextPath $contextPath | Out-Host + if ($LASTEXITCODE -ne 0) { + throw "Invoke-PesterExecutionFinalize.ps1 failed with exit code $LASTEXITCODE." + } + return $true + } catch { + Write-Warning "Failed to finalize execution side effects: $_" + return $false + } +} + # Optional pre-clean of LabVIEW if explicitly requested if ($CleanLabVIEW) { Write-Host "Pre-run cleanup: stopping LabVIEW.exe" -ForegroundColor DarkGray @@ -1655,6 +1722,7 @@ if ($CleanLabVIEW) { # Artifact tracking pre-snapshot (optional) $script:artifactTrail = $null +$script:executionFinalizeContextPath = $null $preIndex = $null $artifactRoots = @() if ($TrackArtifacts) { @@ -1959,31 +2027,29 @@ if ($testFiles.Count -eq 0) { $placeholder | Out-File -FilePath $xmlPathEmpty -Encoding utf8 -ErrorAction SilentlyContinue } $summaryPathEarly = Join-Path $resultsDir 'pester-summary.txt' - if (-not (Test-Path -LiteralPath $summaryPathEarly)) { - "=== Pester Test Summary ===`nTotal Tests: 0`nPassed: 0`nFailed: 0`nErrors: 0`nSkipped: 0`nDuration: 0.00s" | Out-File -FilePath $summaryPathEarly -Encoding utf8 -ErrorAction SilentlyContinue - } - $jsonSummaryEarly = Join-Path $resultsDir $JsonSummaryPath - if (-not (Test-Path -LiteralPath $jsonSummaryEarly)) { - $jsonObj = [pscustomobject]@{ - total = 0 - passed = 0 - failed = 0 - errors = 0 - skipped = 0 - duration_s = 0.0 - timestamp = (Get-Date).ToString('o') - pesterVersion = '' - includeIntegration= [bool]$includeIntegrationBool - integrationMode = $script:integrationModeResolved - integrationSource = $script:integrationModeReason - meanTest_ms = $null - p95Test_ms = $null - maxTest_ms = $null - timedOut = $false - discoveryFailures = 0 - schemaVersion = $SchemaSummaryVersion - } - $jsonObj | ConvertTo-Json -Depth 4 | Out-File -FilePath $jsonSummaryEarly -Encoding utf8 -ErrorAction SilentlyContinue + $summaryTextEarly = "=== Pester Test Summary ===`nTotal Tests: 0`nPassed: 0`nFailed: 0`nErrors: 0`nSkipped: 0`nDuration: 0.00s" + $jsonObj = [pscustomobject]@{ + total = 0 + passed = 0 + failed = 0 + errors = 0 + skipped = 0 + duration_s = 0.0 + timestamp = (Get-Date).ToString('o') + pesterVersion = '' + includeIntegration = [bool]$includeIntegrationBool + integrationMode = $script:integrationModeResolved + integrationSource = $script:integrationModeReason + meanTest_ms = $null + p95Test_ms = $null + maxTest_ms = $null + timedOut = $false + discoveryFailures = 0 + executionPostprocessStatus = 'complete' + resultsXmlStatus = 'complete' + resultsXmlSummarySource = 'placeholder' + resultsXmlCloseTagPresent = $true + schemaVersion = $SchemaSummaryVersion } # Optional: run leak detection even when no tests discovered @@ -2030,8 +2096,7 @@ if ($testFiles.Count -eq 0) { if ($leakDetected -and $FailOnLeaks) { Write-Error 'Failing run due to detected leaks (processes/jobs)'; exit 1 } } catch { Write-Warning "Leak detection (early-exit) failed: $_" } } - Write-ArtifactManifest -Directory $resultsDir -SummaryJsonPath $jsonSummaryEarly -ManifestVersion $SchemaManifestVersion - Write-SessionIndex -ResultsDirectory $resultsDir -SummaryJsonPath $jsonSummaryEarly + Invoke-ExecutionFinalizeHelper -SummaryText $summaryTextEarly -SummaryPayload $jsonObj -ArtifactTrail $script:artifactTrail | Out-Null Write-Host 'No test files found. Placeholder artifacts emitted.' -ForegroundColor Yellow exit 0 } @@ -2477,31 +2542,39 @@ if (-not $script:UseSingleInvoker) { $testDuration = $testEndTime - $testStartTime if ($script:stuckGuardEnabled) { _Write-HeartbeatLine 'stop' } - # Emit minimal JSON summary so downstream artifact/indices have data without running the classic path try { - $jsonSummaryPath = Join-Path $resultsDir $JsonSummaryPath $loadedPester = Get-Module Pester -ListAvailable | Sort-Object Version -Descending | Select-Object -First 1 + $summaryText = @( + '=== Pester Test Summary ===', + ('Total Tests: {0}' -f [int]($aggregate.passed + $aggregate.failed + $aggregate.errors + $aggregate.skipped)), + ('Passed: {0}' -f [int]$aggregate.passed), + ('Failed: {0}' -f [int]$aggregate.failed), + ('Errors: {0}' -f [int]$aggregate.errors), + ('Skipped: {0}' -f [int]$aggregate.skipped), + ('Duration: {0}s' -f ([math]::Round($testDuration.TotalSeconds, 2).ToString('0.00'))) + ) -join "`n" $jsonObj = [PSCustomObject]@{ - total = [int]($aggregate.passed + $aggregate.failed + $aggregate.errors + $aggregate.skipped) - passed = [int]$aggregate.passed - failed = [int]$aggregate.failed - errors = [int]$aggregate.errors - skipped = [int]$aggregate.skipped - duration_s = [math]::Round($testDuration.TotalSeconds, 6) - timestamp = (Get-Date).ToString('o') - pesterVersion = $loadedPester.Version.ToString() - includeIntegration = [bool]$includeIntegrationBool - schemaVersion = $SchemaSummaryVersion - timedOut = $false - discoveryFailures = 0 - } - if (-not (Test-Path -LiteralPath $resultsDir -PathType Container)) { New-Item -ItemType Directory -Force -Path $resultsDir | Out-Null } - $jsonObj | ConvertTo-Json -Depth 4 | Out-File -FilePath $jsonSummaryPath -Encoding utf8 -ErrorAction Stop - } catch { Write-Warning "[single-invoker] Failed to write JSON summary: $_" } - - # Write artifact manifest and session index for parity - try { Write-ArtifactManifest -Directory $resultsDir -SummaryJsonPath $jsonSummaryPath -ManifestVersion $SchemaManifestVersion } catch {} - try { Write-SessionIndex -ResultsDirectory $resultsDir -SummaryJsonPath $jsonSummaryPath } catch {} + total = [int]($aggregate.passed + $aggregate.failed + $aggregate.errors + $aggregate.skipped) + passed = [int]$aggregate.passed + failed = [int]$aggregate.failed + errors = [int]$aggregate.errors + skipped = [int]$aggregate.skipped + duration_s = [math]::Round($testDuration.TotalSeconds, 6) + timestamp = (Get-Date).ToString('o') + pesterVersion = if ($loadedPester) { $loadedPester.Version.ToString() } else { '' } + includeIntegration = [bool]$includeIntegrationBool + integrationMode = $script:integrationModeResolved + integrationSource = $script:integrationModeReason + schemaVersion = $SchemaSummaryVersion + timedOut = $false + discoveryFailures = 0 + executionPostprocessStatus = 'complete' + resultsXmlStatus = 'complete' + resultsXmlSummarySource = 'single-invoker-aggregate' + resultsXmlCloseTagPresent = $true + } + Invoke-ExecutionFinalizeHelper -SummaryText $summaryText -SummaryPayload $jsonObj -ArtifactTrail $script:artifactTrail | Out-Null + } catch { Write-Warning "[single-invoker] Failed to finalize summary/artifact side effects: $_" } # Print concise outcome and exit early to avoid re-entering the classic path $failTotal = [int]($aggregate.failed + $aggregate.errors) @@ -3084,58 +3157,6 @@ if ($EmitResultShapeDiagnostics) { } catch { Write-Warning "Failed to emit result shape diagnostics: $_" } } -# Write summary to file -$summaryPath = Join-Path $resultsDir 'pester-summary.txt' -try { - $summary | Out-File -FilePath $summaryPath -Encoding utf8 -ErrorAction Stop - Write-Host "Summary written to: $summaryPath" -ForegroundColor Gray -} catch { - Write-Warning "Failed to write summary file: $_" -} - -# Optional: append diagnostics footer to Pester summary -try { - $diagJsonPath = Join-Path $resultsDir 'result-shapes.json' - $diagTotalEntries = $null; $diagHasPath = $null; $diagHasTags = $null - if (Test-Path -LiteralPath $diagJsonPath -PathType Leaf) { - try { - $diagJsonRaw = Get-Content -LiteralPath $diagJsonPath -Raw - $diagObj = $diagJsonRaw | ConvertFrom-Json -ErrorAction Stop - $diagTotalEntries = [int]$diagObj.totalEntries - $diagHasPath = [int]$diagObj.overall.hasPath - $diagHasTags = [int]$diagObj.overall.hasTags - } catch {} - } - if ($null -eq $diagTotalEntries -and $result -and $result.Tests) { - try { $testsLocal=@($result.Tests); $diagTotalEntries=$testsLocal.Count; $diagHasPath=@($testsLocal | Where-Object { $_.PSObject.Properties.Name -contains 'Path' }).Count; $diagHasTags=@($testsLocal | Where-Object { $_.PSObject.Properties.Name -contains 'Tags' }).Count } catch {} - } - if ($null -ne $diagTotalEntries) { - function _pctTxt { param([int]$n,[int]$d) if ($d -le 0) { return '0%' } ('{0:P1}' -f ([double]$n/[double]$d)) } - $pPath = _pctTxt $diagHasPath $diagTotalEntries - $pTags = _pctTxt $diagHasTags $diagTotalEntries - $footer = @() - $footer += '' - $footer += '---' - $footer += 'Diagnostics Summary' - $footer += '' - $footer += ('Total entries: {0}' -f $diagTotalEntries) - $footer += ('Has Path: {0} ({1})' -f $diagHasPath,$pPath) - $footer += ('Has Tags: {0} ({1})' -f $diagHasTags,$pTags) - Add-Content -LiteralPath $summaryPath -Value ($footer -join "`n") -Encoding utf8 - } -} catch { Write-Host "(warn) failed to append diagnostics footer: $_" -ForegroundColor DarkYellow } - -# Persist artifact trail (if collected) -if ($TrackArtifacts -and $script:artifactTrail) { - try { - # Update procsAfter right before writing trail - $script:artifactTrail.procsAfter = @(_Get-ProcsSummary -Names @('LVCompare','LabVIEW')) - $trailPath = Join-Path $resultsDir 'pester-artifacts-trail.json' - $script:artifactTrail | ConvertTo-Json -Depth 6 | Out-File -FilePath $trailPath -Encoding utf8 -ErrorAction Stop - Write-Host "Artifact trail written to: $trailPath" -ForegroundColor Gray - } catch { Write-Warning "Failed to write artifact trail: $_" } -} - # Machine-readable JSON summary (adjacent enhancement for CI consumers) $jsonSummaryPath = Join-Path $resultsDir $JsonSummaryPath try { @@ -3451,224 +3472,13 @@ try { } catch { Write-Warning "Failed to attach fallback aggregation hints: $_" } } } - $jsonObj | ConvertTo-Json -Depth 4 | Out-File -FilePath $jsonSummaryPath -Encoding utf8 -ErrorAction Stop - Write-Host "JSON summary written to: $jsonSummaryPath" -ForegroundColor Gray } catch { - Write-Warning "Failed to write JSON summary file: $_" + Write-Warning "Failed to build JSON summary payload: $_" } Write-Host "Results written to: $xmlPath" -ForegroundColor Gray Write-Host "" -# Best-effort: copy any compare report produced by tests into the results directory for standardized artifact pickup -try { - $destReport = Join-Path $resultsDir 'compare-report.html' - $candidates = @() - $fixedCandidates = @( - (Join-Path $root 'tests' 'results' 'integration-compare-report.html'), - (Join-Path $root 'tests' 'results' 'compare-report.html'), - (Join-Path $root 'tests' 'results-single' 'pr-body-compare-report.html') - ) - foreach ($p in $fixedCandidates) { if (Test-Path -LiteralPath $p -PathType Leaf) { try { $candidates += (Get-Item -LiteralPath $p -ErrorAction SilentlyContinue) } catch {} } } - try { - $dynamic = Get-ChildItem -LiteralPath (Join-Path $root 'tests' 'results') -Filter '*compare-report*.html' -Recurse -File -ErrorAction SilentlyContinue - if ($dynamic) { $candidates += $dynamic } - } catch {} - if ($candidates.Count -gt 0) { - $latest = $candidates | Sort-Object LastWriteTimeUtc -Descending | Select-Object -First 1 - # Copy the latest to the canonical filename (skip if it's already the canonical file) - try { - $normalizePath = { - param( - [string]$Path, - [string]$BasePath = $null - ) - - if ([string]::IsNullOrWhiteSpace($Path)) { return $null } - - $candidate = $Path - $basePath = if ([string]::IsNullOrWhiteSpace($BasePath)) { (Get-Location).ProviderPath } else { $BasePath } - - if (-not [System.IO.Path]::IsPathRooted($candidate)) { - try { - $candidate = [System.IO.Path]::Combine($basePath, $candidate) - } catch { - return $candidate - } - } - - $attempts = @($candidate) - if (-not $candidate.StartsWith('\?\', [System.StringComparison]::OrdinalIgnoreCase)) { - if ($candidate.StartsWith('\', [System.StringComparison]::Ordinal)) { - $attempts += ('\?\UNC\' + $candidate.Substring(2)) - } else { - $attempts += ('\?\' + $candidate) - } - } - - foreach ($probe in $attempts) { - try { - $full = [System.IO.Path]::GetFullPath($probe) - try { - $resolved = Resolve-Path -LiteralPath $full -ErrorAction Stop - if ($resolved -and $resolved.ProviderPath) { - $full = $resolved.ProviderPath - } - } catch { - # Resolve-Path can fail when the target does not exist yet; fall back to the computed path. - } - if ($full.StartsWith('\?\UNC\', [System.StringComparison]::OrdinalIgnoreCase)) { - return ('\' + $full.Substring(8)) - } - if ($full.StartsWith('\?\', [System.StringComparison]::OrdinalIgnoreCase)) { - return $full.Substring(4) - } - return $full - } catch { - } - } - - return $candidate - } - - $destFullPath = & $normalizePath $destReport $root - $latestFullPath = & $normalizePath $latest.FullName $root - $shouldCopyLatest = $true - if ($latestFullPath -and $destFullPath) { - if ([string]::Equals($latestFullPath, $destFullPath, [System.StringComparison]::OrdinalIgnoreCase)) { - $shouldCopyLatest = $false - } - } - - if ($shouldCopyLatest) { - $destDir = [System.IO.Path]::GetDirectoryName($destReport) - if ($destDir -and $latest.DirectoryName) { - if ([string]::Equals($latest.DirectoryName, $destDir, [System.StringComparison]::OrdinalIgnoreCase) -and - [string]::Equals($latest.Name, 'compare-report.html', [System.StringComparison]::OrdinalIgnoreCase)) { - $shouldCopyLatest = $false - } - } - } - - if ($shouldCopyLatest) { - try { - Copy-Item -LiteralPath $latest.FullName -Destination $destReport -Force -ErrorAction Stop - Write-Host ("Compare report copied to: {0}" -f $destReport) -ForegroundColor Gray - } catch { - if ($_.Exception -and $_.Exception.Message -match 'Cannot overwrite .+ with itself') { - Write-Verbose 'Compare report already present at destination; skipping copy.' - } else { - Write-Warning "Failed to copy compare report: $_" - } - } - } - } catch { Write-Warning "Failed to copy compare report: $_" } - # Also copy all candidates preserving their base filenames to the results directory - foreach ($cand in ($candidates | Sort-Object LastWriteTimeUtc)) { - try { - $destName = (Split-Path -Leaf $cand.FullName) - $destFull = Join-Path $resultsDir $destName - $destFullPath = & $normalizePath $destFull $root - $candFullPath = & $normalizePath $cand.FullName $root - $shouldCopyCandidate = $true - if ($destFullPath -and $candFullPath) { - if ([string]::Equals($destFullPath, $candFullPath, [System.StringComparison]::OrdinalIgnoreCase)) { - $shouldCopyCandidate = $false - } - } - - if ($shouldCopyCandidate) { - if ([string]::Equals($cand.DirectoryName, $resultsDir, [System.StringComparison]::OrdinalIgnoreCase) -and - [string]::Equals($cand.Name, $destName, [System.StringComparison]::OrdinalIgnoreCase)) { - $shouldCopyCandidate = $false - } - } - - if ($shouldCopyCandidate) { - try { - Copy-Item -LiteralPath $cand.FullName -Destination $destFull -Force -ErrorAction Stop - } catch { - if ($_.Exception -and $_.Exception.Message -match 'Cannot overwrite .+ with itself') { - continue - } - Write-Host "(warn) failed to copy extra report '$($cand.FullName)': $_" -ForegroundColor DarkYellow - } - } - } catch { Write-Host "(warn) failed to copy extra report '$($cand.FullName)': $_" -ForegroundColor DarkYellow } - } - # Generate a small deterministic index HTML linking to all report variants - try { - $indexPath = Join-Path $resultsDir 'results-index.html' - # Gather all report htmls in results dir (including canonical) - $reports = @(Get-ChildItem -LiteralPath $resultsDir -Filter '*compare-report*.html' -File -ErrorAction SilentlyContinue | Sort-Object Name) - function _HtmlEncode { - param([string]$s) - if ([string]::IsNullOrEmpty($s)) { return '' } - $t = $s.Replace('&','&').Replace('<','<').Replace('>','>').Replace('"','"').Replace("'",''') - return $t - } - $now = (Get-Date).ToString('u') - $lines = @() - $lines += '' - $lines += '' - $lines += 'Compare Reports Index' - $lines += '' - $lines += '

      Compare Reports Index

      ' - $lines += ('

      Generated at {0}

      ' -f (_HtmlEncode $now)) - $lines += ('

      Total reports: {0} — canonical: compare-report.html

      ' -f $reports.Count) - if ($reports.Count -gt 0) { - $lines += '
        ' - foreach ($r in $reports) { - $nameEnc = _HtmlEncode $r.Name - $ts = _HtmlEncode ($r.LastWriteTimeUtc.ToString('u')) - $size = '{0:N0} bytes' -f $r.Length - $meta = ('last write: {0}; size: {1}' -f $ts, (_HtmlEncode $size)) - $canonicalTag = if ($r.Name -ieq 'compare-report.html') { ' (canonical)' } else { '' } - $lines += ('
      • {0}{2} ({1})
        • ' -f $nameEnc,$meta,$canonicalTag) - } - $lines += '
      ' - } else { - $lines += '

      No compare-report HTML files found in this results directory.

      ' - } - # Diagnostics links (if present) - try { - $diagTxt = Join-Path $resultsDir 'result-shapes.txt' - $diagJson = Join-Path $resultsDir 'result-shapes.json' - if ((Test-Path -LiteralPath $diagTxt) -or (Test-Path -LiteralPath $diagJson)) { - $lines += '
      ' - $lines += '

      Diagnostics

      ' - $lines += '' - # Optional: show a compact summary table if JSON exists - if (Test-Path -LiteralPath $diagJson) { - try { - $diagObj = Get-Content -LiteralPath $diagJson -Raw | ConvertFrom-Json -ErrorAction Stop - $total = [int]($diagObj.totalEntries) - $hasPath = [int]($diagObj.overall.hasPath) - $hasTags = [int]($diagObj.overall.hasTags) - function _pct { param([int]$num,[int]$den) if ($den -le 0) { return '0%' } else { return ('{0:P1}' -f ([double]$num/[double]$den)) } } - $pPath = _pct $hasPath $total - $pTags = _pct $hasTags $total - $lines += '
    SignalCollapsed NoiseLineageSignalCollapsed NoiseLineage
    ' - $lines += '' - $lines += '' - $lines += ('' -f $total) - $lines += ('' -f $hasPath,$pPath) - $lines += ('' -f $hasTags,$pTags) - $lines += '
    MetricCountPercent
    Total entries{0}-
    Has Path{0}{1}
    Has Tags{0}{1}
    ' - } catch { } - } - } - } catch {} - $lines += '' - Set-Content -LiteralPath $indexPath -Value ($lines -join "`n") -Encoding UTF8 - Write-Host ("Results index written to: {0}" -f $indexPath) -ForegroundColor Gray - } catch { Write-Host "(warn) failed to write results index: $_" -ForegroundColor DarkYellow } - } -} catch { Write-Host "(warn) compare report copy step failed: $_" -ForegroundColor DarkYellow } - # Optional: Write diagnostics summary to GitHub Step Summary (Markdown) try { $stepSummary = $env:GITHUB_STEP_SUMMARY @@ -3834,12 +3644,28 @@ try { } } catch { Write-Warning "Failed to evaluate integration execution note: $_" } +# Normalize failure diagnostics before finalizing execution side effects so the manifest and session index see the +# same failure surface that the exit path will expose. +if ($failed -gt 0 -or $errors -gt 0) { + if ($null -ne $result) { + Write-FailureDiagnostics -PesterResult $result -ResultsDirectory $resultsDir -SkippedCount $skipped -FailuresSchemaVersion $SchemaFailuresVersion + } elseif ($EmitFailuresJsonAlways) { + Ensure-FailuresJson -Directory $resultsDir -Force + } +} elseif ($EmitFailuresJsonAlways) { + Ensure-FailuresJson -Directory $resultsDir -Normalize -Quiet +} + +if ($TrackArtifacts -and $script:artifactTrail) { + try { + $script:artifactTrail.procsAfter = @(_Get-ProcsSummary -Names @('LVCompare','LabVIEW')) + } catch { Write-Warning "Failed to finalize artifact trail process snapshot: $_" } +} + +Invoke-ExecutionFinalizeHelper -SummaryText $summary -SummaryPayload $jsonObj -ArtifactTrail $script:artifactTrail | Out-Null + # Exit with appropriate code if ($failed -gt 0 -or $errors -gt 0) { - # Emit failure diagnostics using helper function (guard null result) - if ($null -ne $result) { Write-FailureDiagnostics -PesterResult $result -ResultsDirectory $resultsDir -SkippedCount $skipped -FailuresSchemaVersion $SchemaFailuresVersion } - elseif ($EmitFailuresJsonAlways) { Ensure-FailuresJson -Directory $resultsDir -Force } - Write-ArtifactManifest -Directory $resultsDir -SummaryJsonPath $jsonSummaryPath -ManifestVersion $SchemaManifestVersion $failureLine = "❌ Tests failed: $failed failure(s), $errors error(s)" if ($discoveryFailureCount -gt 0) { $failureLine += " (includes $discoveryFailureCount discovery failure(s))" } Write-Host $failureLine -ForegroundColor Red @@ -3862,9 +3688,6 @@ if ($discoveryFailureCount -gt 0) { Write-Error "Test execution completed with discovery failures" exit 1 } -if ($EmitFailuresJsonAlways) { Ensure-FailuresJson -Directory $resultsDir -Normalize -Quiet } - Write-ArtifactManifest -Directory $resultsDir -SummaryJsonPath $jsonSummaryPath -ManifestVersion $SchemaManifestVersion - Write-SessionIndex -ResultsDirectory $resultsDir -SummaryJsonPath $jsonSummaryPath } finally { # Ensure any background Pester job is stopped/removed to avoid lingering runs across sessions try { @@ -3912,8 +3735,8 @@ if ($EmitFailuresJsonAlways) { Ensure-FailuresJson -Directory $resultsDir -Norma notes = @('Final sweep leak report to ensure artifact presence; see main leak block for full details when enabled') } $report | ConvertTo-Json -Depth 6 | Out-File -FilePath $finalLeakPath -Encoding utf8 -ErrorAction SilentlyContinue - # Opportunistically refresh manifest to include jsonLeaks entry - try { Write-ArtifactManifest -Directory $resultsDir -SummaryJsonPath (Join-Path $resultsDir $JsonSummaryPath) -ManifestVersion $SchemaManifestVersion } catch {} + # Refresh finalize-owned artifacts so the late leak report is reflected in the manifest/session index. + Invoke-ExecutionFinalizeHelper -ReuseExistingContext | Out-Null } } } catch { Write-Warning "Failed to emit final sweep leak report: $_" } diff --git a/scripts/Pester-Invoker.psm1 b/scripts/Pester-Invoker.psm1 index 04e235e84..bd7bc0172 100644 --- a/scripts/Pester-Invoker.psm1 +++ b/scripts/Pester-Invoker.psm1 @@ -29,7 +29,7 @@ function Write-InvokerEvent { if ($RunId) { $payload.runId = $RunId } if ($Seed) { $payload.seed = $Seed } if ($Data) { foreach ($k in $Data.Keys) { $payload[$k] = $Data[$k] } } - ($payload | ConvertTo-Json -Compress) | Add-Content -Path $LogPath + ($payload | ConvertTo-Json -Depth 8 -Compress) | Add-Content -Path $LogPath } catch { Write-Warning ("[pester-invoker] failed to write crumb: {0}" -f $_.Exception.Message) } } diff --git a/tests/Invoke-PesterExecutionFinalize.Tests.ps1 b/tests/Invoke-PesterExecutionFinalize.Tests.ps1 new file mode 100644 index 000000000..99684c449 --- /dev/null +++ b/tests/Invoke-PesterExecutionFinalize.Tests.ps1 @@ -0,0 +1,116 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Describe 'Invoke-PesterExecutionFinalize' { + BeforeAll { + $repoRoot = Split-Path -Parent $PSScriptRoot + $toolPath = Join-Path $repoRoot 'tools/Invoke-PesterExecutionFinalize.ps1' + } + + It 'writes summary, trail, compare-report index, manifest, and session index from the finalize context' { + $tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("pester-finalize-" + [Guid]::NewGuid().ToString('N')) + $resultsDir = Join-Path $tempRoot 'artifacts' + $repoResultsDir = Join-Path $tempRoot 'tests/results' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + New-Item -ItemType Directory -Path $repoResultsDir -Force | Out-Null + + try { + @( + '', + '' + ) -join [Environment]::NewLine | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-results.xml') -Encoding UTF8 + '[]' | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-failures.json') -Encoding UTF8 + '{"schema":"pester-leak-report@v1","leakDetected":false}' | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-leak-report.json') -Encoding UTF8 + '{"schema":"pester-result-shapes/v1","schemaVersion":"1.1.0","generatedAt":"2026-03-31T00:00:00Z","totalEntries":3,"overall":{"hasPath":3,"hasTags":2},"byType":[]}' | Set-Content -LiteralPath (Join-Path $resultsDir 'result-shapes.json') -Encoding UTF8 + 'shape text' | Set-Content -LiteralPath (Join-Path $resultsDir 'result-shapes.txt') -Encoding UTF8 + 'compare report' | Set-Content -LiteralPath (Join-Path $repoResultsDir 'integration-compare-report.html') -Encoding UTF8 + + $contextPath = Join-Path $resultsDir 'pester-execution-finalize-context.json' + $context = [ordered]@{ + schema = 'pester-execution-finalize-context@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + repoRoot = $tempRoot + resultsDir = $resultsDir + jsonSummaryPath = 'pester-summary.json' + summaryText = "=== Pester Test Summary ===`nTotal Tests: 3`nPassed: 3`nFailed: 0`nErrors: 0`nSkipped: 0`nDuration: 1.23s" + summaryPayload = [ordered]@{ + total = 3 + passed = 3 + failed = 0 + errors = 0 + skipped = 0 + duration_s = 1.23 + timestamp = '2026-03-31T00:00:00Z' + schemaVersion = '1.7.1' + meanTest_ms = 10 + p95Test_ms = 20 + maxTest_ms = 30 + aggregatorBuildMs = 4.5 + executionPostprocessStatus = 'complete' + resultsXmlStatus = 'complete' + } + artifactTrail = [ordered]@{ + schema = 'pester-artifact-trail/v1' + generatedAt = [DateTime]::UtcNow.ToString('o') + created = @() + deleted = @() + modified = @() + procsBefore = @() + procsAfter = @() + } + includeIntegration = $false + integrationMode = 'exclude' + integrationSource = 'explicit' + summarySchemaVersion = '1.7.1' + manifestVersion = '1.0.0' + failuresSchemaVersion = '1.0.0' + leakReportSchemaVersion = '1.0.0' + diagnosticsSchemaVersion = '1.1.0' + } + $context | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $contextPath -Encoding UTF8 + + & $toolPath -ContextPath $contextPath | Out-Host + $LASTEXITCODE | Should -Be 0 + + $summaryPath = Join-Path $resultsDir 'pester-summary.txt' + $summaryJsonPath = Join-Path $resultsDir 'pester-summary.json' + $trailPath = Join-Path $resultsDir 'pester-artifacts-trail.json' + $indexPath = Join-Path $resultsDir 'results-index.html' + $manifestPath = Join-Path $resultsDir 'pester-artifacts.json' + $sessionIndexPath = Join-Path $resultsDir 'session-index.json' + $compareReportPath = Join-Path $resultsDir 'compare-report.html' + + Test-Path -LiteralPath $summaryPath | Should -BeTrue + Test-Path -LiteralPath $summaryJsonPath | Should -BeTrue + Test-Path -LiteralPath $trailPath | Should -BeTrue + Test-Path -LiteralPath $compareReportPath | Should -BeTrue + Test-Path -LiteralPath $indexPath | Should -BeTrue + Test-Path -LiteralPath $manifestPath | Should -BeTrue + Test-Path -LiteralPath $sessionIndexPath | Should -BeTrue + + (Get-Content -LiteralPath $summaryPath -Raw) | Should -Match 'Diagnostics Summary' + + $summaryJson = Get-Content -LiteralPath $summaryJsonPath -Raw | ConvertFrom-Json + $summaryJson.total | Should -Be 3 + $summaryJson.executionPostprocessStatus | Should -Be 'complete' + + $sessionIndex = Get-Content -LiteralPath $sessionIndexPath -Raw | ConvertFrom-Json + $sessionIndex.summary.total | Should -Be 3 + $sessionIndex.files.pesterSummaryJson | Should -Be 'pester-summary.json' + $sessionIndex.files.compareReportHtml | Should -Be 'compare-report.html' + $sessionIndex.files.resultsIndexHtml | Should -Be 'results-index.html' + + $manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json + $artifactFiles = @($manifest.artifacts | ForEach-Object { $_.file }) + $artifactFiles | Should -Contain 'pester-summary.json' + $artifactFiles | Should -Contain 'pester-artifacts-trail.json' + $artifactFiles | Should -Contain 'session-index.json' + $artifactFiles | Should -Contain 'compare-report.html' + $artifactFiles | Should -Contain 'results-index.html' + } finally { + if (Test-Path -LiteralPath $tempRoot) { + Remove-Item -LiteralPath $tempRoot -Recurse -Force -ErrorAction SilentlyContinue + } + } + } +} diff --git a/tools/Invoke-PesterExecutionFinalize.ps1 b/tools/Invoke-PesterExecutionFinalize.ps1 new file mode 100644 index 000000000..ce373147f --- /dev/null +++ b/tools/Invoke-PesterExecutionFinalize.ps1 @@ -0,0 +1,728 @@ +param( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string]$ContextPath +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Read-JsonObject { + param([Parameter(Mandatory = $true)][string]$PathValue) + + if (-not (Test-Path -LiteralPath $PathValue -PathType Leaf)) { + throw "JSON file not found: $PathValue" + } + + return (Get-Content -LiteralPath $PathValue -Raw | ConvertFrom-Json -ErrorAction Stop) +} + +function Set-ObjectProperty { + param( + [Parameter(Mandatory = $true)]$InputObject, + [Parameter(Mandatory = $true)][string]$Name, + $Value + ) + + $property = $InputObject.PSObject.Properties[$Name] + if ($property) { + $property.Value = $Value + } else { + Add-Member -InputObject $InputObject -Name $Name -MemberType NoteProperty -Value $Value + } +} + +function Get-RunnerProfileSnapshot { + $runnerProfile = $null + try { + if (-not (Get-Command -Name Get-RunnerProfile -ErrorAction SilentlyContinue)) { + $repoRoot = Split-Path -Parent $PSScriptRoot + $runnerModule = Join-Path $repoRoot 'tools/RunnerProfile.psm1' + if (Test-Path -LiteralPath $runnerModule -PathType Leaf) { + Import-Module $runnerModule -Force + } + } + if (Get-Command -Name Get-RunnerProfile -ErrorAction SilentlyContinue) { + $runnerProfile = Get-RunnerProfile + } + } catch {} + + return $runnerProfile +} + +function Get-ResultShapeSummary { + param([Parameter(Mandatory = $true)][string]$ResultsDirectory) + + $diagJsonPath = Join-Path $ResultsDirectory 'result-shapes.json' + if (-not (Test-Path -LiteralPath $diagJsonPath -PathType Leaf)) { + return $null + } + + try { + $diagObj = Get-Content -LiteralPath $diagJsonPath -Raw | ConvertFrom-Json -ErrorAction Stop + return [pscustomobject]@{ + totalEntries = [int]$diagObj.totalEntries + hasPath = [int]$diagObj.overall.hasPath + hasTags = [int]$diagObj.overall.hasTags + } + } catch { + return $null + } +} + +function Append-DiagnosticsFooterToSummary { + param( + [Parameter(Mandatory = $true)][string]$SummaryPath, + [Parameter(Mandatory = $true)][string]$ResultsDirectory + ) + + $diag = Get-ResultShapeSummary -ResultsDirectory $ResultsDirectory + if ($null -eq $diag) { + return + } + + function Get-PercentText { + param([int]$Numerator,[int]$Denominator) + if ($Denominator -le 0) { return '0%' } + return ('{0:P1}' -f ([double]$Numerator / [double]$Denominator)) + } + + $footer = @() + $footer += '' + $footer += '---' + $footer += 'Diagnostics Summary' + $footer += '' + $footer += ('Total entries: {0}' -f $diag.totalEntries) + $footer += ('Has Path: {0} ({1})' -f $diag.hasPath, (Get-PercentText -Numerator $diag.hasPath -Denominator $diag.totalEntries)) + $footer += ('Has Tags: {0} ({1})' -f $diag.hasTags, (Get-PercentText -Numerator $diag.hasTags -Denominator $diag.totalEntries)) + Add-Content -LiteralPath $SummaryPath -Value ($footer -join "`n") -Encoding utf8 +} + +function Copy-CompareReportsAndWriteIndex { + param( + [Parameter(Mandatory = $true)][string]$RepoRoot, + [Parameter(Mandatory = $true)][string]$ResultsDirectory + ) + + $destReport = Join-Path $ResultsDirectory 'compare-report.html' + $candidates = @() + $fixedCandidates = @( + (Join-Path $RepoRoot 'tests' 'results' 'integration-compare-report.html'), + (Join-Path $RepoRoot 'tests' 'results' 'compare-report.html'), + (Join-Path $RepoRoot 'tests' 'results-single' 'pr-body-compare-report.html') + ) + foreach ($pathValue in $fixedCandidates) { + if (Test-Path -LiteralPath $pathValue -PathType Leaf) { + try { $candidates += (Get-Item -LiteralPath $pathValue -ErrorAction SilentlyContinue) } catch {} + } + } + try { + $dynamic = Get-ChildItem -LiteralPath (Join-Path $RepoRoot 'tests' 'results') -Filter '*compare-report*.html' -Recurse -File -ErrorAction SilentlyContinue + if ($dynamic) { $candidates += $dynamic } + } catch {} + + function Normalize-PathValue { + param( + [string]$PathValue, + [string]$BasePath = $null + ) + + if ([string]::IsNullOrWhiteSpace($PathValue)) { return $null } + + $candidate = $PathValue + $basePath = if ([string]::IsNullOrWhiteSpace($BasePath)) { (Get-Location).ProviderPath } else { $BasePath } + + if (-not [System.IO.Path]::IsPathRooted($candidate)) { + try { + $candidate = [System.IO.Path]::Combine($basePath, $candidate) + } catch { + return $candidate + } + } + + $attempts = @($candidate) + if (-not $candidate.StartsWith('\\?\', [System.StringComparison]::OrdinalIgnoreCase)) { + if ($candidate.StartsWith('\\', [System.StringComparison]::Ordinal)) { + $attempts += ('\\?\UNC\' + $candidate.Substring(2)) + } else { + $attempts += ('\\?\' + $candidate) + } + } + + foreach ($probe in $attempts) { + try { + $full = [System.IO.Path]::GetFullPath($probe) + try { + $resolved = Resolve-Path -LiteralPath $full -ErrorAction Stop + if ($resolved -and $resolved.ProviderPath) { + $full = $resolved.ProviderPath + } + } catch {} + + if ($full.StartsWith('\\?\UNC\', [System.StringComparison]::OrdinalIgnoreCase)) { + return ('\\' + $full.Substring(8)) + } + if ($full.StartsWith('\\?\', [System.StringComparison]::OrdinalIgnoreCase)) { + return $full.Substring(4) + } + return $full + } catch {} + } + + return $candidate + } + + if ($candidates.Count -gt 0) { + $latest = $candidates | Sort-Object LastWriteTimeUtc -Descending | Select-Object -First 1 + try { + $destFullPath = Normalize-PathValue -PathValue $destReport -BasePath $RepoRoot + $latestFullPath = Normalize-PathValue -PathValue $latest.FullName -BasePath $RepoRoot + $shouldCopyLatest = $true + if ($latestFullPath -and $destFullPath -and [string]::Equals($latestFullPath, $destFullPath, [System.StringComparison]::OrdinalIgnoreCase)) { + $shouldCopyLatest = $false + } + + if ($shouldCopyLatest) { + $destDir = [System.IO.Path]::GetDirectoryName($destReport) + if ($destDir -and $latest.DirectoryName -and + [string]::Equals($latest.DirectoryName, $destDir, [System.StringComparison]::OrdinalIgnoreCase) -and + [string]::Equals($latest.Name, 'compare-report.html', [System.StringComparison]::OrdinalIgnoreCase)) { + $shouldCopyLatest = $false + } + } + + if ($shouldCopyLatest) { + try { + Copy-Item -LiteralPath $latest.FullName -Destination $destReport -Force -ErrorAction Stop + Write-Host ("Compare report copied to: {0}" -f $destReport) -ForegroundColor Gray + } catch { + if (-not ($_.Exception -and $_.Exception.Message -match 'Cannot overwrite .+ with itself')) { + Write-Warning "Failed to copy compare report: $_" + } + } + } + } catch { + Write-Warning "Failed to copy compare report: $_" + } + + foreach ($candidate in ($candidates | Sort-Object LastWriteTimeUtc)) { + try { + $destName = Split-Path -Leaf $candidate.FullName + $destFull = Join-Path $ResultsDirectory $destName + $destFullPath = Normalize-PathValue -PathValue $destFull -BasePath $RepoRoot + $candidateFullPath = Normalize-PathValue -PathValue $candidate.FullName -BasePath $RepoRoot + $shouldCopyCandidate = $true + if ($destFullPath -and $candidateFullPath -and [string]::Equals($destFullPath, $candidateFullPath, [System.StringComparison]::OrdinalIgnoreCase)) { + $shouldCopyCandidate = $false + } + if ($shouldCopyCandidate -and + [string]::Equals($candidate.DirectoryName, $ResultsDirectory, [System.StringComparison]::OrdinalIgnoreCase) -and + [string]::Equals($candidate.Name, $destName, [System.StringComparison]::OrdinalIgnoreCase)) { + $shouldCopyCandidate = $false + } + + if ($shouldCopyCandidate) { + try { + Copy-Item -LiteralPath $candidate.FullName -Destination $destFull -Force -ErrorAction Stop + } catch { + if (-not ($_.Exception -and $_.Exception.Message -match 'Cannot overwrite .+ with itself')) { + Write-Host "(warn) failed to copy extra report '$($candidate.FullName)': $_" -ForegroundColor DarkYellow + } + } + } + } catch { + Write-Host "(warn) failed to copy extra report '$($candidate.FullName)': $_" -ForegroundColor DarkYellow + } + } + } + + try { + $indexPath = Join-Path $ResultsDirectory 'results-index.html' + $reports = @(Get-ChildItem -LiteralPath $ResultsDirectory -Filter '*compare-report*.html' -File -ErrorAction SilentlyContinue | Sort-Object Name) + function Html-Encode { + param([string]$TextValue) + if ([string]::IsNullOrEmpty($TextValue)) { return '' } + return $TextValue.Replace('&','&').Replace('<','<').Replace('>','>').Replace('"','"').Replace("'",''') + } + $now = (Get-Date).ToString('u') + $lines = @() + $lines += '' + $lines += '' + $lines += 'Compare Reports Index' + $lines += '' + $lines += '

    Compare Reports Index

    ' + $lines += ("

    Generated at {0}

    " -f (Html-Encode -TextValue $now)) + $lines += ("

    Total reports: {0} — canonical: compare-report.html

    " -f $reports.Count) + if ($reports.Count -gt 0) { + $lines += '
      ' + foreach ($report in $reports) { + $nameEnc = Html-Encode -TextValue $report.Name + $ts = Html-Encode -TextValue ($report.LastWriteTimeUtc.ToString('u')) + $size = '{0:N0} bytes' -f $report.Length + $meta = ('last write: {0}; size: {1}' -f $ts, (Html-Encode -TextValue $size)) + $canonicalTag = if ($report.Name -ieq 'compare-report.html') { ' (canonical)' } else { '' } + $lines += ('
    • {0}{2} ({1})
      • ' -f $nameEnc, $meta, $canonicalTag) + } + $lines += '
    ' + } else { + $lines += '

    No compare-report HTML files found in this results directory.

    ' + } + try { + $diagTxt = Join-Path $ResultsDirectory 'result-shapes.txt' + $diagJson = Join-Path $ResultsDirectory 'result-shapes.json' + if ((Test-Path -LiteralPath $diagTxt) -or (Test-Path -LiteralPath $diagJson)) { + $lines += '
    ' + $lines += '

    Diagnostics

    ' + $lines += '' + if (Test-Path -LiteralPath $diagJson) { + try { + $diagObj = Get-Content -LiteralPath $diagJson -Raw | ConvertFrom-Json -ErrorAction Stop + $totalEntries = [int]$diagObj.totalEntries + $hasPath = [int]$diagObj.overall.hasPath + $hasTags = [int]$diagObj.overall.hasTags + function Get-PercentHtml { + param([int]$Numerator,[int]$Denominator) + if ($Denominator -le 0) { return '0%' } + return ('{0:P1}' -f ([double]$Numerator / [double]$Denominator)) + } + $lines += '' + $lines += '' + $lines += '' + $lines += ('' -f $totalEntries) + $lines += ('' -f $hasPath, (Get-PercentHtml -Numerator $hasPath -Denominator $totalEntries)) + $lines += ('' -f $hasTags, (Get-PercentHtml -Numerator $hasTags -Denominator $totalEntries)) + $lines += '
    MetricCountPercent
    Total entries{0}-
    Has Path{0}{1}
    Has Tags{0}{1}
    ' + } catch {} + } + } + } catch {} + $lines += '' + Set-Content -LiteralPath $indexPath -Value ($lines -join "`n") -Encoding UTF8 + Write-Host ("Results index written to: {0}" -f $indexPath) -ForegroundColor Gray + } catch { + Write-Host "(warn) failed to write results index: $_" -ForegroundColor DarkYellow + } +} + +function Write-ArtifactManifest { + param( + [Parameter(Mandatory = $true)][string]$Directory, + [Parameter(Mandatory = $true)][string]$SummaryJsonPath, + [Parameter(Mandatory = $true)][string]$ManifestVersion, + [Parameter(Mandatory = $true)][string]$SummarySchemaVersion, + [Parameter(Mandatory = $true)][string]$FailuresSchemaVersion, + [Parameter(Mandatory = $true)][string]$LeakReportSchemaVersion, + [Parameter(Mandatory = $true)][string]$DiagnosticsSchemaVersion + ) + + if (-not (Test-Path -LiteralPath $Directory -PathType Container)) { + New-Item -ItemType Directory -Force -Path $Directory | Out-Null + } + + $artifacts = @() + $xmlPath = Join-Path $Directory 'pester-results.xml' + if (Test-Path -LiteralPath $xmlPath) { + $artifacts += [pscustomobject]@{ file = 'pester-results.xml'; type = 'nunitXml' } + } + $txtPath = Join-Path $Directory 'pester-summary.txt' + if (Test-Path -LiteralPath $txtPath) { + $artifacts += [pscustomobject]@{ file = 'pester-summary.txt'; type = 'textSummary' } + } + $cmpPath = Join-Path $Directory 'compare-report.html' + if (Test-Path -LiteralPath $cmpPath) { + $artifacts += [pscustomobject]@{ file = 'compare-report.html'; type = 'htmlCompare' } + } + $idxPath = Join-Path $Directory 'results-index.html' + if (Test-Path -LiteralPath $idxPath) { + $artifacts += [pscustomobject]@{ file = 'results-index.html'; type = 'htmlIndex' } + } + try { + $extraHtml = @(Get-ChildItem -LiteralPath $Directory -Filter '*compare-report*.html' -File -ErrorAction SilentlyContinue | Where-Object { $_.Name -ne 'compare-report.html' }) + foreach ($item in $extraHtml) { + $artifacts += [pscustomobject]@{ file = $item.Name; type = 'htmlCompare' } + } + } catch {} + + $jsonSummaryFile = Split-Path -Leaf $SummaryJsonPath + if ($jsonSummaryFile) { + $jsonPath = Join-Path $Directory $jsonSummaryFile + if (Test-Path -LiteralPath $jsonPath) { + $artifacts += [pscustomobject]@{ file = $jsonSummaryFile; type = 'jsonSummary'; schemaVersion = $SummarySchemaVersion } + } + } + $failuresPath = Join-Path $Directory 'pester-failures.json' + if (Test-Path -LiteralPath $failuresPath) { + $artifacts += [pscustomobject]@{ file = 'pester-failures.json'; type = 'jsonFailures'; schemaVersion = $FailuresSchemaVersion } + } + $trailPath = Join-Path $Directory 'pester-artifacts-trail.json' + if (Test-Path -LiteralPath $trailPath) { + $artifacts += [pscustomobject]@{ file = 'pester-artifacts-trail.json'; type = 'jsonTrail' } + } + $sessionIdx = Join-Path $Directory 'session-index.json' + if (Test-Path -LiteralPath $sessionIdx) { + $artifacts += [pscustomobject]@{ file = 'session-index.json'; type = 'jsonSessionIndex' } + } + $leakPath = Join-Path $Directory 'pester-leak-report.json' + if (Test-Path -LiteralPath $leakPath) { + $artifacts += [pscustomobject]@{ file = 'pester-leak-report.json'; type = 'jsonLeaks'; schemaVersion = $LeakReportSchemaVersion } + } + $diagTxt = Join-Path $Directory 'result-shapes.txt' + if (Test-Path -LiteralPath $diagTxt) { + $artifacts += [pscustomobject]@{ file = 'result-shapes.txt'; type = 'textDiagnostics' } + } + $diagJson = Join-Path $Directory 'result-shapes.json' + if (Test-Path -LiteralPath $diagJson) { + $artifacts += [pscustomobject]@{ file = 'result-shapes.json'; type = 'jsonDiagnostics'; schemaVersion = $DiagnosticsSchemaVersion } + } + + $metrics = $null + try { + if ($jsonSummaryFile) { + $jsonPath = Join-Path $Directory $jsonSummaryFile + if (Test-Path -LiteralPath $jsonPath) { + $summaryJson = Get-Content -LiteralPath $jsonPath -Raw | ConvertFrom-Json -ErrorAction Stop + $meanTestMs = if ($summaryJson.PSObject.Properties['meanTest_ms']) { $summaryJson.meanTest_ms } else { $null } + $p95TestMs = if ($summaryJson.PSObject.Properties['p95Test_ms']) { $summaryJson.p95Test_ms } else { $null } + $maxTestMs = if ($summaryJson.PSObject.Properties['maxTest_ms']) { $summaryJson.maxTest_ms } else { $null } + $aggMsValue = $null + if ($summaryJson.PSObject.Properties.Name -contains 'aggregatorBuildMs' -and $null -ne $summaryJson.aggregatorBuildMs) { + $aggMsValue = $summaryJson.aggregatorBuildMs + } + $metrics = [pscustomobject]@{ + totalTests = $summaryJson.total + failed = $summaryJson.failed + skipped = $summaryJson.skipped + duration_s = $summaryJson.duration_s + meanTest_ms = $meanTestMs + p95Test_ms = $p95TestMs + maxTest_ms = $maxTestMs + aggregatorBuildMs = $aggMsValue + } + } + } + } catch { + Write-Warning "Failed to enrich manifest metrics: $_" + } + + $manifest = [pscustomobject]@{ + manifestVersion = $ManifestVersion + generatedAt = (Get-Date).ToString('o') + artifacts = $artifacts + metrics = $metrics + } + $manifestPath = Join-Path $Directory 'pester-artifacts.json' + $manifest | ConvertTo-Json -Depth 5 | Out-File -FilePath $manifestPath -Encoding utf8 -ErrorAction Stop + Write-Host ("Artifact manifest written to: {0}" -f $manifestPath) -ForegroundColor Gray + return $manifestPath +} + +function Write-SessionIndex { + param( + [Parameter(Mandatory = $true)][string]$ResultsDirectory, + [Parameter(Mandatory = $true)][string]$SummaryJsonPath, + [Parameter(Mandatory = $true)][bool]$IncludeIntegration, + [Parameter()][string]$IntegrationMode, + [Parameter()][string]$IntegrationSource + ) + + if (-not (Test-Path -LiteralPath $ResultsDirectory -PathType Container)) { + throw "Results directory not found: $ResultsDirectory" + } + + $idx = [ordered]@{ + schema = 'session-index/v1' + schemaVersion = '1.0.0' + generatedAtUtc = (Get-Date).ToUniversalTime().ToString('o') + resultsDir = $ResultsDirectory + includeIntegration = $IncludeIntegration + integrationMode = $IntegrationMode + integrationSource = $IntegrationSource + files = [ordered]@{} + } + + $runnerProfile = Get-RunnerProfileSnapshot + $addIf = { + param($Name, $File) + $pathValue = Join-Path $ResultsDirectory $File + if (Test-Path -LiteralPath $pathValue -PathType Leaf) { + $idx.files[$Name] = $File + } + } + + & $addIf 'pesterResultsXml' 'pester-results.xml' + & $addIf 'pesterSummaryTxt' 'pester-summary.txt' + $jsonLeaf = Split-Path -Leaf $SummaryJsonPath + if ($jsonLeaf) { + & $addIf 'pesterSummaryJson' $jsonLeaf + try { + $sumPath = Join-Path $ResultsDirectory $jsonLeaf + if (Test-Path -LiteralPath $sumPath -PathType Leaf) { + $summary = Get-Content -LiteralPath $sumPath -Raw | ConvertFrom-Json -ErrorAction Stop + $idx['summary'] = [ordered]@{ + total = $summary.total + passed = $summary.passed + failed = $summary.failed + errors = $summary.errors + skipped = $summary.skipped + duration_s = $summary.duration_s + meanTest_ms = $summary.meanTest_ms + p95Test_ms = $summary.p95Test_ms + maxTest_ms = $summary.maxTest_ms + schemaVersion = $summary.schemaVersion + } + $status = if (($summary.failed -gt 0) -or ($summary.errors -gt 0)) { 'fail' } else { 'ok' } + $idx['status'] = $status + $resultsRelative = $ResultsDirectory + try { + $cwd = (Get-Location).Path + if ($resultsRelative.StartsWith($cwd, [System.StringComparison]::OrdinalIgnoreCase)) { + $relative = $resultsRelative.Substring($cwd.Length).TrimStart('\', '/') + if (-not [string]::IsNullOrWhiteSpace($relative)) { + $resultsRelative = $relative + } + } + } catch {} + $lines = @() + $lines += '### Session Overview' + $lines += '' + $lines += ("- Status: {0}" -f $status) + $lines += ("- Total: {0} | Passed: {1} | Failed: {2} | Errors: {3} | Skipped: {4}" -f $summary.total, $summary.passed, $summary.failed, $summary.errors, $summary.skipped) + $lines += ("- Duration (s): {0}" -f $summary.duration_s) + $lines += ("- Include Integration: {0}" -f $IncludeIntegration) + $lines += ("- Integration Mode: {0}" -f $IntegrationMode) + if ($IntegrationSource) { $lines += ("- Integration Source: {0}" -f $IntegrationSource) } + $lines += '' + $lines += 'Artifacts (paths):' + $present = @() + foreach ($key in @('pesterSummaryJson','pesterResultsXml','pesterSummaryTxt','artifactManifestJson','artifactTrailJson','leakReportJson','compareReportHtml','resultsIndexHtml')) { + if ($idx.files[$key]) { $present += (Join-Path $resultsRelative $idx.files[$key]) } + } + foreach ($pathValue in $present) { $lines += ("- {0}" -f $pathValue) } + $runnerName = if ($runnerProfile -and $runnerProfile.PSObject.Properties.Name -contains 'name' -and $runnerProfile.name) { $runnerProfile.name } else { $env:RUNNER_NAME } + $runnerOs = if ($runnerProfile -and $runnerProfile.PSObject.Properties.Name -contains 'os' -and $runnerProfile.os) { $runnerProfile.os } else { $env:RUNNER_OS } + $runnerArch = if ($runnerProfile -and $runnerProfile.PSObject.Properties.Name -contains 'arch' -and $runnerProfile.arch) { $runnerProfile.arch } else { $env:RUNNER_ARCH } + $runnerEnvironment = if ($runnerProfile -and $runnerProfile.PSObject.Properties.Name -contains 'environment' -and $runnerProfile.environment) { $runnerProfile.environment } else { $env:RUNNER_ENVIRONMENT } + $runnerMachine = if ($runnerProfile -and $runnerProfile.PSObject.Properties.Name -contains 'machine' -and $runnerProfile.machine) { $runnerProfile.machine } else { [System.Environment]::MachineName } + $runnerLabels = @() + try { + if ($runnerProfile -and $runnerProfile.PSObject.Properties.Name -contains 'labels') { + $runnerLabels = @($runnerProfile.labels | Where-Object { $_ -and $_ -ne '' }) + } elseif (Get-Command -Name Get-RunnerLabels -ErrorAction SilentlyContinue) { + $runnerLabels = @(Get-RunnerLabels | Where-Object { $_ -and $_ -ne '' }) + } + } catch {} + if ($runnerName -or $runnerOs -or $runnerArch -or $runnerEnvironment -or $runnerMachine -or ($runnerLabels -and $runnerLabels.Count -gt 0)) { + $lines += '' + $lines += '### Runner' + $lines += '' + if ($runnerName) { $lines += ("- Name: {0}" -f $runnerName) } + if ($runnerOs -and $runnerArch) { + $lines += ("- OS/Arch: {0}/{1}" -f $runnerOs, $runnerArch) + } elseif ($runnerOs) { + $lines += ("- OS: {0}" -f $runnerOs) + } elseif ($runnerArch) { + $lines += ("- Arch: {0}" -f $runnerArch) + } + if ($runnerEnvironment) { $lines += ("- Environment: {0}" -f $runnerEnvironment) } + if ($runnerMachine) { $lines += ("- Machine: {0}" -f $runnerMachine) } + if ($runnerLabels -and $runnerLabels.Count -gt 0) { + $lines += ("- Labels: {0}" -f (($runnerLabels | Select-Object -Unique) -join ', ')) + } + } + $idx['stepSummary'] = ($lines -join "`n") + } + } catch {} + } + + & $addIf 'pesterFailuresJson' 'pester-failures.json' + & $addIf 'artifactManifestJson' 'pester-artifacts.json' + & $addIf 'artifactTrailJson' 'pester-artifacts-trail.json' + & $addIf 'dispatcherEventsNdjson' 'dispatcher-events.ndjson' + & $addIf 'leakReportJson' 'pester-leak-report.json' + & $addIf 'compareReportHtml' 'compare-report.html' + & $addIf 'resultsIndexHtml' 'results-index.html' + + try { + $driftRoot = Join-Path (Get-Location) 'results/fixture-drift' + if (Test-Path -LiteralPath $driftRoot -PathType Container) { + $dirs = Get-ChildItem -LiteralPath $driftRoot -Directory + $tsDirs = @($dirs | Where-Object { $_.Name -match '^[0-9]{8}T[0-9]{6}Z$' }) + $latest = if ($tsDirs.Count -gt 0) { $tsDirs | Sort-Object Name -Descending | Select-Object -First 1 } else { $dirs | Sort-Object LastWriteTimeUtc -Descending | Select-Object -First 1 } + if ($latest) { + $sumPath = Join-Path $latest.FullName 'drift-summary.json' + $status = $null + if (Test-Path -LiteralPath $sumPath) { + try { $driftSummary = Get-Content -LiteralPath $sumPath -Raw | ConvertFrom-Json -ErrorAction Stop; $status = $driftSummary.status } catch {} + } + $idx['drift'] = [ordered]@{ + latestRunDir = $latest.FullName + latestSummary = if (Test-Path -LiteralPath $sumPath) { $sumPath } else { $null } + status = $status + } + } + } + } catch {} + + try { + $runContext = [ordered]@{ + repository = $env:GITHUB_REPOSITORY + ref = (if ($env:GITHUB_HEAD_REF) { $env:GITHUB_HEAD_REF } else { $env:GITHUB_REF }) + commitSha = $env:GITHUB_SHA + workflow = $env:GITHUB_WORKFLOW + runId = $env:GITHUB_RUN_ID + runAttempt = $env:GITHUB_RUN_ATTEMPT + } + if ($env:GITHUB_JOB) { $runContext['job'] = $env:GITHUB_JOB } + if ($env:RUNNER_NAME) { $runContext['runner'] = $env:RUNNER_NAME } + if ($env:RUNNER_OS) { $runContext['runnerOS'] = $env:RUNNER_OS } + if ($env:RUNNER_ARCH) { $runContext['runnerArch'] = $env:RUNNER_ARCH } + if ($env:RUNNER_ENVIRONMENT) { $runContext['runnerEnvironment'] = $env:RUNNER_ENVIRONMENT } + $machineName = try { [System.Environment]::MachineName } catch { $null } + if ($machineName) { $runContext['runnerMachine'] = $machineName } + if ($env:RUNNER_TRACKING_ID) { $runContext['runnerTrackingId'] = $env:RUNNER_TRACKING_ID } + if ($env:ImageOS) { $runContext['runnerImageOS'] = $env:ImageOS } + if ($env:ImageVersion) { $runContext['runnerImageVersion'] = $env:ImageVersion } + if ($runnerProfile) { + $map = @{ + name = 'runner' + os = 'runnerOS' + arch = 'runnerArch' + environment = 'runnerEnvironment' + machine = 'runnerMachine' + trackingId = 'runnerTrackingId' + imageOS = 'runnerImageOS' + imageVersion = 'runnerImageVersion' + } + foreach ($entry in $map.GetEnumerator()) { + $source = $entry.Key + $target = $entry.Value + if ($runnerProfile.PSObject.Properties.Name -contains $source) { + $value = $runnerProfile.$source + if ($null -ne $value -and "$value" -ne '') { + $runContext[$target] = $value + } + } + } + if ($runnerProfile.PSObject.Properties.Name -contains 'labels') { + $labelValues = @($runnerProfile.labels | Where-Object { $_ -and $_ -ne '' }) + if ($labelValues.Count -gt 0) { $runContext['runnerLabels'] = $labelValues } + } + } elseif (Get-Command -Name Get-RunnerLabels -ErrorAction SilentlyContinue) { + try { + $labelsFallback = @(Get-RunnerLabels | Where-Object { $_ -and $_ -ne '' }) + if ($labelsFallback.Count -gt 0) { $runContext['runnerLabels'] = $labelsFallback } + } catch {} + } + $idx['runContext'] = $runContext + if ($env:GITHUB_REPOSITORY) { + $repoUrl = "https://github.com/$($env:GITHUB_REPOSITORY)" + $urls = [ordered]@{ repository = $repoUrl } + if ($env:GITHUB_RUN_ID) { $urls.run = "$repoUrl/actions/runs/$($env:GITHUB_RUN_ID)" } + if ($env:GITHUB_SHA) { $urls.commit = "$repoUrl/commit/$($env:GITHUB_SHA)" } + try { + $refValue = $env:GITHUB_REF + if ($refValue -and $refValue -match 'refs/pull/(?\d+)/') { + $urls.pullRequest = "$repoUrl/pull/$($Matches.num)" + } + } catch {} + $idx['urls'] = $urls + } + } catch {} + + try { + $handshakeFiles = @(Get-ChildItem -Path $ResultsDirectory -Recurse -Filter 'handshake-*.json' -File -ErrorAction SilentlyContinue) + if ($handshakeFiles.Count -gt 0) { + $sortedHandshake = @($handshakeFiles | Sort-Object LastWriteTimeUtc) + $last = $sortedHandshake[-1] + $lastRel = try { ($last.FullName).Substring(((Get-Location).Path).Length).TrimStart('\','/') } catch { $last.Name } + $lastJson = $null + try { $lastJson = Get-Content -LiteralPath $last.FullName -Raw | ConvertFrom-Json -ErrorAction Stop } catch {} + $lastPhase = if ($lastJson.name) { [string]$lastJson.name } else { [string]([IO.Path]::GetFileNameWithoutExtension($last.Name) -replace '^handshake-','') } + $lastAtUtc = if ($lastJson.atUtc) { [string]$lastJson.atUtc } else { $last.LastWriteTimeUtc.ToString('o') } + $lastStatus = if ($lastJson.status) { [string]$lastJson.status } else { $null } + $markerRel = @() + foreach ($file in $sortedHandshake) { + $relativePath = try { ($file.FullName).Substring(((Get-Location).Path).Length).TrimStart('\','/') } catch { $file.Name } + $markerRel += $relativePath + } + if (-not $idx['runContext']) { $idx['runContext'] = [ordered]@{} } + $idx.runContext['handshake'] = [ordered]@{ + lastPhase = $lastPhase + lastAtUtc = $lastAtUtc + lastStatus = $lastStatus + markerPaths = $markerRel + } + try { + $handshakeLines = @() + $handshakeLines += ("- Handshake Last Phase: {0}" -f $lastPhase) + if ($lastStatus) { $handshakeLines += ("- Handshake Last Status: {0}" -f $lastStatus) } + $firstTwo = @($markerRel | Select-Object -First 2) + foreach ($marker in $firstTwo) { $handshakeLines += ("- Marker: {0}" -f $marker) } + if ($idx['stepSummary']) { + $idx['stepSummary'] = $idx['stepSummary'] + "`n`n" + ($handshakeLines -join "`n") + } else { + $idx['stepSummary'] = ($handshakeLines -join "`n") + } + } catch {} + } + } catch {} + + $dest = Join-Path $ResultsDirectory 'session-index.json' + $idx | ConvertTo-Json -Depth 6 | Out-File -FilePath $dest -Encoding utf8 -ErrorAction Stop + Write-Host ("Session index written to: {0}" -f $dest) -ForegroundColor Gray + return $dest +} + +$resolvedContextPath = [System.IO.Path]::GetFullPath($ContextPath) +$context = Read-JsonObject -PathValue $resolvedContextPath +$resultsDir = [System.IO.Path]::GetFullPath([string]$context.resultsDir) +$repoRoot = [System.IO.Path]::GetFullPath([string]$context.repoRoot) +$jsonSummaryLeaf = if ([string]::IsNullOrWhiteSpace([string]$context.jsonSummaryPath)) { 'pester-summary.json' } else { [string]$context.jsonSummaryPath } +$summaryPath = Join-Path $resultsDir 'pester-summary.txt' +$summaryJsonPath = Join-Path $resultsDir $jsonSummaryLeaf +$artifactTrailPath = Join-Path $resultsDir 'pester-artifacts-trail.json' +$summaryTextValue = if ($context.PSObject.Properties['summaryText']) { [string]$context.summaryText } else { $null } +$hasSummaryPayload = [bool]$context.PSObject.Properties['summaryPayload'] +$hasArtifactTrail = [bool]$context.PSObject.Properties['artifactTrail'] + +if (-not (Test-Path -LiteralPath $resultsDir -PathType Container)) { + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null +} + +if (-not [string]::IsNullOrWhiteSpace($summaryTextValue)) { + $summaryTextValue | Out-File -FilePath $summaryPath -Encoding utf8 -ErrorAction Stop + Append-DiagnosticsFooterToSummary -SummaryPath $summaryPath -ResultsDirectory $resultsDir + Write-Host ("Summary written to: {0}" -f $summaryPath) -ForegroundColor Gray +} + +if ($hasSummaryPayload -and $null -ne $context.summaryPayload) { + $context.summaryPayload | ConvertTo-Json -Depth 12 | Out-File -FilePath $summaryJsonPath -Encoding utf8 -ErrorAction Stop + Write-Host ("JSON summary written to: {0}" -f $summaryJsonPath) -ForegroundColor Gray +} + +if ($hasArtifactTrail -and $null -ne $context.artifactTrail) { + $context.artifactTrail | ConvertTo-Json -Depth 8 | Out-File -FilePath $artifactTrailPath -Encoding utf8 -ErrorAction Stop + Write-Host ("Artifact trail written to: {0}" -f $artifactTrailPath) -ForegroundColor Gray +} + +Copy-CompareReportsAndWriteIndex -RepoRoot $repoRoot -ResultsDirectory $resultsDir +$sessionIndexPath = Write-SessionIndex -ResultsDirectory $resultsDir -SummaryJsonPath $jsonSummaryLeaf -IncludeIntegration ([bool]$context.includeIntegration) -IntegrationMode ([string]$context.integrationMode) -IntegrationSource ([string]$context.integrationSource) +$manifestPath = Write-ArtifactManifest -Directory $resultsDir -SummaryJsonPath $jsonSummaryLeaf -ManifestVersion ([string]$context.manifestVersion) -SummarySchemaVersion ([string]$context.summarySchemaVersion) -FailuresSchemaVersion ([string]$context.failuresSchemaVersion) -LeakReportSchemaVersion ([string]$context.leakReportSchemaVersion) -DiagnosticsSchemaVersion ([string]$context.diagnosticsSchemaVersion) + +if ($env:GITHUB_OUTPUT) { + "summary_path=$summaryPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "summary_json_path=$summaryJsonPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "session_index_path=$sessionIndexPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "manifest_path=$manifestPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 +} + +Write-Host '### Pester execution finalize' -ForegroundColor Cyan +Write-Host ("summary : {0}" -f $summaryPath) +Write-Host ("summary json : {0}" -f $summaryJsonPath) +Write-Host ("session idx : {0}" -f $sessionIndexPath) +Write-Host ("manifest : {0}" -f $manifestPath) + +exit 0 diff --git a/tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs b/tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs index 4f71a2154..a1e5c944f 100644 --- a/tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs +++ b/tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs @@ -21,6 +21,7 @@ test('package.json exposes the local execution harness as a first-class entrypoi test('local execution harness owns lock lifecycle, preflight, dispatch, and receipt generation', () => { const harness = readRepoFile('tools/Run-PesterExecutionOnly.Local.ps1'); + const invoker = readRepoFile('scripts/Pester-Invoker.psm1'); assert.match(harness, /\[string\]\$SessionLockRoot/); assert.match(harness, /\$resolvedSessionLockRoot = if \(\[string\]::IsNullOrWhiteSpace\(\$SessionLockRoot\)\)/); @@ -39,6 +40,23 @@ test('local execution harness owns lock lifecycle, preflight, dispatch, and rece assert.match(harness, /results-xml-truncated/); assert.match(harness, /summaryPresent/); assert.match(harness, /sessionLockRoot = ConvertTo-PortablePath \$resolvedSessionLockRoot/); + assert.match(invoker, /ConvertTo-Json -Depth 8 -Compress/); +}); + +test('dispatcher path delegates summary, artifact, and session-index side effects to the execution finalize helper', () => { + const dispatcher = readRepoFile('Invoke-PesterTests.ps1'); + const finalize = readRepoFile('tools/Invoke-PesterExecutionFinalize.ps1'); + + assert.match(dispatcher, /Invoke-PesterExecutionFinalize\.ps1/); + assert.match(dispatcher, /pester-execution-finalize-context@v1/); + assert.match(dispatcher, /Invoke-ExecutionFinalizeHelper\s+-SummaryText\s+\$summary\s+-SummaryPayload\s+\$jsonObj\s+-ArtifactTrail\s+\$script:artifactTrail/); + assert.match(dispatcher, /Invoke-ExecutionFinalizeHelper\s+-ReuseExistingContext/); + assert.doesNotMatch(dispatcher, /Write-ArtifactManifest\s+-Directory/); + assert.doesNotMatch(dispatcher, /Write-SessionIndex\s+-ResultsDirectory/); + assert.match(finalize, /pester-artifacts\.json/); + assert.match(finalize, /session-index\.json/); + assert.match(finalize, /compare-report\.html/); + assert.match(finalize, /results-index\.html/); }); test('knowledgebase documents the local harness as the workflow-shell-free execution entrypoint', () => { From 304d82aae7f885dd2894aec46158ddf6a3225672 Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Tue, 31 Mar 2026 21:44:16 -0700 Subject: [PATCH 34/44] Add local proof autonomy packets and Windows staging contracts (#2087) Co-authored-by: svelderrainruiz --- .github/workflows/pester-evidence.yml | 213 ++-- .github/workflows/pester-run.yml | 82 +- .github/workflows/pester-selection.yml | 55 +- .../pester-service-model-release-evidence.yml | 16 + Invoke-PesterTests.ps1 | 659 ++++------- ...al-proof-autonomy-program-control-plane.md | 38 + .../pester-service-model-control-plane.md | 54 +- .../vi-history-local-proof-control-plane.md | 40 + ...ows-docker-shared-surface-control-plane.md | 44 + .../Local-Proof-Autonomy-Program.md | 75 ++ docs/knowledgebase/Pester-Service-Model.md | 76 +- docs/knowledgebase/VI-History-Local-Proof.md | 103 ++ .../Windows-Docker-Shared-Surface.md | 65 ++ ...er-service-model-promotion-comparison.json | 48 + ...ements-local-proof-autonomy-program-srs.md | 37 + docs/requirements-pester-service-model-srs.md | 35 +- ...requirements-vi-history-local-proof-srs.md | 44 + ...ments-windows-docker-shared-surface-srs.md | 44 + docs/rtm-local-proof-autonomy-program.csv | 5 + docs/rtm-pester-service-model.csv | 21 + docs/rtm-vi-history-local-proof.csv | 10 + docs/rtm-windows-docker-shared-surface.csv | 9 + ...evi-local-program-ci-report-v1.schema.json | 218 ++++ ...evi-local-program-next-step-v1.schema.json | 141 +++ .../pester-derived-provenance-v1.schema.json | 122 ++ ...pester-promotion-comparison-v1.schema.json | 86 ++ ...rvice-model-local-ci-report-v1.schema.json | 297 +++++ ...ter-service-model-next-step-v1.schema.json | 132 +++ ...ry-live-candidate-readiness-v1.schema.json | 96 ++ .../vi-history-live-candidate-v1.schema.json | 77 ++ .../vi-history-local-ci-report-v1.schema.json | 231 ++++ .../vi-history-local-next-step-v1.schema.json | 132 +++ ...red-surface-local-ci-report-v1.schema.json | 164 +++ ...er-shared-surface-next-step-v1.schema.json | 58 + .../local-proof-autonomy-program-test-plan.md | 27 + .../testing/pester-service-model-test-plan.md | 52 +- .../vi-history-local-proof-test-plan.md | 47 + ...windows-docker-shared-surface-test-plan.md | 44 + package.json | 20 +- scripts/Write-PesterSummaryToStepSummary.ps1 | 95 +- ...oke-PesterEvidenceClassification.Tests.ps1 | 124 ++ .../Invoke-PesterEvidenceProvenance.Tests.ps1 | 113 ++ .../Invoke-PesterExecutionFinalize.Tests.ps1 | 175 ++- ...nvoke-PesterExecutionPostprocess.Tests.ps1 | 76 ++ ...nvoke-PesterExecutionPublication.Tests.ps1 | 99 ++ .../Invoke-PesterExecutionTelemetry.Tests.ps1 | 89 ++ tests/Invoke-PesterOperatorOutcome.Tests.ps1 | 70 ++ ...sterWindowsContainerSurfaceProbe.Tests.ps1 | 48 + tests/PesterExecutionPacks.Tests.ps1 | 32 + tests/PesterFailurePayloadShape.Tests.ps1 | 169 +++ ...PesterFailureProducerConsistency.Tests.ps1 | 62 + tests/PesterPathHygiene.Tests.ps1 | 44 + tests/PesterServiceModelSchema.Tests.ps1 | 47 + tests/PesterSummary.Context.Tests.ps1 | 7 +- ...esterServiceModelArtifacts.Local.Tests.ps1 | 132 +++ ...rviceModelRepresentativeArtifact.Tests.ps1 | 39 + tests/Run-NIWindowsContainerCompare.Tests.ps1 | 39 + ...rExecutionOnly.Local.PathHygiene.Tests.ps1 | 91 ++ ...SummaryToStepSummary.CompactMode.Tests.ps1 | 65 ++ ...Write-PesterSummaryToStepSummary.Tests.ps1 | 81 +- .../pester-run-receipt.json | 13 + .../raw/dispatcher-events.ndjson | 2 + .../raw/pester-failures.json | 20 + .../raw/pester-summary.json | 15 + tools/Invoke-CompareCli.ps1 | 25 +- tools/Invoke-PesterEvidenceClassification.ps1 | 249 ++++ tools/Invoke-PesterEvidenceProvenance.ps1 | 370 ++++++ tools/Invoke-PesterExecutionFinalize.ps1 | 124 +- tools/Invoke-PesterExecutionPostprocess.ps1 | 72 +- tools/Invoke-PesterExecutionPublication.ps1 | 186 +++ tools/Invoke-PesterExecutionTelemetry.ps1 | 254 ++++ tools/Invoke-PesterOperatorOutcome.ps1 | 220 ++++ ...oke-PesterWindowsContainerSurfaceProbe.ps1 | 154 +++ tools/PesterExecutionPacks.ps1 | 154 +++ tools/PesterFailurePayload.ps1 | 317 +++++ tools/PesterPathHygiene.ps1 | 154 +++ tools/PesterServiceModelSchema.ps1 | 167 +++ tools/Print-PesterTopFailures.ps1 | 31 +- ...play-PesterServiceModelArtifacts.Local.ps1 | 238 ++++ tools/Run-NIWindowsContainerCompare.ps1 | 295 ++++- tools/Run-PesterExecutionOnly.Local.ps1 | 199 +++- tools/Test-WindowsNI2026q1HostPreflight.ps1 | 5 +- tools/Write-PesterTopFailures.ps1 | 74 +- tools/Write-PesterTotals.ps1 | 63 + .../comparevi-local-program-ci.test.mjs | 239 ++++ .../pester-service-model-local-ci.test.mjs | 208 ++++ ...vice-model-local-harness-contract.test.mjs | 222 +++- ...model-release-evidence-provenance.test.mjs | 120 ++ ...elease-evidence-workflow-contract.test.mjs | 6 + ...r-service-model-workflow-contract.test.mjs | 92 +- .../__tests__/vi-history-local-ci.test.mjs | 219 ++++ .../vi-history-local-proof-contract.test.mjs | 114 ++ ...ws-docker-shared-surface-contract.test.mjs | 100 ++ ...ws-docker-shared-surface-local-ci.test.mjs | 205 ++++ .../__tests__/windows-host-bridge.test.mjs | 132 +++ tools/priority/comparevi-local-program-ci.mjs | 450 ++++++++ ...-pester-service-model-release-evidence.mjs | 108 +- .../pester-service-model-audit-surface.yaml | 67 ++ .../pester-service-model-autonomy-policy.json | 80 ++ .../pester-service-model-local-ci.mjs | 807 +++++++++++++ .../pester-service-model-provenance.mjs | 114 ++ ...pester-service-model-promotion-dossier.mjs | 117 +- tools/priority/vi-history-live-candidate.json | 21 + tools/priority/vi-history-local-ci.mjs | 1017 +++++++++++++++++ .../vi-history-local-proof-audit-surface.yaml | 42 + ...i-history-local-proof-autonomy-policy.json | 37 + ...s-docker-shared-surface-audit-surface.yaml | 29 + ...docker-shared-surface-autonomy-policy.json | 36 + ...windows-docker-shared-surface-local-ci.mjs | 968 ++++++++++++++++ tools/priority/windows-host-bridge.mjs | 240 ++++ 110 files changed, 13845 insertions(+), 760 deletions(-) create mode 100644 docs/architecture/local-proof-autonomy-program-control-plane.md create mode 100644 docs/architecture/vi-history-local-proof-control-plane.md create mode 100644 docs/architecture/windows-docker-shared-surface-control-plane.md create mode 100644 docs/knowledgebase/Local-Proof-Autonomy-Program.md create mode 100644 docs/knowledgebase/VI-History-Local-Proof.md create mode 100644 docs/knowledgebase/Windows-Docker-Shared-Surface.md create mode 100644 docs/pester-service-model-promotion-comparison.json create mode 100644 docs/requirements-local-proof-autonomy-program-srs.md create mode 100644 docs/requirements-vi-history-local-proof-srs.md create mode 100644 docs/requirements-windows-docker-shared-surface-srs.md create mode 100644 docs/rtm-local-proof-autonomy-program.csv create mode 100644 docs/rtm-vi-history-local-proof.csv create mode 100644 docs/rtm-windows-docker-shared-surface.csv create mode 100644 docs/schemas/comparevi-local-program-ci-report-v1.schema.json create mode 100644 docs/schemas/comparevi-local-program-next-step-v1.schema.json create mode 100644 docs/schemas/pester-derived-provenance-v1.schema.json create mode 100644 docs/schemas/pester-promotion-comparison-v1.schema.json create mode 100644 docs/schemas/pester-service-model-local-ci-report-v1.schema.json create mode 100644 docs/schemas/pester-service-model-next-step-v1.schema.json create mode 100644 docs/schemas/vi-history-live-candidate-readiness-v1.schema.json create mode 100644 docs/schemas/vi-history-live-candidate-v1.schema.json create mode 100644 docs/schemas/vi-history-local-ci-report-v1.schema.json create mode 100644 docs/schemas/vi-history-local-next-step-v1.schema.json create mode 100644 docs/schemas/windows-docker-shared-surface-local-ci-report-v1.schema.json create mode 100644 docs/schemas/windows-docker-shared-surface-next-step-v1.schema.json create mode 100644 docs/testing/local-proof-autonomy-program-test-plan.md create mode 100644 docs/testing/vi-history-local-proof-test-plan.md create mode 100644 docs/testing/windows-docker-shared-surface-test-plan.md create mode 100644 tests/Invoke-PesterEvidenceClassification.Tests.ps1 create mode 100644 tests/Invoke-PesterEvidenceProvenance.Tests.ps1 create mode 100644 tests/Invoke-PesterExecutionPublication.Tests.ps1 create mode 100644 tests/Invoke-PesterExecutionTelemetry.Tests.ps1 create mode 100644 tests/Invoke-PesterOperatorOutcome.Tests.ps1 create mode 100644 tests/Invoke-PesterWindowsContainerSurfaceProbe.Tests.ps1 create mode 100644 tests/PesterExecutionPacks.Tests.ps1 create mode 100644 tests/PesterFailurePayloadShape.Tests.ps1 create mode 100644 tests/PesterFailureProducerConsistency.Tests.ps1 create mode 100644 tests/PesterPathHygiene.Tests.ps1 create mode 100644 tests/PesterServiceModelSchema.Tests.ps1 create mode 100644 tests/Replay-PesterServiceModelArtifacts.Local.Tests.ps1 create mode 100644 tests/Replay-PesterServiceModelRepresentativeArtifact.Tests.ps1 create mode 100644 tests/Run-PesterExecutionOnly.Local.PathHygiene.Tests.ps1 create mode 100644 tests/fixtures/pester-service-model/legacy-results-xml-truncated/pester-run-receipt.json create mode 100644 tests/fixtures/pester-service-model/legacy-results-xml-truncated/raw/dispatcher-events.ndjson create mode 100644 tests/fixtures/pester-service-model/legacy-results-xml-truncated/raw/pester-failures.json create mode 100644 tests/fixtures/pester-service-model/legacy-results-xml-truncated/raw/pester-summary.json create mode 100644 tools/Invoke-PesterEvidenceClassification.ps1 create mode 100644 tools/Invoke-PesterEvidenceProvenance.ps1 create mode 100644 tools/Invoke-PesterExecutionPublication.ps1 create mode 100644 tools/Invoke-PesterExecutionTelemetry.ps1 create mode 100644 tools/Invoke-PesterOperatorOutcome.ps1 create mode 100644 tools/Invoke-PesterWindowsContainerSurfaceProbe.ps1 create mode 100644 tools/PesterExecutionPacks.ps1 create mode 100644 tools/PesterFailurePayload.ps1 create mode 100644 tools/PesterPathHygiene.ps1 create mode 100644 tools/PesterServiceModelSchema.ps1 create mode 100644 tools/Replay-PesterServiceModelArtifacts.Local.ps1 create mode 100644 tools/Write-PesterTotals.ps1 create mode 100644 tools/priority/__tests__/comparevi-local-program-ci.test.mjs create mode 100644 tools/priority/__tests__/pester-service-model-local-ci.test.mjs create mode 100644 tools/priority/__tests__/pester-service-model-release-evidence-provenance.test.mjs create mode 100644 tools/priority/__tests__/vi-history-local-ci.test.mjs create mode 100644 tools/priority/__tests__/vi-history-local-proof-contract.test.mjs create mode 100644 tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs create mode 100644 tools/priority/__tests__/windows-docker-shared-surface-local-ci.test.mjs create mode 100644 tools/priority/__tests__/windows-host-bridge.test.mjs create mode 100644 tools/priority/comparevi-local-program-ci.mjs create mode 100644 tools/priority/pester-service-model-audit-surface.yaml create mode 100644 tools/priority/pester-service-model-autonomy-policy.json create mode 100644 tools/priority/pester-service-model-local-ci.mjs create mode 100644 tools/priority/pester-service-model-provenance.mjs create mode 100644 tools/priority/vi-history-live-candidate.json create mode 100644 tools/priority/vi-history-local-ci.mjs create mode 100644 tools/priority/vi-history-local-proof-audit-surface.yaml create mode 100644 tools/priority/vi-history-local-proof-autonomy-policy.json create mode 100644 tools/priority/windows-docker-shared-surface-audit-surface.yaml create mode 100644 tools/priority/windows-docker-shared-surface-autonomy-policy.json create mode 100644 tools/priority/windows-docker-shared-surface-local-ci.mjs create mode 100644 tools/priority/windows-host-bridge.mjs diff --git a/.github/workflows/pester-evidence.yml b/.github/workflows/pester-evidence.yml index 5043e8817..9feaea609 100644 --- a/.github/workflows/pester-evidence.yml +++ b/.github/workflows/pester-evidence.yml @@ -138,7 +138,7 @@ jobs: uses: actions/download-artifact@v5 with: name: ${{ steps.artifact_name.outputs.name }} - path: . + path: tests/results - name: Ensure results directory shell: pwsh @@ -158,19 +158,24 @@ jobs: "status=missing" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 exit 0 } - $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json -ErrorAction Stop - if ($receipt.schema -ne 'pester-execution-receipt@v1') { - throw ("Unexpected execution receipt schema: {0}" -f $receipt.schema) + . (Join-Path (Get-Location) 'tools/PesterServiceModelSchema.ps1') + $receiptState = Test-PesterServiceModelSchemaContract ` + -DocumentState (Read-PesterServiceModelJsonDocument -PathValue $receiptPath -ContractName 'execution-receipt') ` + -ExpectedSchema 'pester-execution-receipt@v1' + if (-not $receiptState.valid) { + "present=true" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "status=unsupported-schema" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "dispatcher_exit_code=-1" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "execution_pack=" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "execution_pack_source=" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + exit 0 } + $receipt = $receiptState.document "present=true" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 "status=$($receipt.status)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 "dispatcher_exit_code=$($receipt.dispatcherExitCode)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 - - - name: Publish Pester summary - if: always() - continue-on-error: true - shell: pwsh - run: pwsh -File scripts/Write-PesterSummaryToStepSummary.ps1 -ResultsDir 'tests/results' + "execution_pack=$($receipt.selectionExecutionPack)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "execution_pack_source=$($receipt.selectionExecutionPackSource)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 - name: Validate Pester summary schema-lite (notice-only) if: always() @@ -240,148 +245,67 @@ jobs: upload: true artifact-name: session-index - - name: Append session summary + - name: Write compact totals JSON if: always() - continue-on-error: true shell: pwsh - run: pwsh -File tools/Write-SessionIndexSummary.ps1 -ResultsDir 'tests/results' + run: pwsh -File tools/Write-PesterTotals.ps1 -ResultsDir 'tests/results' - - name: Append top Pester failures + - name: Classify evidence outcome + id: classify if: always() - continue-on-error: true shell: pwsh - run: pwsh -File tools/Write-PesterTopFailures.ps1 -ResultsDir 'tests/results' -Top 10 + run: | + $rawArtifactDownload = if ('${{ steps.artifact_name.outputs.should_download }}' -eq 'true') { '${{ steps.download.outcome }}' } else { 'skipped' } + pwsh -File tools/Invoke-PesterEvidenceClassification.ps1 ` + -ResultsDir 'tests/results' ` + -ExecutionReceiptPath 'tests/execution-contract/pester-run-receipt.json' ` + -ContextStatus '${{ inputs.context_status }}' ` + -ReadinessStatus '${{ inputs.readiness_status }}' ` + -SelectionStatus '${{ inputs.selection_status }}' ` + -ExecutionJobResult '${{ inputs.execution_job_result }}' ` + -DispatcherExitCode '${{ inputs.dispatcher_exit_code }}' ` + -RawArtifactDownload $rawArtifactDownload - - name: Write compact totals JSON + - name: Generate operator outcome + id: operator_outcome if: always() shell: pwsh run: | - $outDir = 'tests/results' - $sum = Join-Path $outDir 'pester-summary.json' - $obj = [ordered]@{ - schema = 'pester-totals/v1' - includeIntegration = $null - status = 'missing-summary' - } - if (Test-Path $sum) { - try { - $js = Get-Content $sum -Raw | ConvertFrom-Json -ErrorAction Stop - $obj.total = $js.total - $obj.passed = $js.passed - $obj.failed = $js.failed - $obj.errors = $js.errors - $obj.duration_s = $js.duration_s - $obj.status = if (($js.failed + $js.errors) -gt 0) { 'fail' } else { 'ok' } - } catch { - $obj.status = 'unknown' - } - } - $obj | ConvertTo-Json -Depth 5 | Out-File -FilePath (Join-Path $outDir 'pester-totals.json') -Encoding utf8 + pwsh -File tools/Invoke-PesterOperatorOutcome.ps1 ` + -ResultsDir 'tests/results' ` + -ContinueOnError '${{ inputs.continue_on_error }}' - - name: Classify evidence outcome - id: classify + - name: Generate evidence provenance + id: evidence_provenance if: always() shell: pwsh run: | - $resultsDir = 'tests/results' - $summaryPath = Join-Path $resultsDir 'pester-summary.json' - $classification = 'seam-defect' - $reasons = New-Object System.Collections.Generic.List[string] - $contextStatus = '${{ inputs.context_status }}' - $readinessStatus = '${{ inputs.readiness_status }}' - $selectionStatus = '${{ inputs.selection_status }}' - $executionJobResult = '${{ inputs.execution_job_result }}' - $executionReceiptPresent = '${{ steps.execution_receipt.outputs.present }}' - $executionReceiptStatus = '${{ steps.execution_receipt.outputs.status }}' - if ($contextStatus -ne 'ready') { - $reasons.Add(("context-status={0}" -f $contextStatus)) | Out-Null - } - if ($readinessStatus -ne 'ready') { - $reasons.Add(("readiness-status={0}" -f $readinessStatus)) | Out-Null - } - if ($selectionStatus -ne 'ready') { - $reasons.Add(("selection-status={0}" -f $selectionStatus)) | Out-Null - } - if ($executionJobResult -eq 'skipped') { - $reasons.Add('execution-job-skipped') | Out-Null - } elseif ($executionJobResult -eq 'cancelled') { - $reasons.Add('execution-job-cancelled') | Out-Null - } elseif ($executionJobResult -eq 'results-xml-truncated') { - $reasons.Add('execution-job-results-xml-truncated') | Out-Null - } elseif ($executionJobResult -eq 'invalid-results-xml') { - $reasons.Add('execution-job-invalid-results-xml') | Out-Null - } elseif ($executionJobResult -eq 'missing-results-xml') { - $reasons.Add('execution-job-missing-results-xml') | Out-Null - } elseif ($executionJobResult -eq 'seam-defect') { - $reasons.Add('execution-job-seam-defect') | Out-Null - } elseif ($executionJobResult -eq 'unknown') { - $reasons.Add('execution-job-unknown') | Out-Null - } - if ('${{ steps.artifact_name.outputs.should_download }}' -eq 'true' -and '${{ steps.download.outcome }}' -ne 'success') { - $reasons.Add(("raw-artifact-download={0}" -f '${{ steps.download.outcome }}')) | Out-Null - } - $dispatcherExitCode = '${{ inputs.dispatcher_exit_code }}' - if ([string]::IsNullOrWhiteSpace($dispatcherExitCode)) { $dispatcherExitCode = '-1' } - if ($executionReceiptPresent -ne 'true') { - $reasons.Add('execution-receipt-missing') | Out-Null - } elseif (($contextStatus -ne 'ready' -and $executionJobResult -in @('skipped','cancelled')) -or $executionReceiptStatus -eq 'context-blocked') { - $classification = 'context-blocked' - } elseif ($readinessStatus -ne 'ready' -and $executionJobResult -in @('skipped','cancelled')) { - $classification = 'readiness-blocked' - } elseif (($selectionStatus -ne 'ready' -and $executionJobResult -in @('skipped','cancelled')) -or $executionReceiptStatus -eq 'selection-blocked') { - $classification = 'selection-blocked' - } elseif ($executionReceiptStatus -eq 'results-xml-truncated') { - $classification = 'results-xml-truncated' - $reasons.Add('execution-receipt-results-xml-truncated') | Out-Null - } elseif ($executionReceiptStatus -eq 'invalid-results-xml') { - $classification = 'invalid-results-xml' - $reasons.Add('execution-receipt-invalid-results-xml') | Out-Null - } elseif ($executionReceiptStatus -eq 'missing-results-xml') { - $classification = 'missing-results-xml' - $reasons.Add('execution-receipt-missing-results-xml') | Out-Null - } elseif ($executionReceiptStatus -eq 'seam-defect') { - $reasons.Add('execution-receipt-seam-defect') | Out-Null - } elseif ($executionReceiptStatus -eq 'test-failures') { - $classification = 'test-failures' - } elseif (Test-Path -LiteralPath $summaryPath) { - try { - $summary = Get-Content -LiteralPath $summaryPath -Raw | ConvertFrom-Json -ErrorAction Stop - if ('${{ steps.execution_receipt.outputs.dispatcher_exit_code }}' -and '${{ steps.execution_receipt.outputs.dispatcher_exit_code }}' -ne $dispatcherExitCode) { - $reasons.Add('dispatcher-exit-mismatch') | Out-Null - } - if (($summary.PSObject.Properties.Name -contains 'resultsXmlStatus') -and [string]$summary.resultsXmlStatus -like 'truncated*') { - $classification = 'results-xml-truncated' - $reasons.Add(("results-xml-status={0}" -f [string]$summary.resultsXmlStatus)) | Out-Null - } elseif (($summary.PSObject.Properties.Name -contains 'resultsXmlStatus') -and [string]$summary.resultsXmlStatus -like 'invalid*') { - $classification = 'invalid-results-xml' - $reasons.Add(("results-xml-status={0}" -f [string]$summary.resultsXmlStatus)) | Out-Null - } elseif (($summary.failed + $summary.errors) -gt 0 -or $dispatcherExitCode -ne '0') { - $classification = 'test-failures' - } else { - $classification = 'ok' - } - } catch { - $classification = 'seam-defect' - $reasons.Add('summary-unparseable') | Out-Null - } - } else { - $reasons.Add('summary-missing') | Out-Null - } - $receipt = [ordered]@{ - schema = 'pester-evidence-classification@v1' - generatedAtUtc = [DateTime]::UtcNow.ToString('o') - contextStatus = $contextStatus - readinessStatus = $readinessStatus - selectionStatus = $selectionStatus - executionJobResult = $executionJobResult - rawArtifactDownload = if ('${{ steps.artifact_name.outputs.should_download }}' -eq 'true') { '${{ steps.download.outcome }}' } else { 'skipped' } - dispatcherExitCode = [int]$dispatcherExitCode - summaryPresent = Test-Path -LiteralPath $summaryPath - classification = $classification - reasons = @($reasons) - } - $receipt | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-evidence-classification.json') -Encoding UTF8 - "classification=$classification" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + $rawArtifactDownload = if ('${{ steps.artifact_name.outputs.should_download }}' -eq 'true') { '${{ steps.download.outcome }}' } else { 'skipped' } + pwsh -File tools/Invoke-PesterEvidenceProvenance.ps1 ` + -ResultsDir 'tests/results' ` + -ExecutionReceiptPath 'tests/execution-contract/pester-run-receipt.json' ` + -RawArtifactName '${{ steps.artifact_name.outputs.name }}' ` + -RawArtifactDownload $rawArtifactDownload ` + -ExecutionReceiptArtifactName '${{ inputs.execution_receipt_artifact_name }}' ` + -OutputPath 'tests/results/pester-evidence-provenance.json' + + - name: Publish Pester summary + if: always() + continue-on-error: true + shell: pwsh + run: pwsh -File scripts/Write-PesterSummaryToStepSummary.ps1 -ResultsDir 'tests/results' + + - name: Append session summary + if: always() + continue-on-error: true + shell: pwsh + run: pwsh -File tools/Write-SessionIndexSummary.ps1 -ResultsDir 'tests/results' + + - name: Append top Pester failures + if: always() + continue-on-error: true + shell: pwsh + run: pwsh -File tools/Write-PesterTopFailures.ps1 -ResultsDir 'tests/results' -Top 10 - name: Generate dev dashboard report if: always() @@ -399,4 +323,13 @@ jobs: - name: Propagate gate outcome if: ${{ steps.classify.outputs.classification != 'ok' && inputs.continue_on_error != 'true' }} - run: exit 1 + shell: bash + run: | + echo "::error title=Pester gate outcome::classification=${{ steps.classify.outputs.classification }};next_action=${{ steps.operator_outcome.outputs.next_action }}" + if [[ -f tests/results/pester-evidence-classification.json ]]; then + cat tests/results/pester-evidence-classification.json + fi + if [[ -f tests/results/pester-operator-outcome.json ]]; then + cat tests/results/pester-operator-outcome.json + fi + exit 1 diff --git a/.github/workflows/pester-run.yml b/.github/workflows/pester-run.yml index b318cb6ae..1aee154a4 100644 --- a/.github/workflows/pester-run.yml +++ b/.github/workflows/pester-run.yml @@ -121,6 +121,8 @@ jobs: dispatcher_exit_code: ${{ steps.execution_receipt.outputs.dispatcher_exit_code }} raw_artifact_name: ${{ steps.execution_receipt.outputs.raw_artifact_name }} execution_receipt_status: ${{ steps.execution_receipt.outputs.status }} + selection_execution_pack: ${{ steps.selection_receipt.outputs.execution_pack }} + selection_execution_pack_source: ${{ steps.selection_receipt.outputs.execution_pack_source }} steps: - uses: actions/checkout@v5 with: @@ -211,12 +213,17 @@ jobs: if ($receipt.status -ne 'ready') { throw ("Selection receipt status is not ready: {0}" -f $receipt.status) } - $patternsJson = @($receipt.selection.includePatterns) | ConvertTo-Json -Compress - if (-not $patternsJson) { $patternsJson = '[]' } + $refinePatternsJson = @($receipt.selection.refineIncludePatterns) | ConvertTo-Json -Compress + if (-not $refinePatternsJson) { $refinePatternsJson = '[]' } + $effectivePatternsJson = @($receipt.selection.effectiveIncludePatterns) | ConvertTo-Json -Compress + if (-not $effectivePatternsJson) { $effectivePatternsJson = '[]' } "path=$receiptPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "execution_pack=$($receipt.selection.executionPack)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "execution_pack_source=$($receipt.selection.executionPackSource)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 "integration_mode=$($receipt.selection.integrationMode)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 "fixture_required=$(([string]$receipt.selection.fixtureRequired).ToLowerInvariant())" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 - "include_patterns_json=$patternsJson" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "refine_include_patterns_json=$refinePatternsJson" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "effective_include_patterns_json=$effectivePatternsJson" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 "timeout_seconds=$($receipt.dispatcherProfile.timeoutSeconds)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 "emit_failures_json_always=$($receipt.dispatcherProfile.emitFailuresJsonAlways)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 "detect_leaks=$($receipt.dispatcherProfile.detectLeaks)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 @@ -292,6 +299,10 @@ jobs: } $bound = [ordered]@{} $bound.TestsPath = 'tests' + $executionPack = '${{ steps.selection_receipt.outputs.execution_pack }}' + if ($executionPack) { + $bound.ExecutionPack = $executionPack + } $integrationMode = '${{ steps.selection_receipt.outputs.integration_mode }}' if ($integrationMode) { $bound.IntegrationMode = $integrationMode @@ -299,7 +310,7 @@ jobs: $bound.ResultsPath = 'tests/results' if ($env:DISPATCHER_LIVE_OUTPUT -ne '0') { $bound.LiveOutput = $true } if ('${{ steps.dprofile.outputs.emit_failures_json_always }}' -eq 'true') { $bound.EmitFailuresJsonAlways = $true } - $patterns = '${{ steps.selection_receipt.outputs.include_patterns_json }}' | ConvertFrom-Json -ErrorAction Stop + $patterns = '${{ steps.selection_receipt.outputs.refine_include_patterns_json }}' | ConvertFrom-Json -ErrorAction Stop if ($patterns -and @($patterns).Count -gt 0) { $bound.IncludePatterns = @($patterns) } @@ -308,6 +319,12 @@ jobs: $bound.TimeoutSeconds = [double]$timeoutSeconds } $lockScript = Join-Path (Get-Location) 'tools/Session-Lock.ps1' + $stepOutputPath = $env:GITHUB_OUTPUT + $dispatcherOutputTrace = Join-Path (Get-Location) 'tests/results/dispatcher-github-output.txt' + $dispatcherOutputDir = Split-Path -Parent $dispatcherOutputTrace + if ($dispatcherOutputDir -and -not (Test-Path -LiteralPath $dispatcherOutputDir)) { + New-Item -ItemType Directory -Path $dispatcherOutputDir -Force | Out-Null + } $heartbeatSeconds = 15 $heartbeatJob = Start-ThreadJob -ScriptBlock { param($scriptPath,$seconds) @@ -331,15 +348,13 @@ jobs: } pwsh -NoLogo -NoProfile -File $lockScript -Action Heartbeat | Out-Null } - if (-not $env:GITHUB_OUTPUT) { - $fallbackOutput = Join-Path (Get-Location) 'tests/results/dispatcher-github-output.txt' - $fallbackDir = Split-Path -Parent $fallbackOutput - if (-not (Test-Path -LiteralPath $fallbackDir)) { - New-Item -ItemType Directory -Path $fallbackDir -Force | Out-Null - } - $env:GITHUB_OUTPUT = [System.IO.Path]::GetFullPath($fallbackOutput) + if (-not [string]::IsNullOrWhiteSpace($stepOutputPath)) { + $env:GITHUB_OUTPUT = $stepOutputPath + } else { + $env:GITHUB_OUTPUT = [System.IO.Path]::GetFullPath($dispatcherOutputTrace) } "exit_code=$exitCode" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "exit_code=$exitCode" | Out-File -FilePath $dispatcherOutputTrace -Append -Encoding utf8 if ($exitCode -ne 0) { Write-Warning ("Dispatcher exited with code {0}" -f $exitCode) } @@ -356,6 +371,17 @@ jobs: } pwsh -NoLogo -NoProfile -File $toolPath -ResultsDir 'tests/results' + - name: Materialize execution telemetry + id: telemetry + if: always() + shell: pwsh + run: | + $toolPath = Join-Path (Get-Location) 'tools/Invoke-PesterExecutionTelemetry.ps1' + if (-not (Test-Path -LiteralPath $toolPath -PathType Leaf)) { + throw "Telemetry tool not found at $toolPath" + } + pwsh -NoLogo -NoProfile -File $toolPath -ResultsDir 'tests/results' + - name: Release session lock if: always() shell: pwsh @@ -372,10 +398,21 @@ jobs: } $summaryPath = Join-Path $resultsDir 'pester-summary.json' $postprocessPath = Join-Path $resultsDir 'pester-execution-postprocess.json' + $telemetryPath = Join-Path $resultsDir 'pester-execution-telemetry.json' $dispatcherExitCode = '${{ steps.dispatcher.outputs.exit_code }}' + $dispatcherOutputTrace = Join-Path $resultsDir 'dispatcher-github-output.txt' $status = 'seam-defect' $postprocessStatus = '' $resultsXmlStatus = '' + $telemetryStatus = '' + $telemetryLastKnownPhase = '' + $telemetryEventCount = 0 + if ($dispatcherExitCode -eq '' -and (Test-Path -LiteralPath $dispatcherOutputTrace)) { + $traceLine = Get-Content -LiteralPath $dispatcherOutputTrace -ErrorAction SilentlyContinue | Select-Object -Last 1 + if ($traceLine -match '^exit_code=(?-?\d+)$') { + $dispatcherExitCode = $matches.value + } + } if ($dispatcherExitCode -eq '') { $dispatcherExitCode = '-1' } @@ -391,7 +428,17 @@ jobs: Write-Warning ("Failed to parse postprocess report: {0}" -f $_.Exception.Message) } } - if ($postprocessStatus -in @('results-xml-truncated', 'invalid-results-xml', 'missing-results-xml')) { + if (Test-Path -LiteralPath $telemetryPath) { + try { + $telemetry = Get-Content -LiteralPath $telemetryPath -Raw | ConvertFrom-Json -ErrorAction Stop + $telemetryStatus = [string]$telemetry.telemetryStatus + $telemetryLastKnownPhase = [string]$telemetry.lastKnownPhase + $telemetryEventCount = [int]$telemetry.eventCount + } catch { + Write-Warning ("Failed to parse execution telemetry report: {0}" -f $_.Exception.Message) + } + } + if ($postprocessStatus -in @('results-xml-truncated', 'invalid-results-xml', 'missing-results-xml', 'unsupported-schema')) { $status = $postprocessStatus } elseif ((Test-Path -LiteralPath $summaryPath) -and $dispatcherExitCode -eq '0') { $status = 'completed' @@ -414,9 +461,16 @@ jobs: selectionStatus = '${{ inputs.selection_status }}' selectionReceiptPath = '${{ steps.selection_receipt.outputs.path }}' selectionReceiptPresent = $selectionReceiptPresent + selectionExecutionPack = '${{ steps.selection_receipt.outputs.execution_pack }}' + selectionExecutionPackSource = '${{ steps.selection_receipt.outputs.execution_pack_source }}' selectionIntegrationMode = '${{ steps.selection_receipt.outputs.integration_mode }}' selectionFixtureRequired = '${{ steps.selection_receipt.outputs.fixture_required }}' dispatcherExitCode = [int]$dispatcherExitCode + telemetryPath = $telemetryPath + telemetryPresent = Test-Path -LiteralPath $telemetryPath + telemetryStatus = $telemetryStatus + telemetryLastKnownPhase = $telemetryLastKnownPhase + telemetryEventCount = $telemetryEventCount postprocessStatus = $postprocessStatus resultsXmlStatus = $resultsXmlStatus summaryPresent = Test-Path -LiteralPath $summaryPath @@ -465,7 +519,7 @@ jobs: } elseif ('${{ inputs.selection_status }}' -ne 'ready') { $executionStatus = 'skipped' if ([string]::IsNullOrWhiteSpace($receiptStatus)) { $receiptStatus = 'selection-blocked' } - } elseif ($receiptStatus -in @('results-xml-truncated', 'invalid-results-xml', 'missing-results-xml')) { + } elseif ($receiptStatus -in @('results-xml-truncated', 'invalid-results-xml', 'missing-results-xml', 'unsupported-schema')) { $executionStatus = $receiptStatus $rawArtifactName = 'pester-run-raw' } else { @@ -508,6 +562,8 @@ jobs: contextStatus = '${{ inputs.context_status }}' readinessStatus = '${{ inputs.readiness_status }}' selectionStatus = '${{ inputs.selection_status }}' + selectionExecutionPack = '${{ needs.pester.outputs.selection_execution_pack }}' + selectionExecutionPackSource = '${{ needs.pester.outputs.selection_execution_pack_source }}' executionJobResult = '${{ needs.pester.result }}' dispatcherExitCode = [int]$dispatcherExitCode status = $receiptStatus diff --git a/.github/workflows/pester-selection.yml b/.github/workflows/pester-selection.yml index bedacc057..ecfe4781b 100644 --- a/.github/workflows/pester-selection.yml +++ b/.github/workflows/pester-selection.yml @@ -3,6 +3,10 @@ name: Pester selection on: workflow_call: inputs: + execution_pack: + required: false + type: string + default: 'full' include_integration: required: false type: string @@ -29,6 +33,12 @@ on: value: ${{ jobs.selection.outputs.receipt_artifact_name }} workflow_dispatch: inputs: + execution_pack: + description: 'Named execution pack or test group to resolve before execution' + required: false + default: 'full' + type: choice + options: ['full', 'comparevi', 'dispatcher', 'workflow', 'fixtures', 'psummary', 'schema', 'loop'] include_integration: description: "Include Integration-tagged tests in the execution pack" required: false @@ -78,26 +88,27 @@ jobs: with: value: ${{ inputs.include_integration || 'false' }} - - name: Shape include patterns - id: include_patterns + - name: Resolve execution pack + id: execution_pack shell: pwsh env: + RAW_EXECUTION_PACK: ${{ inputs.execution_pack || github.event.inputs.execution_pack || 'full' }} RAW_INCLUDE_PATTERNS: ${{ inputs.include_patterns || '' }} run: | - $raw = $env:RAW_INCLUDE_PATTERNS - $tokens = New-Object System.Collections.Generic.List[string] - foreach ($candidate in ($raw -split "[`r`n,;]")) { - $token = $candidate.Trim() - if ([string]::IsNullOrWhiteSpace($token)) { continue } - $leaf = Split-Path -Leaf $token - if ([string]::IsNullOrWhiteSpace($leaf)) { continue } - if (-not $tokens.Contains($leaf)) { - $tokens.Add($leaf) | Out-Null - } - } - $patternsJson = @($tokens.ToArray()) | ConvertTo-Json -Compress - if (-not $patternsJson) { $patternsJson = '[]' } - "patterns_json=$patternsJson" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + . (Join-Path (Get-Location) 'tools/PesterExecutionPacks.ps1') + $resolution = Resolve-PesterExecutionPack -ExecutionPack $env:RAW_EXECUTION_PACK -RefineIncludePatterns $env:RAW_INCLUDE_PATTERNS + $basePatternsJson = @($resolution.baseIncludePatterns) | ConvertTo-Json -Compress + if (-not $basePatternsJson) { $basePatternsJson = '[]' } + $refinePatternsJson = @($resolution.refineIncludePatterns) | ConvertTo-Json -Compress + if (-not $refinePatternsJson) { $refinePatternsJson = '[]' } + $effectivePatternsJson = @($resolution.effectiveIncludePatterns) | ConvertTo-Json -Compress + if (-not $effectivePatternsJson) { $effectivePatternsJson = '[]' } + "execution_pack=$($resolution.executionPack)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "execution_pack_source=$($resolution.executionPackSource)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "execution_pack_description=$($resolution.executionPackDescription)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "base_patterns_json=$basePatternsJson" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "refine_patterns_json=$refinePatternsJson" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "effective_patterns_json=$effectivePatternsJson" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 - name: Resolve dispatcher profile id: dispatcher_profile @@ -121,16 +132,24 @@ jobs: New-Item -ItemType Directory -Force -Path $outDir | Out-Null $includeIntegration = '${{ steps.include_integration.outputs.normalized }}' $integrationMode = if ($includeIntegration -eq 'true') { 'include' } else { 'exclude' } - $includePatterns = '${{ steps.include_patterns.outputs.patterns_json }}' | ConvertFrom-Json -ErrorAction Stop + $baseIncludePatterns = '${{ steps.execution_pack.outputs.base_patterns_json }}' | ConvertFrom-Json -ErrorAction Stop + $refineIncludePatterns = '${{ steps.execution_pack.outputs.refine_patterns_json }}' | ConvertFrom-Json -ErrorAction Stop + $effectiveIncludePatterns = '${{ steps.execution_pack.outputs.effective_patterns_json }}' | ConvertFrom-Json -ErrorAction Stop $receipt = [ordered]@{ schema = 'pester-selection-receipt@v1' generatedAtUtc = [DateTime]::UtcNow.ToString('o') status = 'ready' sampleId = '${{ inputs.sample_id || github.event.inputs.sample_id || '' }}' selection = [ordered]@{ + executionPack = '${{ steps.execution_pack.outputs.execution_pack }}' + executionPackSource = '${{ steps.execution_pack.outputs.execution_pack_source }}' + executionPackDescription = '${{ steps.execution_pack.outputs.execution_pack_description }}' includeIntegrationNormalized = $includeIntegration integrationMode = $integrationMode - includePatterns = @($includePatterns) + baseIncludePatterns = @($baseIncludePatterns) + refineIncludePatterns = @($refineIncludePatterns) + effectiveIncludePatterns = @($effectiveIncludePatterns) + includePatterns = @($effectiveIncludePatterns) fixtureRequired = ($includeIntegration -eq 'true') } dispatcherProfile = [ordered]@{ diff --git a/.github/workflows/pester-service-model-release-evidence.yml b/.github/workflows/pester-service-model-release-evidence.yml index 9d067533b..ef77845f7 100644 --- a/.github/workflows/pester-service-model-release-evidence.yml +++ b/.github/workflows/pester-service-model-release-evidence.yml @@ -18,17 +18,22 @@ on: - 'docs/architecture/pester-service-model-control-plane.md' - 'docs/architecture/ADR-2078-pester-service-model-requirements.md' - 'docs/cm-plan-pester-service-model.md' + - 'docs/pester-service-model-promotion-comparison.json' - 'docs/release-procedure-pester-service-model.md' - 'docs/requirements-pester-service-model-srs.md' - 'docs/rtm-pester-service-model.csv' + - 'docs/schemas/pester-derived-provenance-v1.schema.json' + - 'docs/schemas/pester-promotion-comparison-v1.schema.json' - 'docs/testing/pester-service-model-test-plan.md' - 'docs/pester-service-model-quality-report.md' - 'docs/pester-service-model-information-item-map.md' - 'tools/priority/materialize-pester-service-model-release-evidence.mjs' + - 'tools/priority/pester-service-model-provenance.mjs' - 'tools/priority/render-pester-service-model-promotion-dossier.mjs' - 'tools/priority/write-node-test-coverage-xml.mjs' - 'tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs' - 'tools/priority/__tests__/pester-service-model-quality-workflow-contract.test.mjs' + - 'tools/priority/__tests__/pester-service-model-release-evidence-provenance.test.mjs' - 'tools/priority/__tests__/pester-service-model-release-evidence-workflow-contract.test.mjs' - 'tools/priority/__tests__/write-node-test-coverage-xml.test.mjs' push: @@ -49,17 +54,22 @@ on: - 'docs/architecture/pester-service-model-control-plane.md' - 'docs/architecture/ADR-2078-pester-service-model-requirements.md' - 'docs/cm-plan-pester-service-model.md' + - 'docs/pester-service-model-promotion-comparison.json' - 'docs/release-procedure-pester-service-model.md' - 'docs/requirements-pester-service-model-srs.md' - 'docs/rtm-pester-service-model.csv' + - 'docs/schemas/pester-derived-provenance-v1.schema.json' + - 'docs/schemas/pester-promotion-comparison-v1.schema.json' - 'docs/testing/pester-service-model-test-plan.md' - 'docs/pester-service-model-quality-report.md' - 'docs/pester-service-model-information-item-map.md' - 'tools/priority/materialize-pester-service-model-release-evidence.mjs' + - 'tools/priority/pester-service-model-provenance.mjs' - 'tools/priority/render-pester-service-model-promotion-dossier.mjs' - 'tools/priority/write-node-test-coverage-xml.mjs' - 'tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs' - 'tools/priority/__tests__/pester-service-model-quality-workflow-contract.test.mjs' + - 'tools/priority/__tests__/pester-service-model-release-evidence-provenance.test.mjs' - 'tools/priority/__tests__/pester-service-model-release-evidence-workflow-contract.test.mjs' - 'tools/priority/__tests__/write-node-test-coverage-xml.test.mjs' @@ -93,6 +103,7 @@ jobs: node --test --experimental-test-coverage \ tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs \ tools/priority/__tests__/pester-service-model-quality-workflow-contract.test.mjs \ + tools/priority/__tests__/pester-service-model-release-evidence-provenance.test.mjs \ tools/priority/__tests__/write-node-test-coverage-xml.test.mjs \ 2>&1 | tee tests/results/_agent/pester-service-model/node-test-coverage.log @@ -125,16 +136,21 @@ jobs: shell: bash run: | node tools/priority/materialize-pester-service-model-release-evidence.mjs \ + --output-dir tests/results/_agent/pester-service-model/release-evidence \ --version v0.1.0 \ --upstream-issue "$PSM_UPSTREAM_ISSUE" \ --fork-issue "$PSM_FORK_ISSUE" \ --fork-basis-commit "$PSM_FORK_BASIS_COMMIT" \ --fork-basis-url "$PSM_FORK_BASIS_URL" node tools/priority/render-pester-service-model-promotion-dossier.mjs \ + --release-evidence-dir tests/results/_agent/pester-service-model/release-evidence \ + --output tests/results/_agent/pester-service-model/release-evidence/promotion-dossier.md \ --upstream-issue "$PSM_UPSTREAM_ISSUE" \ --fork-issue "$PSM_FORK_ISSUE" \ --fork-basis-commit "$PSM_FORK_BASIS_COMMIT" \ --fork-basis-url "$PSM_FORK_BASIS_URL" + test -f tests/results/_agent/pester-service-model/release-evidence/release-evidence-provenance.json + test -f tests/results/_agent/pester-service-model/release-evidence/promotion-dossier-provenance.json - name: Upload release-evidence bundle uses: actions/upload-artifact@v7 diff --git a/Invoke-PesterTests.ps1 b/Invoke-PesterTests.ps1 index 658290888..e4a7ee1c4 100644 --- a/Invoke-PesterTests.ps1 +++ b/Invoke-PesterTests.ps1 @@ -30,6 +30,10 @@ param( [ValidateNotNullOrEmpty()] [string]$TestsPath = 'tests', + [Parameter(Mandatory = $false)] + [ValidateSet('full', 'comparevi', 'dispatcher', 'workflow', 'fixtures', 'psummary', 'schema', 'loop', 'all', 'default', 'compare-vi', 'dispatch', 'orchestration', 'fixture', 'summary', 'schemas', 'control-loop')] + [string]$ExecutionPack = 'full', + [Parameter(Mandatory = $false)] [ValidateSet('auto','include','exclude')] [string]$IntegrationMode = 'auto', @@ -159,6 +163,10 @@ $dispatcherSelectionModule = Join-Path $PSScriptRoot 'tools' 'Dispatcher' 'TestS if (Test-Path -LiteralPath $dispatcherSelectionModule) { Import-Module $dispatcherSelectionModule -Force } +$failurePayloadTool = Join-Path $PSScriptRoot 'tools' 'PesterFailurePayload.ps1' +if (Test-Path -LiteralPath $failurePayloadTool -PathType Leaf) { + . $failurePayloadTool +} $PesterPolicyVersion = '5.7.1' try { @@ -553,11 +561,18 @@ function _Finalize-LabVIEWPidTracker { Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' +. (Join-Path $PSScriptRoot 'tools/PesterExecutionPacks.ps1') # Default for includeIntegrationBool to avoid uninitialized usage during early helper calls if (-not (Get-Variable -Name includeIntegrationBool -Scope Script -ErrorAction SilentlyContinue)) { $script:includeIntegrationBool = $false } +$script:executionPackResolved = 'full' +$script:executionPackReason = 'default' +$script:executionPackBaseIncludePatterns = @() +$script:executionPackRefineIncludePatterns = @() +$script:executionPackEffectiveIncludePatterns = @() +$script:executionPackDescription = 'Full Pester suite' function Test-EnvTruthy { param([string]$Value) @@ -734,6 +749,14 @@ switch ($resolvedIntegrationMode) { } $script:integrationModeResolved = $resolvedIntegrationMode $includeIntegrationBool = [bool]$script:includeIntegrationBool +$executionPackResolution = Resolve-PesterExecutionPack -ExecutionPack $ExecutionPack -RefineIncludePatterns $IncludePatterns +$script:executionPackResolved = [string]$executionPackResolution.executionPack +$script:executionPackReason = [string]$executionPackResolution.executionPackSource +$script:executionPackBaseIncludePatterns = @($executionPackResolution.baseIncludePatterns) +$script:executionPackRefineIncludePatterns = @($executionPackResolution.refineIncludePatterns) +$script:executionPackEffectiveIncludePatterns = @($executionPackResolution.effectiveIncludePatterns) +$script:executionPackDescription = [string]$executionPackResolution.executionPackDescription +$IncludePatterns = @($script:executionPackEffectiveIncludePatterns) $script:fastModeTemporarilySet = $false if (-not $includeIntegrationBool) { $fastPesterPresent = $false @@ -787,7 +810,7 @@ elseif ($TimeoutMinutes -gt 0) { $effectiveTimeoutSeconds = [double]$TimeoutMinu # Schema version identifiers for emitted JSON artifacts (increment on breaking schema changes) $SchemaSummaryVersion = '1.7.1' -$SchemaFailuresVersion = '1.0.0' +$SchemaFailuresVersion = '1.1.0' $SchemaManifestVersion = '1.0.0' ${SchemaLeakReportVersion} = '1.0.0' ${SchemaDiagnosticsVersion} = '1.1.0' @@ -813,6 +836,8 @@ if ($sessionLockEnabled) { function Ensure-FailuresJson { param( [Parameter(Mandatory)][string]$Directory, + [Parameter()]$SummaryObject, + [Parameter()][string]$FailuresSchemaVersion = '1.1.0', [Parameter()][switch]$Force, [Parameter()][switch]$Normalize, [Parameter()][switch]$Quiet @@ -823,14 +848,34 @@ function Ensure-FailuresJson { } $path = Join-Path $Directory 'pester-failures.json' if ($Force -or -not (Test-Path -LiteralPath $path -PathType Leaf)) { - '[]' | Out-File -FilePath $path -Encoding utf8 -ErrorAction Stop + $payloadSummary = if ($null -ne $SummaryObject) { + $SummaryObject + } else { + [pscustomobject]@{ + total = 0 + failed = 0 + errors = 0 + skipped = 0 + } + } + Write-PesterFailurePayload -PathValue $path -SummaryObject $payloadSummary -FailureEntries @() -SchemaVersion $FailuresSchemaVersion | Out-Null if (-not $Quiet) { Write-Host "Created empty failures JSON at: $path" -ForegroundColor Gray } } elseif ($Normalize) { try { $info = Get-Item -LiteralPath $path -ErrorAction Stop if ($info.Length -eq 0 -or -not (Get-Content -LiteralPath $path -Raw).Trim()) { - '[]' | Out-File -FilePath $path -Encoding utf8 -Force - if (-not $Quiet) { Write-Host 'Normalized zero-byte failures JSON to []' -ForegroundColor Gray } + $payloadSummary = if ($null -ne $SummaryObject) { + $SummaryObject + } else { + [pscustomobject]@{ + total = 0 + failed = 0 + errors = 0 + skipped = 0 + } + } + Write-PesterFailurePayload -PathValue $path -SummaryObject $payloadSummary -FailureEntries @() -SchemaVersion $FailuresSchemaVersion | Out-Null + if (-not $Quiet) { Write-Host 'Normalized zero-byte failures JSON to canonical empty payload' -ForegroundColor Gray } } } catch { Write-Warning "Failed to normalize failures JSON: $_" @@ -885,134 +930,14 @@ function Clear-DispatcherGuardCrumb { } } -function Write-ArtifactManifest { - param( - [Parameter(Mandatory)] [string]$Directory, - [Parameter(Mandatory)] [string]$SummaryJsonPath, - [Parameter(Mandatory)] [string]$ManifestVersion - ) - try { - if ([string]::IsNullOrWhiteSpace($Directory)) { - Write-Warning "Artifact manifest emission skipped: Directory parameter was null or empty" - return - } - # Ensure directory exists (it should, but tests may simulate deletion scenarios) - if (-not (Test-Path -LiteralPath $Directory -PathType Container)) { - try { New-Item -ItemType Directory -Force -Path $Directory | Out-Null } catch { Write-Warning "Failed to (re)create artifact directory '$Directory': $_" } - } - - $artifacts = @() - - # Add artifacts if they exist - $xmlPath = Join-Path $Directory 'pester-results.xml' - if (Test-Path -LiteralPath $xmlPath) { - $artifacts += [PSCustomObject]@{ file = 'pester-results.xml'; type = 'nunitXml' } - } - - $txtPath = Join-Path $Directory 'pester-summary.txt' - if (Test-Path -LiteralPath $txtPath) { - $artifacts += [PSCustomObject]@{ file = 'pester-summary.txt'; type = 'textSummary' } - } - # Include rendered compare report(s) if present - $cmpPath = Join-Path $Directory 'compare-report.html' - if (Test-Path -LiteralPath $cmpPath) { - $artifacts += [PSCustomObject]@{ file = 'compare-report.html'; type = 'htmlCompare' } - } - # Include results index if present - $idxPath = Join-Path $Directory 'results-index.html' - if (Test-Path -LiteralPath $idxPath) { - $artifacts += [PSCustomObject]@{ file = 'results-index.html'; type = 'htmlIndex' } - } - try { - $extraHtml = @(Get-ChildItem -LiteralPath $Directory -Filter '*compare-report*.html' -File -ErrorAction SilentlyContinue | Where-Object { $_.Name -ne 'compare-report.html' }) - foreach ($h in $extraHtml) { - $artifacts += [PSCustomObject]@{ file = $h.Name; type = 'htmlCompare' } - } - } catch {} - - if (-not [string]::IsNullOrWhiteSpace($SummaryJsonPath)) { - try { - $jsonSummaryFile = Split-Path -Leaf $SummaryJsonPath - $jsonPath = Join-Path $Directory $jsonSummaryFile - if (Test-Path -LiteralPath $jsonPath) { - $artifacts += [PSCustomObject]@{ file = $jsonSummaryFile; type = 'jsonSummary'; schemaVersion = $SchemaSummaryVersion } - } - } catch { - Write-Warning "Failed to process summary JSON path '$SummaryJsonPath' for manifest: $_" - } - } - - $failuresPath = Join-Path $Directory 'pester-failures.json' - if (Test-Path -LiteralPath $failuresPath) { - $artifacts += [PSCustomObject]@{ file = 'pester-failures.json'; type = 'jsonFailures'; schemaVersion = $SchemaFailuresVersion } - } - $trailPath = Join-Path $Directory 'pester-artifacts-trail.json' - if (Test-Path -LiteralPath $trailPath) { - $artifacts += [PSCustomObject]@{ file = 'pester-artifacts-trail.json'; type = 'jsonTrail' } - } - $sessionIdx = Join-Path $Directory 'session-index.json' - if (Test-Path -LiteralPath $sessionIdx) { - $artifacts += [PSCustomObject]@{ file = 'session-index.json'; type = 'jsonSessionIndex' } - } - $leakPath = Join-Path $Directory 'pester-leak-report.json' - if (Test-Path -LiteralPath $leakPath) { - $artifacts += [PSCustomObject]@{ file = 'pester-leak-report.json'; type = 'jsonLeaks'; schemaVersion = $SchemaLeakReportVersion } - } - # Optional diagnostics files (result shapes) - $diagTxt = Join-Path $Directory 'result-shapes.txt' - if (Test-Path -LiteralPath $diagTxt) { - $artifacts += [PSCustomObject]@{ file = 'result-shapes.txt'; type = 'textDiagnostics' } - } - $diagJson = Join-Path $Directory 'result-shapes.json' - if (Test-Path -LiteralPath $diagJson) { - $artifacts += [PSCustomObject]@{ file = 'result-shapes.json'; type = 'jsonDiagnostics'; schemaVersion = ${SchemaDiagnosticsVersion} } - } - - # Optional: include lightweight metrics if summary JSON exists - $metrics = $null - try { - $jsonSummaryFile = Split-Path -Leaf $SummaryJsonPath - if ($jsonSummaryFile) { - $jsonPath = Join-Path $Directory $jsonSummaryFile - if (Test-Path -LiteralPath $jsonPath) { - $summaryJson = Get-Content -LiteralPath $jsonPath -Raw | ConvertFrom-Json -ErrorAction Stop - $aggMsValue = $null - if ($summaryJson.PSObject.Properties.Name -contains 'aggregatorBuildMs' -and $null -ne $summaryJson.aggregatorBuildMs) { - $aggMsValue = $summaryJson.aggregatorBuildMs - } - $metrics = [PSCustomObject]@{ - totalTests = $summaryJson.total - failed = $summaryJson.failed - skipped = $summaryJson.skipped - duration_s = $summaryJson.duration_s - meanTest_ms = $summaryJson.meanTest_ms - p95Test_ms = $summaryJson.p95Test_ms - maxTest_ms = $summaryJson.maxTest_ms - aggregatorBuildMs = $aggMsValue - } - } - } - } catch { Write-Warning "Failed to enrich manifest metrics: $_" } - - $manifest = [PSCustomObject]@{ - manifestVersion = $ManifestVersion - generatedAt = (Get-Date).ToString('o') - artifacts = $artifacts - metrics = $metrics - } - $manifestPath = Join-Path $Directory 'pester-artifacts.json' - $manifest | ConvertTo-Json -Depth 5 | Out-File -FilePath $manifestPath -Encoding utf8 -ErrorAction Stop - Write-Host "Artifact manifest written to: $manifestPath" -ForegroundColor Gray - } catch { - Write-Warning "Failed to write artifact manifest: $_" - } -} - function Write-FailureDiagnostics { param( [Parameter(Mandatory)] $PesterResult, [Parameter(Mandatory)] [string]$ResultsDirectory, [Parameter(Mandatory)] [int]$SkippedCount, + [Parameter(Mandatory)] [int]$TotalCount, + [Parameter(Mandatory)] [int]$FailedCount, + [Parameter(Mandatory)] [int]$ErrorCount, [Parameter(Mandatory)] [string]$FailuresSchemaVersion ) try { @@ -1041,18 +966,26 @@ function Write-FailureDiagnostics { $failArray = @() foreach ($t in $failedTests) { $failArray += [PSCustomObject]@{ - name = $t.Name - path = $t.Path - duration_ms = if ($t.Duration) { [math]::Round($t.Duration.TotalMilliseconds,2) } else { $null } - message = if ($t.ErrorRecord) { ($t.ErrorRecord.Exception.Message | Out-String).Trim() } else { $null } - schemaVersion = $FailuresSchemaVersion + name = $t.Name + result = 'Failed' + path = $t.Path + file = $t.Path + duration = if ($t.Duration) { [math]::Round($t.Duration.TotalSeconds, 6) } else { $null } + duration_ms = if ($t.Duration) { [math]::Round($t.Duration.TotalMilliseconds, 2) } else { $null } + message = if ($t.ErrorRecord) { ($t.ErrorRecord.Exception.Message | Out-String).Trim() } else { $null } } } $failJsonPath = Join-Path $ResultsDirectory 'pester-failures.json' if (-not (Test-Path -LiteralPath $ResultsDirectory -PathType Container)) { New-Item -ItemType Directory -Force -Path $ResultsDirectory | Out-Null } - $failArray | ConvertTo-Json -Depth 4 | Out-File -FilePath $failJsonPath -Encoding utf8 -ErrorAction Stop + $failSummary = [pscustomobject]@{ + total = $TotalCount + failed = $FailedCount + errors = $ErrorCount + skipped = $SkippedCount + } + Write-PesterFailurePayload -PathValue $failJsonPath -SummaryObject $failSummary -FailureEntries $failArray -SchemaVersion $FailuresSchemaVersion | Out-Null Write-Host "Failures JSON written to: $failJsonPath" -ForegroundColor Gray } catch { Write-Warning "Failed to write failures JSON: $_" @@ -1087,6 +1020,11 @@ Write-Host "Configuration:" -ForegroundColor Yellow Write-Host " Tests Path: $TestsPath" Write-Host (" Integration Mode: {0}" -f $script:integrationModeResolved) Write-Host (" Include Integration: {0}" -f ([bool]$includeIntegrationBool)) +Write-Host (" Execution Pack: {0}" -f $script:executionPackResolved) +if ($script:executionPackReason) { Write-Host (" Pack Source: {0}" -f $script:executionPackReason) -ForegroundColor DarkGray } +if ($script:executionPackRefineIncludePatterns.Count -gt 0) { + Write-Host (" Pack Refinements: {0}" -f ($script:executionPackRefineIncludePatterns -join ', ')) -ForegroundColor DarkGray +} if ($script:integrationModeReason) { Write-Host (" Mode Source: {0}" -f $script:integrationModeReason) -ForegroundColor DarkGray } if ($legacyIncludeSpecified) { Write-Host (" Legacy IncludeIntegration Argument: {0}" -f $IncludeIntegration) -ForegroundColor DarkGray } Write-Host " Results Path: $ResultsPath" @@ -1399,6 +1337,8 @@ function Write-SessionIndex { schemaVersion = '1.0.0' generatedAtUtc = (Get-Date).ToUniversalTime().ToString('o') resultsDir = $ResultsDirectory + executionPack = $script:executionPackResolved + executionPackSource = $script:executionPackReason includeIntegration = [bool]$includeIntegrationBool integrationMode = $script:integrationModeResolved integrationSource = $script:integrationModeReason @@ -1462,6 +1402,7 @@ function Write-SessionIndex { $lines += ("- Status: {0}" -f $status) $lines += ("- Total: {0} | Passed: {1} | Failed: {2} | Errors: {3} | Skipped: {4}" -f $s.total,$s.passed,$s.failed,$s.errors,$s.skipped) $lines += ("- Duration (s): {0}" -f $s.duration_s) + if ($s.executionPack) { $lines += ("- Execution Pack: {0}" -f $s.executionPack) } $lines += ("- Include Integration: {0}" -f [bool]$includeIntegrationBool) $lines += ("- Integration Mode: {0}" -f $script:integrationModeResolved) if ($script:integrationModeReason) { $lines += ("- Integration Source: {0}" -f $script:integrationModeReason) } @@ -1655,6 +1596,12 @@ function Invoke-ExecutionFinalizeHelper { $SummaryPayload, [Parameter(ParameterSetName = 'Build')] $ArtifactTrail, + [Parameter(ParameterSetName = 'Build')] + [Parameter(ParameterSetName = 'Reuse')] + $LeakReportPayload, + [Parameter(ParameterSetName = 'Build')] + [Parameter(ParameterSetName = 'Reuse')] + $PublicationPayload, [Parameter(ParameterSetName = 'Reuse')] [switch]$ReuseExistingContext ) @@ -1671,6 +1618,24 @@ function Invoke-ExecutionFinalizeHelper { return $false } $contextPath = $script:executionFinalizeContextPath + if ($null -ne $LeakReportPayload -or $null -ne $PublicationPayload) { + $existingContext = Get-Content -LiteralPath $contextPath -Raw | ConvertFrom-Json -ErrorAction Stop + foreach ($entry in @( + @{ Name = 'leakReportPayload'; Value = $LeakReportPayload }, + @{ Name = 'publication'; Value = $PublicationPayload } + )) { + if ($null -eq $entry.Value) { + continue + } + $property = $existingContext.PSObject.Properties[$entry.Name] + if ($property) { + $property.Value = $entry.Value + } else { + Add-Member -InputObject $existingContext -Name $entry.Name -MemberType NoteProperty -Value $entry.Value + } + } + $existingContext | ConvertTo-Json -Depth 12 | Out-File -FilePath $contextPath -Encoding utf8 -ErrorAction Stop + } } else { $finalizeContext = [ordered]@{ schema = 'pester-execution-finalize-context@v1' @@ -1678,6 +1643,11 @@ function Invoke-ExecutionFinalizeHelper { repoRoot = $root resultsDir = $resultsDir jsonSummaryPath = $JsonSummaryPath + executionPack = $script:executionPackResolved + executionPackSource = $script:executionPackReason + executionPackBaseIncludePatterns = @($script:executionPackBaseIncludePatterns) + executionPackRefineIncludePatterns = @($script:executionPackRefineIncludePatterns) + executionPackEffectiveIncludePatterns = @($script:executionPackEffectiveIncludePatterns) includeIntegration = [bool]$includeIntegrationBool integrationMode = $script:integrationModeResolved integrationSource = $script:integrationModeReason @@ -1696,6 +1666,12 @@ function Invoke-ExecutionFinalizeHelper { if ($null -ne $ArtifactTrail) { $finalizeContext['artifactTrail'] = $ArtifactTrail } + if ($null -ne $LeakReportPayload) { + $finalizeContext['leakReportPayload'] = $LeakReportPayload + } + if ($null -ne $PublicationPayload) { + $finalizeContext['publication'] = $PublicationPayload + } $contextPath = Join-Path $resultsDir 'pester-execution-finalize-context.json' $finalizeContext | ConvertTo-Json -Depth 12 | Out-File -FilePath $contextPath -Encoding utf8 -ErrorAction Stop $script:executionFinalizeContextPath = $contextPath @@ -1712,6 +1688,75 @@ function Invoke-ExecutionFinalizeHelper { } } +function New-ExecutionPublicationPayload { + [CmdletBinding()] + param( + [Parameter()] + [object[]]$SelectedTests = @(), + [Parameter()] + [string]$DiscoveryDescriptor = 'manual-scan', + [Parameter()] + [string]$PartialLogPath + ) + + $selectedNames = @() + foreach ($item in @($SelectedTests)) { + if ($null -eq $item) { continue } + if ($item -is [System.IO.FileInfo]) { + $selectedNames += $item.Name + continue + } + if ($item -is [string]) { + $selectedNames += (Split-Path -Leaf $item) + continue + } + if ($item.PSObject.Properties['FullName']) { + $selectedNames += (Split-Path -Leaf $item.FullName) + continue + } + if ($item.PSObject.Properties['Name']) { + $selectedNames += [string]$item.Name + } + } + $selectedNames = @($selectedNames | Where-Object { $_ -and $_ -ne '' } | Sort-Object -Unique) + + $ghCommand = $null + $workflowName = if ($env:GITHUB_WORKFLOW) { $env:GITHUB_WORKFLOW } else { 'ci-orchestrated.yml' } + $ghCommand = "gh workflow run `"$workflowName`"" + if ($env:GITHUB_REPOSITORY) { $ghCommand += (" -R {0}" -f $env:GITHUB_REPOSITORY) } + if ($env:GITHUB_REF_NAME) { $ghCommand += (" -r `"{0}`"" -f $env:GITHUB_REF_NAME) } + $ghCommand += (" -f include_integration={0}" -f ([bool]$includeIntegrationBool).ToString().ToLowerInvariant()) + if ($env:EV_SAMPLE_ID) { $ghCommand += (" -f sample_id={0}" -f $env:EV_SAMPLE_ID) } + + $payload = [ordered]@{ + disableStepSummary = [bool]$DisableStepSummary + selectedTests = $selectedNames + includeIntegration = [bool]$includeIntegrationBool + integrationMode = $script:integrationModeResolved + integrationSource = $script:integrationModeReason + discovery = $DiscoveryDescriptor + rerunCommand = $ghCommand + } + + if ($script:stuckGuardEnabled) { + $heartbeatCount = 0 + if ($script:hbPath -and (Test-Path -LiteralPath $script:hbPath -PathType Leaf)) { + try { + $lines = Get-Content -LiteralPath $script:hbPath -ErrorAction SilentlyContinue + $heartbeatCount = @($lines | Where-Object { $_ -like '*"type":"beat"*' }).Count + } catch {} + } + $payload['guard'] = [ordered]@{ + enabled = [bool]$script:stuckGuardEnabled + heartbeats = $heartbeatCount + heartbeatPath = $script:hbPath + partialLogPath = $PartialLogPath + } + } + + return [pscustomobject]$payload +} + # Optional pre-clean of LabVIEW if explicitly requested if ($CleanLabVIEW) { Write-Host "Pre-run cleanup: stopping LabVIEW.exe" -ForegroundColor DarkGray @@ -1891,10 +1936,21 @@ if ($limitToSingle) { } } if ($patternFilters.Include.Applied) { - $includePatternsText = if ($patternFilters.Include.Patterns) { ($patternFilters.Include.Patterns -join ', ') } else { '' } - Write-Host ( - "Applied IncludePatterns ({0}) -> kept {1}/{2} file(s)" -f $includePatternsText, $patternFilters.Include.After, $patternFilters.Include.Before - ) -ForegroundColor DarkGray + if ($script:executionPackResolved -and $script:executionPackResolved -ne 'full') { + $detail = if ($script:executionPackRefineIncludePatterns.Count -gt 0) { + "ExecutionPack={0}; refinements={1}" -f $script:executionPackResolved, ($script:executionPackRefineIncludePatterns -join ', ') + } else { + "ExecutionPack={0}" -f $script:executionPackResolved + } + Write-Host ( + "Applied selection contract ({0}) -> kept {1}/{2} file(s)" -f $detail, $patternFilters.Include.After, $patternFilters.Include.Before + ) -ForegroundColor DarkGray + } else { + $includePatternsText = if ($patternFilters.Include.Patterns) { ($patternFilters.Include.Patterns -join ', ') } else { '' } + Write-Host ( + "Applied IncludePatterns ({0}) -> kept {1}/{2} file(s)" -f $includePatternsText, $patternFilters.Include.After, $patternFilters.Include.Before + ) -ForegroundColor DarkGray + } } if ($patternFilters.Exclude.Applied -and $patternFilters.Exclude.Removed -gt 0) { $excludePatternsText = if ($patternFilters.Exclude.Patterns) { ($patternFilters.Exclude.Patterns -join ', ') } else { '' } @@ -1995,7 +2051,8 @@ if (-not (Get-Variable -Name originalTestFileCount -Scope Script -ErrorAction Si } $selectedTestPaths = @($testFiles | ForEach-Object { $_.FullName }) $selectionReasons = New-Object System.Collections.Generic.List[string] -if ($IncludePatterns -and $IncludePatterns.Count -gt 0) { [void]$selectionReasons.Add('IncludePatterns') } +if ($script:executionPackResolved -and $script:executionPackResolved -ne 'full') { [void]$selectionReasons.Add('ExecutionPack') } +if ($script:executionPackRefineIncludePatterns -and $script:executionPackRefineIncludePatterns.Count -gt 0) { [void]$selectionReasons.Add('IncludePatterns') } if ($ExcludePatterns -and $ExcludePatterns.Count -gt 0) { [void]$selectionReasons.Add('ExcludePatterns') } if ($maxTestFilesApplied) { [void]$selectionReasons.Add('MaxTestFiles') } if ($selectedTestPaths.Count -lt $originalTestFileCount) { [void]$selectionReasons.Add('SelectionReduced') } @@ -2037,6 +2094,10 @@ if ($testFiles.Count -eq 0) { duration_s = 0.0 timestamp = (Get-Date).ToString('o') pesterVersion = '' + executionPack = $script:executionPackResolved + executionPackSource = $script:executionPackReason + refineIncludePatterns = @($script:executionPackRefineIncludePatterns) + effectiveIncludePatterns = @($script:executionPackEffectiveIncludePatterns) includeIntegration = [bool]$includeIntegrationBool integrationMode = $script:integrationModeResolved integrationSource = $script:integrationModeReason @@ -2051,6 +2112,9 @@ if ($testFiles.Count -eq 0) { resultsXmlCloseTagPresent = $true schemaVersion = $SchemaSummaryVersion } + $earlyDiscoveryDescriptor = if ($usedNodeDiscovery) { 'manifest' } else { 'manual-scan' } + $publicationPayload = New-ExecutionPublicationPayload -SelectedTests @() -DiscoveryDescriptor $earlyDiscoveryDescriptor + $leakReportPayload = $null # Optional: run leak detection even when no tests discovered if ($DetectLeaks) { @@ -2073,7 +2137,7 @@ if ($testFiles.Count -eq 0) { $runningJobs = @($pesterJobs | Where-Object { $_.state -eq 'Running' -or $_.state -eq 'NotStarted' }) $leakDetected = (($procsAfter.Count -gt 0) -or ($runningJobs.Count -gt 0)) } - $leakReport = [pscustomobject]@{ + $leakReportPayload = [pscustomobject]@{ schema = 'pester-leak-report/v1' schemaVersion = ${SchemaLeakReportVersion} generatedAt = (Get-Date).ToString('o') @@ -2091,12 +2155,10 @@ if ($testFiles.Count -eq 0) { stoppedJobs = $stoppedJobs notes = @('Leak = LabVIEW/LVCompare (or configured targets) still running or Pester jobs still active after test run') } - $leakPathOut = Join-Path $resultsDir 'pester-leak-report.json' - $leakReport | ConvertTo-Json -Depth 6 | Out-File -FilePath $leakPathOut -Encoding utf8 -ErrorAction SilentlyContinue if ($leakDetected -and $FailOnLeaks) { Write-Error 'Failing run due to detected leaks (processes/jobs)'; exit 1 } } catch { Write-Warning "Leak detection (early-exit) failed: $_" } } - Invoke-ExecutionFinalizeHelper -SummaryText $summaryTextEarly -SummaryPayload $jsonObj -ArtifactTrail $script:artifactTrail | Out-Null + Invoke-ExecutionFinalizeHelper -SummaryText $summaryTextEarly -SummaryPayload $jsonObj -ArtifactTrail $script:artifactTrail -LeakReportPayload $leakReportPayload -PublicationPayload $publicationPayload | Out-Null Write-Host 'No test files found. Placeholder artifacts emitted.' -ForegroundColor Yellow exit 0 } @@ -2562,6 +2624,10 @@ if (-not $script:UseSingleInvoker) { duration_s = [math]::Round($testDuration.TotalSeconds, 6) timestamp = (Get-Date).ToString('o') pesterVersion = if ($loadedPester) { $loadedPester.Version.ToString() } else { '' } + executionPack = $script:executionPackResolved + executionPackSource = $script:executionPackReason + refineIncludePatterns = @($script:executionPackRefineIncludePatterns) + effectiveIncludePatterns = @($script:executionPackEffectiveIncludePatterns) includeIntegration = [bool]$includeIntegrationBool integrationMode = $script:integrationModeResolved integrationSource = $script:integrationModeReason @@ -2880,221 +2946,9 @@ $summary = $summaryLines -join [Environment]::NewLine Write-Host "" Write-Host $summary -ForegroundColor $(if ($failed -eq 0 -and $errors -eq 0) { 'Green' } else { 'Red' }) Write-Host "" - -# Emit high-level selection summary to GitHub Step Summary (if available) -if ($env:GITHUB_STEP_SUMMARY -and -not $DisableStepSummary) { - try { - $selectedNames = @() - foreach ($item in $testFiles) { - if ($null -eq $item) { continue } - if ($item -is [System.IO.FileInfo]) { - $selectedNames += $item.Name - } elseif ($item -is [string]) { - $selectedNames += (Split-Path -Leaf $item) - } elseif ($item.PSObject.Properties['FullName']) { - $selectedNames += (Split-Path -Leaf $item.FullName) - } - } - $selectedNames = $selectedNames | Sort-Object -Unique - $discoveryDescriptor = if ($usedNodeDiscovery) { 'manifest' } else { 'manual-scan' } - $includeText = ([bool]$includeIntegrationBool).ToString().ToLowerInvariant() - $modeText = if ($script:integrationModeResolved) { $script:integrationModeResolved } else { 'auto' } - $modeSource = if ($script:integrationModeReason) { $script:integrationModeReason } else { 'auto' } - $repoSlug = $env:GITHUB_REPOSITORY - $refName = $env:GITHUB_REF_NAME - $sampleId = $env:EV_SAMPLE_ID - $workflowName = if ($env:GITHUB_WORKFLOW) { $env:GITHUB_WORKFLOW } else { 'ci-orchestrated.yml' } - $ghCommand = "gh workflow run `"$workflowName`"" - if ($repoSlug) { $ghCommand += (" -R {0}" -f $repoSlug) } - if ($refName) { $ghCommand += (" -r `"{0}`"" -f $refName) } - $ghCommand += (" -f include_integration={0}" -f $includeText) - if ($sampleId) { $ghCommand += (" -f sample_id={0}" -f $sampleId) } - - $stepSummaryLines = @() - $stepSummaryLines += '' - $stepSummaryLines += '### Selected Tests' - $stepSummaryLines += '' - if ($selectedNames.Count -eq 0) { - $stepSummaryLines += '- (none)' - } else { - foreach ($name in $selectedNames) { $stepSummaryLines += ("- {0}" -f $name) } - } - $stepSummaryLines += '' - $stepSummaryLines += '### Configuration' - $stepSummaryLines += '' - $stepSummaryLines += ("- IncludeIntegration: {0}" -f ([bool]$includeIntegrationBool)) - $stepSummaryLines += ("- Integration Mode: {0}" -f $modeText) - $stepSummaryLines += ("- Integration Source: {0}" -f $modeSource) - $stepSummaryLines += ("- Discovery: {0}" -f $discoveryDescriptor) - if ($labviewPidTrackerLoaded -and $script:labviewPidTrackerPath) { - $stepSummaryLines += '' - $stepSummaryLines += '### LabVIEW PID Tracker' - $stepSummaryLines += '' - $stepSummaryLines += ("- Tracker Path: {0}" -f $script:labviewPidTrackerPath) - - $initialPid = 'none' - $initialRunning = 'unknown' - $initialReused = 'unknown' - if ($script:labviewPidTrackerState) { - if ($script:labviewPidTrackerState.PSObject.Properties['Pid'] -and $script:labviewPidTrackerState.Pid) { - $initialPid = $script:labviewPidTrackerState.Pid - } - if ($script:labviewPidTrackerState.PSObject.Properties['Running']) { - try { $initialRunning = [bool]$script:labviewPidTrackerState.Running } catch { $initialRunning = 'unknown' } - } - if ($script:labviewPidTrackerState.PSObject.Properties['Reused']) { - try { $initialReused = [bool]$script:labviewPidTrackerState.Reused } catch { $initialReused = 'unknown' } - } - } - $stepSummaryLines += ("- Initial: pid={0}, running={1}, reused={2}" -f $initialPid,$initialRunning,$initialReused) - - $finalPid = 'none' - $finalRunning = 'unknown' - $finalReused = 'unknown' - if ($script:labviewPidTrackerFinalState) { - if ($script:labviewPidTrackerFinalState.PSObject.Properties['Pid'] -and $script:labviewPidTrackerFinalState.Pid) { - $finalPid = $script:labviewPidTrackerFinalState.Pid - } - if ($script:labviewPidTrackerFinalState.PSObject.Properties['Running']) { - try { $finalRunning = [bool]$script:labviewPidTrackerFinalState.Running } catch { $finalRunning = 'unknown' } - } - if ($script:labviewPidTrackerFinalState.PSObject.Properties['Reused']) { - try { $finalReused = [bool]$script:labviewPidTrackerFinalState.Reused } catch { $finalReused = 'unknown' } - } elseif ($script:labviewPidTrackerState -and $script:labviewPidTrackerState.PSObject.Properties['Reused']) { - try { $finalReused = [bool]$script:labviewPidTrackerState.Reused } catch { $finalReused = 'unknown' } - } - } - $stepSummaryLines += ("- Final: pid={0}, running={1}, reused={2}" -f $finalPid,$finalRunning,$finalReused) - - $finalContext = $null - $contextSource = $null - if ($script:labviewPidTrackerFinalState -and $script:labviewPidTrackerFinalState.PSObject.Properties['Context'] -and $script:labviewPidTrackerFinalState.Context) { - $finalContext = _Normalize-LabVIEWPidContext -Value $script:labviewPidTrackerFinalState.Context - $contextSource = 'tracker' - if ($script:labviewPidTrackerFinalState.PSObject.Properties['ContextSource'] -and $script:labviewPidTrackerFinalState.ContextSource) { - $contextDetail = [string]$script:labviewPidTrackerFinalState.ContextSource - } else { - $contextDetail = 'tracker' - } - } elseif ($script:labviewPidTrackerFinalizedContext) { - $finalContext = _Normalize-LabVIEWPidContext -Value $script:labviewPidTrackerFinalizedContext - if ($script:labviewPidTrackerFinalizedContextSource) { - $contextSource = $script:labviewPidTrackerFinalizedContextSource - } else { - $contextSource = 'cached' - } - if ($script:labviewPidTrackerFinalizedContextDetail) { - $contextDetail = $script:labviewPidTrackerFinalizedContextDetail - } - } - if (-not $finalContext -and $script:labviewPidTrackerFinalState -and $script:labviewPidTrackerFinalState.PSObject.Properties['Observation'] -and $script:labviewPidTrackerFinalState.Observation) { - try { - $obs = $script:labviewPidTrackerFinalState.Observation - if ($obs -and $obs.PSObject.Properties['context'] -and $obs.context) { - $finalContext = _Normalize-LabVIEWPidContext -Value $obs.context - if (-not $contextSource) { - $contextSource = if ($obs.PSObject.Properties['contextSource'] -and $obs.contextSource) { [string]$obs.contextSource } else { 'tracker' } - } - if (-not $contextDetail) { $contextDetail = $contextSource } - } - } catch {} - } - if (-not $finalContext -and $script:labviewPidTrackerSummaryContext) { - try { - $finalContext = _Normalize-LabVIEWPidContext -Value $script:labviewPidTrackerSummaryContext - if (-not $contextSource -and $script:labviewPidTrackerSummaryContext.PSObject.Properties['stage']) { - $contextSource = $script:labviewPidTrackerSummaryContext.stage - } - if (-not $contextDetail) { $contextDetail = $contextSource } - } catch {} - } - if (-not $finalContext) { - $finalContext = [pscustomobject]@{ - stage = 'post-summary' - total = $total - failed = $failed - errors = $errors - skipped = $skipped - discoveryFailures = $discoveryFailureCount - timedOut = [bool]$script:timedOut - } - if ($script:labviewPidTrackerState -and $script:labviewPidTrackerState.PSObject.Properties['Pid'] -and $script:labviewPidTrackerState.Pid) { - $finalContext.pid = $script:labviewPidTrackerState.Pid - } - if (-not $contextSource) { $contextSource = 'dispatcher:summary' } - if (-not $contextDetail) { $contextDetail = $contextSource } - } - $contextLabel = $null - if ($contextSource -and $contextDetail -and $contextSource -ne $contextDetail) { - $contextLabel = " ($contextSource via $contextDetail)" - } elseif ($contextSource) { - $contextLabel = " ($contextSource)" - } elseif ($contextDetail) { - $contextLabel = " ($contextDetail)" - } else { - $contextLabel = '' - } - if ($finalContext -and $finalContext.PSObject.Properties['stage']) { - $stageValue = $finalContext.stage - $stepSummaryLines += ("- Final Context Stage{0}: {1}" -f $contextLabel,$stageValue) - } elseif ($finalContext) { - $ctxKeys = @($finalContext.PSObject.Properties.Name) - if ($ctxKeys.Count -gt 0) { - $stepSummaryLines += ("- Final Context Keys{0}: {1}" -f $contextLabel,($ctxKeys -join ', ')) - } - } elseif ($contextSource) { - if ($contextDetail -and $contextDetail -ne $contextSource) { - $stepSummaryLines += ("- Final Context Source: {0} (via {1})" -f $contextSource,$contextDetail) - } else { - $stepSummaryLines += ("- Final Context Source: {0}" -f $contextSource) - } - } - } - $stepSummaryLines += '' - $stepSummaryLines += '### Re-run (gh)' - $stepSummaryLines += '' - $stepSummaryLines += ("- {0}" -f $ghCommand) - $summaryFile = $env:GITHUB_STEP_SUMMARY - if ($summaryFile) { - try { - $summaryDir = Split-Path -Parent $summaryFile - if ($summaryDir -and -not (Test-Path -LiteralPath $summaryDir)) { - New-Item -ItemType Directory -Force -Path $summaryDir | Out-Null - } - $summaryText = ($stepSummaryLines -join [Environment]::NewLine) + [Environment]::NewLine - $encoding = New-Object System.Text.UTF8Encoding($false) - [System.IO.File]::WriteAllText($summaryFile, $summaryText, $encoding) - } catch { - $errMsg = $_.Exception.Message - Write-Host ("Step summary append failed: {0}" -f $errMsg) -ForegroundColor DarkYellow - } - } else { - Write-Host 'Step summary append skipped: GITHUB_STEP_SUMMARY not set.' -ForegroundColor DarkYellow - } - } catch { - $errMsg = $_.Exception.Message - $warnLine = [string]::Concat('Step summary append failed: ', $errMsg) - Write-Host $warnLine -ForegroundColor DarkYellow - } -} - -# Append optional Guard block to step summary (notice-only) -if ($script:stuckGuardEnabled -and $env:GITHUB_STEP_SUMMARY) { - try { - $hbCount = 0; $last = '' - if (Test-Path -LiteralPath $script:hbPath) { - $lines = Get-Content -LiteralPath $script:hbPath -ErrorAction SilentlyContinue - $hbCount = @($lines | Where-Object { $_ -like '*"type":"beat"*' }).Count - $last = $lines | Select-Object -Last 1 - } - $g = @('### Guard','') - $g += ('- Enabled: {0}' -f $script:stuckGuardEnabled) - $g += ('- Heartbeats: {0}' -f $hbCount) - if ($script:hbPath) { $g += ('- Heartbeat file: {0}' -f $script:hbPath) } - if ($partialLogPath) { $g += ('- Partial log: {0}' -f $partialLogPath) } - $g -join "`n" | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append -Encoding utf8 -ErrorAction SilentlyContinue - } catch { Write-Host "::notice::Guard summary append failed: $_" } -} +$publicationDiscoveryDescriptor = if ($usedNodeDiscovery) { 'manifest' } else { 'manual-scan' } +$publicationPayload = New-ExecutionPublicationPayload -SelectedTests $testFiles -DiscoveryDescriptor $publicationDiscoveryDescriptor -PartialLogPath $partialLogPath +$leakReportPayload = $null # Optional: emit result shape diagnostics for $result.Tests (types and property presence) if ($EmitResultShapeDiagnostics) { @@ -3158,7 +3012,7 @@ if ($EmitResultShapeDiagnostics) { } # Machine-readable JSON summary (adjacent enhancement for CI consumers) -$jsonSummaryPath = Join-Path $resultsDir $JsonSummaryPath +$jsonSummaryFullPath = Join-Path $resultsDir $JsonSummaryPath try { $jsonObj = [PSCustomObject]@{ total = $total @@ -3169,6 +3023,10 @@ try { duration_s = [math]::Round($testDuration.TotalSeconds, 6) timestamp = (Get-Date).ToString('o') pesterVersion = $loadedPester.Version.ToString() + executionPack = $script:executionPackResolved + executionPackSource = $script:executionPackReason + refineIncludePatterns = @($script:executionPackRefineIncludePatterns) + effectiveIncludePatterns = @($script:executionPackEffectiveIncludePatterns) includeIntegration = [bool]$includeIntegrationBool meanTest_ms = $meanMs p95Test_ms = $p95Ms @@ -3297,6 +3155,10 @@ try { totalDiscoveredFileCount = $originalTestFileCount selectedTestFileCount = $selectedTestFileCount maxTestFilesApplied = $maxTestFilesApplied + executionPack = $script:executionPackResolved + executionPackSource = $script:executionPackReason + refineIncludePatterns = @($script:executionPackRefineIncludePatterns) + effectiveIncludePatterns = @($script:executionPackEffectiveIncludePatterns) } Add-Member -InputObject $jsonObj -Name environment -MemberType NoteProperty -Value $envBlock Add-Member -InputObject $jsonObj -Name run -MemberType NoteProperty -Value $runBlock @@ -3479,55 +3341,6 @@ try { Write-Host "Results written to: $xmlPath" -ForegroundColor Gray Write-Host "" -# Optional: Write diagnostics summary to GitHub Step Summary (Markdown) -try { - $stepSummary = $env:GITHUB_STEP_SUMMARY - if ($stepSummary -and -not $DisableStepSummary) { - $total = $null; $hasPath = $null; $hasTags = $null - $diagJsonPath = Join-Path $resultsDir 'result-shapes.json' - if (Test-Path -LiteralPath $diagJsonPath -PathType Leaf) { - try { - $diagObj = Get-Content -LiteralPath $diagJsonPath -Raw | ConvertFrom-Json -ErrorAction Stop - $total = [int]($diagObj.totalEntries) - $hasPath = [int]($diagObj.overall.hasPath) - $hasTags = [int]($diagObj.overall.hasTags) - } catch {} - } - # Fallback: derive counts from $result.Tests when JSON not available - if ($null -eq $total -and $result -and $result.Tests) { - try { - $testsLocal = @($result.Tests) - $total = $testsLocal.Count - $hasPath = @($testsLocal | Where-Object { $_.PSObject.Properties.Name -contains 'Path' }).Count - $hasTags = @($testsLocal | Where-Object { $_.PSObject.Properties.Name -contains 'Tags' }).Count - } catch {} - } - if ($null -ne $total) { - function _pctMd { param([int]$n,[int]$d) if ($d -le 0) { return '0%' } ('{0:P1}' -f ([double]$n/[double]$d)) } - $pPath = _pctMd $hasPath $total - $pTags = _pctMd $hasTags $total - $md = @() - $md += '### Diagnostics Summary' - $md += '' - $md += '| Metric | Count | Percent |' - $md += '|---|---:|---:|' - $md += ("| Total entries | {0} | - |" -f $total) - $md += ("| Has Path | {0} | {1} |" -f $hasPath,$pPath) - $md += ("| Has Tags | {0} | {1} |" -f $hasTags,$pTags) - $mdText = ($md -join "`n") + "`n" - try { - $dir = Split-Path -Parent $stepSummary - if ($dir) { New-Item -ItemType Directory -Force -Path $dir -ErrorAction SilentlyContinue | Out-Null } - if (-not (Test-Path -LiteralPath $stepSummary -PathType Leaf)) { - New-Item -ItemType File -Path $stepSummary -Force | Out-Null - } - } catch {} - Add-Content -LiteralPath $stepSummary -Value $mdText -Encoding utf8 - Write-Host ("Step summary updated: {0}" -f $stepSummary) -ForegroundColor Gray - } - } -} catch { Write-Host "(warn) failed to write GitHub Step Summary: $_" -ForegroundColor DarkYellow } - # Leak detection (processes/jobs) and report if ($DetectLeaks) { try { @@ -3598,7 +3411,7 @@ if ($DetectLeaks) { $leakDetected = (($procsAfter.Count -gt 0) -or ($runningJobs.Count -gt 0)) } - $leakReport = [pscustomobject]@{ + $leakReportPayload = [pscustomobject]@{ schema = 'pester-leak-report/v1' schemaVersion = ${SchemaLeakReportVersion} generatedAt = (Get-Date).ToString('o') @@ -3616,10 +3429,8 @@ if ($DetectLeaks) { stoppedJobs = $stoppedJobs notes = @('Leak = LabVIEW/LVCompare (or configured targets) still running or Pester jobs still active after test run') } - $leakPathOut = Join-Path $resultsDir 'pester-leak-report.json' - $leakReport | ConvertTo-Json -Depth 6 | Out-File -FilePath $leakPathOut -Encoding utf8 -ErrorAction Stop if ($leakDetected) { - Write-Warning "Leak detected: see $leakPathOut" + Write-Warning "Leak detected: pester-leak-report.json will be emitted during finalize." if ($FailOnLeaks) { Write-Error "Failing run due to detected leaks (processes/jobs)" exit 1 @@ -3648,12 +3459,21 @@ try { # same failure surface that the exit path will expose. if ($failed -gt 0 -or $errors -gt 0) { if ($null -ne $result) { - Write-FailureDiagnostics -PesterResult $result -ResultsDirectory $resultsDir -SkippedCount $skipped -FailuresSchemaVersion $SchemaFailuresVersion + Write-FailureDiagnostics -PesterResult $result -ResultsDirectory $resultsDir -SkippedCount $skipped -TotalCount $total -FailedCount $failed -ErrorCount $errors -FailuresSchemaVersion $SchemaFailuresVersion } elseif ($EmitFailuresJsonAlways) { - Ensure-FailuresJson -Directory $resultsDir -Force + Ensure-FailuresJson -Directory $resultsDir -SummaryObject $jsonObj -FailuresSchemaVersion $SchemaFailuresVersion -Force } } elseif ($EmitFailuresJsonAlways) { - Ensure-FailuresJson -Directory $resultsDir -Normalize -Quiet + Ensure-FailuresJson -Directory $resultsDir -SummaryObject $jsonObj -FailuresSchemaVersion $SchemaFailuresVersion -Normalize -Quiet +} + +try { + $failurePayload = Sync-PesterFailurePayload -Directory $resultsDir -SummaryObject $jsonObj -SchemaVersion $SchemaFailuresVersion + if ($jsonSummaryFullPath) { + $jsonObj | ConvertTo-Json -Depth 10 | Out-File -FilePath $jsonSummaryFullPath -Encoding utf8 -ErrorAction Stop + } +} catch { + Write-Warning "Failed to synchronize failure-detail payload: $_" } if ($TrackArtifacts -and $script:artifactTrail) { @@ -3662,7 +3482,7 @@ if ($TrackArtifacts -and $script:artifactTrail) { } catch { Write-Warning "Failed to finalize artifact trail process snapshot: $_" } } -Invoke-ExecutionFinalizeHelper -SummaryText $summary -SummaryPayload $jsonObj -ArtifactTrail $script:artifactTrail | Out-Null +Invoke-ExecutionFinalizeHelper -SummaryText $summary -SummaryPayload $jsonObj -ArtifactTrail $script:artifactTrail -LeakReportPayload $leakReportPayload -PublicationPayload $publicationPayload | Out-Null # Exit with appropriate code if ($failed -gt 0 -or $errors -gt 0) { @@ -3678,11 +3498,11 @@ Write-Host "✅ All tests passed!" -ForegroundColor Green if ($discoveryFailureCount -gt 0) { Write-Host "Discovery failures detected ($discoveryFailureCount) but test counts showed success; forcing failure exit." -ForegroundColor Red try { - if ($jsonSummaryPath -and (Test-Path -LiteralPath $jsonSummaryPath)) { - $adjust = Get-Content -LiteralPath $jsonSummaryPath -Raw | ConvertFrom-Json + if ($jsonSummaryFullPath -and (Test-Path -LiteralPath $jsonSummaryFullPath)) { + $adjust = Get-Content -LiteralPath $jsonSummaryFullPath -Raw | ConvertFrom-Json $adjust.errors = ($adjust.errors + $discoveryFailureCount) $adjust.discoveryFailures = $discoveryFailureCount - $adjust | ConvertTo-Json -Depth 4 | Out-File -FilePath $jsonSummaryPath -Encoding utf8 + $adjust | ConvertTo-Json -Depth 4 | Out-File -FilePath $jsonSummaryFullPath -Encoding utf8 } } catch { Write-Warning "Failed to adjust JSON summary for discovery failures: $_" } Write-Error "Test execution completed with discovery failures" @@ -3734,9 +3554,8 @@ if ($discoveryFailureCount -gt 0) { stoppedJobs = @() notes = @('Final sweep leak report to ensure artifact presence; see main leak block for full details when enabled') } - $report | ConvertTo-Json -Depth 6 | Out-File -FilePath $finalLeakPath -Encoding utf8 -ErrorAction SilentlyContinue # Refresh finalize-owned artifacts so the late leak report is reflected in the manifest/session index. - Invoke-ExecutionFinalizeHelper -ReuseExistingContext | Out-Null + Invoke-ExecutionFinalizeHelper -ReuseExistingContext -LeakReportPayload $report | Out-Null } } } catch { Write-Warning "Failed to emit final sweep leak report: $_" } diff --git a/docs/architecture/local-proof-autonomy-program-control-plane.md b/docs/architecture/local-proof-autonomy-program-control-plane.md new file mode 100644 index 000000000..a5f40e3cc --- /dev/null +++ b/docs/architecture/local-proof-autonomy-program-control-plane.md @@ -0,0 +1,38 @@ +# CompareVI Local Proof Autonomy Program Control Plane + +## Scope + +This control plane covers the shared selector above the packet-local loops. Its +job is to aggregate packet-local next-step artifacts, rank the next truthful +local requirement, merge shared escalations, and emit a bounded post-local +promotion escalation when local packet work is complete. + +## Surface View + +| Surface | Responsibility | Technology | +| --- | --- | --- | +| Packet aggregation surface | Consume sibling packet reports and next-step artifacts | Node.js | +| Ranking surface | Rank requirement work ahead of escalation work across packets | Node.js | +| Shared escalation merge surface | Collapse duplicate escalations to the same external surface into one bounded handoff | Node.js | +| Post-local promotion surface | Emit the next truthful integration or hosted proof escalation once local packet work is complete | Node.js | +| Bundle workspace safety surface | Keep packet-local audit bundles run-scoped so concurrent invocations do not delete each other’s evidence | Node.js | + +## Component View + +| Component | Surface | Responsibility | +| --- | --- | --- | +| `comparevi-local-program-ci.mjs` | Packet aggregation surface | Run packet-local CI entrypoints and collect their reports | +| `comparevi-local-program-ci.mjs` | Ranking surface | Rank requirement candidates across packets and select the next local requirement | +| `comparevi-local-program-ci.mjs` | Shared escalation merge surface | Merge shared `windows-docker-desktop-ni-image` escalations and preserve packet metadata | +| `comparevi-local-program-ci.mjs` | Post-local promotion surface | Emit `integration-or-hosted-proof` escalation instead of `null` when local packet work is complete | +| Packet-local `*-local-ci.mjs` scripts | Bundle workspace safety surface | Materialize audit surfaces in run-scoped `surface-bundle/run-*` roots instead of deleting a shared mutable workspace | + +## Design Constraints + +- Requirement work should outrank escalation work. +- Shared escalations to the same external proof surface should merge into one + packet rather than competing advisories. +- The program should not terminate at `null` once local packet work is + complete; it should emit a bounded proof-promotion escalation. +- Packet-local local-CI surfaces should never share one mutable audit-bundle + root across concurrent invocations. diff --git a/docs/architecture/pester-service-model-control-plane.md b/docs/architecture/pester-service-model-control-plane.md index f22cfd40f..40ae086da 100644 --- a/docs/architecture/pester-service-model-control-plane.md +++ b/docs/architecture/pester-service-model-control-plane.md @@ -7,7 +7,9 @@ Replace the monolithic self-hosted Pester transaction with explicit control layers that can be proven, audited, and promoted intentionally. - Scope: - Trusted routing, context, selection, readiness, execution, evidence, and the additive + Trusted routing, context, selection, readiness, execution, local replay, + representative replay compatibility, local Windows-container surrogate proof, + evidence, execution observability, local autonomy guidance, and the additive promotion boundary. ## Stakeholders And Concerns @@ -38,8 +40,14 @@ | Context layer | Certify repository slug and standing-priority control-plane assumptions | GitHub Actions + Node | | Selection layer | Resolve the declared pack and dispatcher profile into a receipt | GitHub Actions + PowerShell | | Readiness layer | Certify self-hosted ingress runtime state and host dependencies | GitHub Actions + PowerShell | -| Execution layer | Run the selected Pester pack after validating upstream receipts | GitHub Actions + PowerShell | +| Execution layer | Run the selected Pester pack after validating upstream receipts and hand off finalize/publication side effects to dedicated helpers | GitHub Actions + PowerShell | | Evidence layer | Classify results, summarize them, and publish operator artifacts | GitHub Actions + PowerShell | +| Local replay surface | Rebuild non-host-dependent layers from retained artifacts without the workflow shell | PowerShell + retained artifacts | +| Observability surface | Preserve durable progress and schema-governed trace artifacts across long-running execution | PowerShell + retained artifacts | +| Provenance surface | Retain lineage from derived views back to raw execution inputs and runs | Retained artifacts + documentation packet | +| Windows-container surrogate surface | Bound local Docker Desktop Windows + NI image proof before hosted reruns | PowerShell + Docker Desktop | +| Local autonomy surface | Rank unresolved requirements and emit the next bounded local-first step, including escalations when the required proof surface is unavailable | Node.js + assurance packet | +| Autonomy policy surface | Constrain local-vs-hosted escalation and active-worktree interpretation | JSON policy + Node.js | ## Component View @@ -48,9 +56,26 @@ | `pester-service-model-on-label.yml` | Trusted router | Admission control for dispatch and same-owner labeled PRs | | `pester-context.yml` | Context layer | Repository and standing-priority receipt | | `pester-selection.yml` | Selection layer | Integration mode, include pattern, and dispatcher profile receipt | +| `tools/PesterExecutionPacks.ps1` | Selection layer | Canonical named execution-pack catalog and refinement resolution | | `selfhosted-readiness.yml` | Readiness layer | Runner labels, session lock, `.NET`, Docker, and LVCompare readiness | | `pester-run.yml` | Execution layer | Receipt validation, dispatcher invocation, execution contract | +| `tools/PesterFailurePayload.ps1` | Execution layer | Canonical failure-detail payload contract and unavailable-details normalization | +| `tools/Invoke-PesterExecutionFinalize.ps1` | Execution layer | Finalize-owned summary, session-index, artifact-manifest, compare-report, and leak-report side effects | +| `tools/Invoke-PesterExecutionPublication.ps1` | Execution layer | Operator-facing step-summary, session-summary, and diagnostics publication outside the dispatcher | +| `tools/Invoke-PesterExecutionTelemetry.ps1` | Execution layer | Durable telemetry normalization from dispatcher events and handshake markers into a stable inspection contract | +| `tools/PesterServiceModelSchema.ps1` | Execution/Evidence/Replay layers | Shared schema-governance helper for retained receipts and derived artifacts | | `pester-evidence.yml` | Evidence layer | Classification, summary, session index, and dashboard publication | +| `tools/Invoke-PesterEvidenceClassification.ps1` | Evidence layer | Shared classification contract for hosted evidence and local replay | +| `tools/Invoke-PesterOperatorOutcome.ps1` | Evidence layer | Shared operator-outcome contract for machine-readable gate status, reasons, and next-action guidance | +| `tools/Invoke-PesterEvidenceProvenance.ps1` | Evidence/Replay layers | Shared provenance contract for exact source artifacts, receipt identity, run context, and derived evidence outputs | +| `tools/Write-PesterTotals.ps1` | Evidence layer | Shared totals contract for hosted evidence and local replay | +| `tools/Run-PesterExecutionOnly.Local.ps1` | Local replay surface | Local-first execution harness | +| `tools/Replay-PesterServiceModelArtifacts.Local.ps1` | Local replay surface | Rebuild retained postprocess, totals, session index, and evidence outputs from mounted artifacts | +| `dispatcher-events.ndjson` + execution trace files | Observability surface | Durable runtime progress and post-run inspection anchor | +| Release-evidence bundles, promotion dossiers, and `pester-service-model-promotion-comparison.json` | Provenance surface | Retained comparison and lineage packet for promotion decisions | +| `tools/Invoke-PesterWindowsContainerSurfaceProbe.ps1` | Windows-container surrogate surface | Emit bounded readiness or advisory status for Docker Desktop Windows engine plus the pinned NI Windows image | +| `pester-service-model-local-ci.mjs` | Local autonomy surface | Machine-readable ranked backlog, next requirement guidance, and next-step escalation packet | +| `pester-service-model-autonomy-policy.json` | Autonomy policy surface | Versioned local-vs-hosted decision policy for autonomous iteration | ## Deployment View @@ -62,7 +87,8 @@ artifact storage. - Runtime dependencies: GitHub token, standing-priority cache, `dotnet`, Windows Docker runtime, - LVCompare, Session-Lock scripts, and `Invoke-PesterTests.ps1`. + LVCompare, Session-Lock scripts, unsynchronized local workspace roots, and + `Invoke-PesterTests.ps1`. ## Correspondence And Rationale @@ -74,6 +100,28 @@ `REQ-PSM-005` maps to execution. `REQ-PSM-006` maps to evidence. `REQ-PSM-007` maps to the additive promotion boundary. + `REQ-PSM-012` maps to selection plus explicit execution-pack entrypoints. + `REQ-PSM-013` maps to `tools/PesterPathHygiene.ps1`, local harness path + hygiene, and session-lock safety before dispatch. + `REQ-PSM-014` maps to `tools/Replay-PesterServiceModelArtifacts.Local.ps1`, `tools/Invoke-PesterEvidenceClassification.ps1`, and local replay of retained postprocess and evidence layers. + `REQ-PSM-015` maps to the dispatch/finalize/postprocess/evidence split. + `REQ-PSM-016` maps to durable execution telemetry. + `REQ-PSM-017` maps to schema-governed retained artifacts and readers, including explicit `unsupported-schema` outcomes for postprocess, evidence, and local replay. + `REQ-PSM-018` maps to `pester-service-model-promotion-comparison.json`, + release-evidence bundles, and representative baseline comparison rendered into + the promotion dossier. + `REQ-PSM-019` maps to stable operator-facing named pack entrypoints. + `REQ-PSM-020` maps to `pester-evidence-provenance.json`, + `release-evidence-provenance.json`, and + `promotion-dossier-provenance.json` across evidence, local replay, and + promotion views. + `REQ-PSM-021` maps to operator-explainable gate outcomes, including `pester-operator-outcome.json` and shared summary or top-failure rendering from the same outcome contract. + `REQ-PSM-022` maps to the local autonomy loop that selects the next bounded requirement slice. + `REQ-PSM-023` maps to the explicit autonomy policy and stop-condition surface. + `REQ-PSM-024` maps to representative retained-artifact replay compatibility, including schema-lite summary repair and legacy receipt tolerance. + `REQ-PSM-025` maps to the bounded local Windows-container surrogate for Docker Desktop Windows engine plus the pinned NI Windows image. + `REQ-PSM-026` maps to proof-check aware autonomy that reopens implemented requirements when representative replay regresses. + `REQ-PSM-027` maps to the machine-readable next-step escalation packet that hands off to the required proof surface when the current host cannot satisfy it. - Decision rationale: The service model exists to separate concerns and make failures classifiable by layer instead of inferred from one coupled self-hosted run. diff --git a/docs/architecture/vi-history-local-proof-control-plane.md b/docs/architecture/vi-history-local-proof-control-plane.md new file mode 100644 index 000000000..5fa480a1e --- /dev/null +++ b/docs/architecture/vi-history-local-proof-control-plane.md @@ -0,0 +1,40 @@ +# VI History Local Proof Control Plane + +## Scope + +This control plane covers the local-first VI History proof surfaces that should +be exercised before another hosted run is chosen. + +## Surface View + +| Surface | Responsibility | Technology | +| --- | --- | --- | +| Windows workflow replay surface | Reproduce `vi-history-scenarios-windows` locally with a workflow-grade receipt | Node.js + PowerShell + Docker Desktop | +| Clone-backed live-history candidate surface | Govern and validate the downstream repo clone and target VI that local VI History iteration should use | Node.js + Git | +| Local refinement surface | Run proof, `dev-fast`, `warm-dev`, and `windows-mirror-proof` profiles with canonical receipts | PowerShell + Docker | +| Local operator-session surface | Wrap local refinement into a stable operator-facing session contract | PowerShell | +| Workflow-readiness surface | Normalize lane state into a bounded verdict and recommendation | PowerShell | +| Local autonomy surface | Emit ranked local guidance and the next step for VI History work | Node.js + assurance packet | + +## Component View + +| Component | Container | Responsibility | +| --- | --- | --- | +| `vi-history-live-candidate.json` | Clone-backed live-history candidate surface | Name the governed downstream repo, clone-root contract, and target VI path for real-history local iteration | +| `windows-workflow-replay-lane.mjs` | Windows workflow replay surface | Govern `vi-history-scenarios-windows` local replay | +| `Invoke-VIHistoryLocalRefinement.ps1` | Local refinement surface | Emit profile-specific local refinement receipts, review-loop receipts, and benchmarks | +| `Invoke-VIHistoryLocalOperatorSession.ps1` | Local operator-session surface | Emit the canonical local operator-session contract for common profiles | +| `Write-VIHistoryWorkflowReadiness.ps1` | Workflow-readiness surface | Emit `vi-history/workflow-readiness@v1` with lifecycle and recommendation | +| `vi-history-local-ci.mjs` | Local autonomy surface | Validate clone-backed candidate readiness, emit report, ranked backlog, proof checks, and next-step escalation packets | + +## Design Constraints + +- VI History should stay packetized separately from the Pester service model. +- The governed live-history candidate should remain explicit and + machine-readable instead of being implied by incidental docs or sample + fixtures. +- The local VI History packet should reuse the shared + `windows-docker-desktop-ni-image` proof surface when the current host cannot + satisfy the Windows replay lane. +- Local proof should prefer declared profiles and wrappers over free-form helper + invocation. diff --git a/docs/architecture/windows-docker-shared-surface-control-plane.md b/docs/architecture/windows-docker-shared-surface-control-plane.md new file mode 100644 index 000000000..fc2d744d9 --- /dev/null +++ b/docs/architecture/windows-docker-shared-surface-control-plane.md @@ -0,0 +1,44 @@ +# Windows Docker Shared Surface Control Plane + +## Scope + +This control plane covers the shared Windows Docker Desktop + pinned NI Windows +image surface that adjacent local proof packets should use before another +hosted rerun is chosen. + +## Surface View + +| Surface | Responsibility | Technology | +| --- | --- | --- | +| Readiness probe surface | Emit a bounded receipt for shared Windows Docker Desktop + NI image readiness | PowerShell + Docker | +| Bootstrap and preflight surface | Prepare and validate the Windows host deterministically | PowerShell + Docker | +| Bridge surface | Reach Windows-local PowerShell and Node execution from a Unix or WSL coordinator | Node.js + PowerShell | +| Path-hygiene surface | Detect synced or externally managed roots such as OneDrive-managed paths before live proof | Node.js | +| Windows-local staging surface | Stage UNC-backed or otherwise non-bindable Windows inputs and outputs into a local mount root for Docker consumption | PowerShell + Docker | +| Local autonomy surface | Emit ranked local guidance, proof checks, and next-step handoffs for the shared surface | Node.js + assurance packet | + +## Component View + +| Component | Container | Responsibility | +| --- | --- | --- | +| `Invoke-PesterWindowsContainerSurfaceProbe.ps1` | Readiness probe surface | Emit `pester-windows-container-surface.json` with bounded surface states | +| `Test-WindowsNI2026q1HostPreflight.ps1` | Bootstrap and preflight surface | Emit deterministic Windows host preflight contracts | +| `Run-NIWindowsContainerCompare.ps1` | Bootstrap and preflight surface | Provide the bounded compare probe on the shared NI Windows image surface | +| `Run-NIWindowsContainerCompare.ps1` | Windows-local staging surface | Stage UNC-backed WSL container-bound paths into a Windows-local mount root and synchronize report artifacts back | +| `windows-host-bridge.mjs` | Bridge surface | Resolve reachable Windows PowerShell/Node entrypoints and run governed Windows-local work from Unix or WSL | +| `windows-docker-shared-surface-local-ci.mjs` | Path-hygiene + local autonomy surfaces | Detect OneDrive-like risks, synthesize the packet, and emit next-step handoffs | + +## Design Constraints + +- The shared Windows surface should stay packetized separately from Pester and + VI History. +- OneDrive-like managed roots should be treated as explicit proof risk on the + shared surface instead of an incidental environment concern. +- The shared surface should remain the first-class owner of + `windows-docker-desktop-ni-image` escalation semantics. +- When a reachable Windows Desktop exists behind Unix or WSL, the bridge + surface should execute Windows-local probe and preflight work before the + packet emits a host-unavailable escalation. +- UNC-backed WSL repo paths should never be passed straight to Docker bind + mounts on the Windows surface; the staging surface should own that + translation and synchronize the output back to the requested repo paths. diff --git a/docs/knowledgebase/Local-Proof-Autonomy-Program.md b/docs/knowledgebase/Local-Proof-Autonomy-Program.md new file mode 100644 index 000000000..4f5842f95 --- /dev/null +++ b/docs/knowledgebase/Local-Proof-Autonomy-Program.md @@ -0,0 +1,75 @@ +# CompareVI Local Proof Autonomy Program + +The local proof program is the shared control plane above the packet-local +loops. Its job is to keep packet authority local while giving an LLM one +machine-readable next step across sibling proof surfaces. + +## Packet Inputs + +- `Pester Service Model` + - `priority:pester:local-ci` + - `tests/results/_agent/pester-service-model/local-ci/pester-service-model-next-step.json` +- `VI History Local Proof` + - `priority:vi-history:local-ci` + - `tests/results/_agent/vi-history-local-proof/local-ci/vi-history-local-next-step.json` +- `Windows Docker Shared Surface` + - `priority:windows-surface:local-ci` + - `tests/results/_agent/windows-docker-shared-surface/local-ci/windows-docker-shared-surface-next-step.json` + +## Selection Rules + +- Requirement work outranks escalation work. +- Active or regressed packet work outranks passive packet work. +- Shared escalations to the same external surface are merged into one handoff. +- The first shared external surface to merge today is the shared `windows-docker-desktop-ni-image` surface. +- The shared Windows surface may also be selected directly as requirement work instead of only appearing as a merged escalation. +- When all tracked packet-local work is complete, the program should emit a + machine-readable post-local promotion escalation instead of `null`. +- Packet-local CI surfaces should use run-scoped audit bundle roots so a packet + CI and the shared selector can overlap without deleting each other’s + `surface-bundle` workspace. + +## Program Commands + +- `npm run priority:program:local-ci` +- `npm run priority:program:next-step` + +## Canonical Artifacts + +- `tests/results/_agent/local-proof-program/local-ci/comparevi-local-program-ci-report.json` +- `tests/results/_agent/local-proof-program/local-ci/comparevi-local-program-next-step.json` +- `tests/results/_agent/local-proof-program/local-ci/comparevi-local-program-ci-summary.md` + +## Shared Windows Surface Rule + +When both the Pester packet and the VI History packet require the shared +`windows-docker-desktop-ni-image` surface, the program selector should emit one +merged escalation packet instead of two competing advisories. That merged packet +should preserve: + +- packet identities +- governing requirements +- blocked requirements +- proof-check identifiers +- receipt paths +- exact next commands + +This keeps the next truthful local step bounded even when more than one packet +is waiting on the same Windows Docker Desktop + NI image proof surface. + +Now that the shared Windows surface is its own packet, the program selector +should also be able to choose `Windows Docker Shared Surface` requirement work +explicitly when the next gap belongs to the shared surface itself rather than to +Pester or VI History. + +## Post-Local Promotion Rule + +When every tracked packet reports `next step: none`, the program should not end +with a silent `null`. It should emit one bounded promotion escalation that +states: + +- local packet work is complete +- the next truthful surface is integration or hosted proof +- the governing program requirement for that transition +- the packet set whose local evidence is ready +- the stop conditions for returning from hosted proof to local packet work diff --git a/docs/knowledgebase/Pester-Service-Model.md b/docs/knowledgebase/Pester-Service-Model.md index f48bd4fa1..9e64a302d 100644 --- a/docs/knowledgebase/Pester-Service-Model.md +++ b/docs/knowledgebase/Pester-Service-Model.md @@ -30,13 +30,44 @@ The additive pilot introduces seven workflow surfaces: - `tools/Run-PesterExecutionOnly.Local.ps1` - local harness for the execution slice without the workflow shell: lock, LV guard, fixture prep, dispatcher profile, dispatch, execution postprocess, and local execution receipt +- `tools/PesterExecutionPacks.ps1` + - shared named execution-pack catalog and resolution contract used by selection, + direct dispatcher use, and local-first entrypoints - `tools/Invoke-PesterExecutionPostprocess.ps1` - execution-post contract for XML integrity classification and machine-readable summary repair +- `tools/Invoke-PesterExecutionFinalize.ps1` + - finalize contract for summary, artifact manifest, session index, compare-report indexing, and leak-report materialization from dispatcher-owned raw outputs +- `tools/Invoke-PesterExecutionPublication.ps1` + - publication contract for step-summary, session-summary, diagnostics, and operator-facing metadata outside the dispatcher +- `tools/Invoke-PesterExecutionTelemetry.ps1` + - durable telemetry contract that normalizes `dispatcher-events.ndjson` and handshake markers into `pester-execution-telemetry.json` +- `tools/PesterServiceModelSchema.ps1` + - shared schema-governance contract that validates retained receipts and derived artifacts before replay, postprocess, or evidence consumers trust them +- `tools/PesterFailurePayload.ps1` + - shared failure-detail contract that keeps dispatcher, finalize, step-summary, and top-failure readers on the same canonical payload and unavailable-details semantics +- `tools/PesterPathHygiene.ps1` + - local path-hygiene contract that classifies synchronized or externally managed roots before the harness writes results or acquires session locks +- `tools/Invoke-PesterEvidenceClassification.ps1` + - shared evidence-classification contract used by the hosted evidence workflow and local retained-artifact replay +- `tools/Invoke-PesterOperatorOutcome.ps1` + - shared operator-outcome contract that turns evidence classification into machine-readable gate status, reasons, and next-action guidance +- `tools/Invoke-PesterEvidenceProvenance.ps1` + - shared provenance contract that records exact source raw artifacts, receipt identity, run context, and derived evidence outputs for hosted evidence and local replay +- `tools/Write-PesterTotals.ps1` + - shared totals writer used by hosted evidence and local replay +- `docs/pester-service-model-promotion-comparison.json` + - retained requirement-to-run comparison packet used by the promotion dossier to compare representative named packs against the current baseline +- `tools/Replay-PesterServiceModelArtifacts.Local.ps1` + - local retained-artifact replay entrypoint for postprocess, summary, totals, session index, and evidence classification without rerunning dispatch +- `tools/Invoke-PesterWindowsContainerSurfaceProbe.ps1` + - bounded local Windows-container surrogate that records whether Docker Desktop Windows engine plus the pinned NI Windows image are ready before another hosted rerun is chosen ## Design Rules - Context certifies repo/control-plane assumptions. It does not probe host readiness or execute tests. - Selection resolves integration mode, include patterns, and dispatcher profile into a receipt before execution begins. +- Selection should resolve a named execution pack or test group as the operator-facing contract. `IncludePatterns` may refine a declared pack, but they should not remain the only externally visible control surface. +- The named execution-pack contract currently exposes `full`, `comparevi`, `dispatcher`, `workflow`, `fixtures`, `psummary`, `schema`, and `loop`. - Selection consumes context. It does not probe host readiness or invoke the dispatcher. - Readiness certifies the environment. It does not execute the test pack. - Readiness consumes context. It does not discover standing-priority state itself. @@ -44,11 +75,48 @@ The additive pilot introduces seven workflow surfaces: - Execution consumes context, selection, and readiness. It does not normalize pack inputs, choose the dispatcher profile, bootstrap Docker runtimes, install core toolchains, or discover standing-priority state. - Execution writes an execution receipt before uploading raw artifacts so evidence can classify the real seam outcome. - Execution must also emit a skip-safe execution contract from an always-on finalize path so reusable-workflow outputs do not collapse when the execution job never starts. +- Execution must preserve control-plane outputs even when in-process tests mutate workflow environment variables; dispatcher exit-code evidence must remain recoverable from a durable trace. +- Long-running execution should emit durable progress telemetry so operators can inspect forward progress even when GitHub withholds live logs until job completion. The retained contract is `pester-execution-telemetry.json`, derived from `dispatcher-events.ndjson` plus handshake markers. - Execution-post shall classify `results-xml-truncated`, `invalid-results-xml`, and `missing-results-xml` explicitly instead of collapsing XML integrity debt into generic `seam-defect`. - Local iteration must not depend on the workflow shell. The local harness should make the execution slice runnable on its own while keeping the same preflight and receipt boundaries. -- Evidence consumes raw execution output plus the execution receipt. It classifies `context-blocked`, `readiness-blocked`, and `seam-defect` explicitly instead of collapsing them into one execution symptom. +- `PathHygieneMode` supports `auto`, `relocate`, `block`, and `off`. The + default local mode is `auto`: risky OneDrive-like roots relocate to a safe + local root, while `block` emits `status=path-hygiene-blocked` from a safe + receipt root before dispatch starts. +- Local iteration must also avoid synchronized or externally managed roots for results and session-lock state. OneDrive-like paths are path-hygiene risk unless the harness explicitly relocates or blocks them before dispatch. +- Retained raw and contract artifacts should be treated as replay inputs. Non-host-dependent layers should be reproducible locally from mounted artifacts instead of forcing another GitHub run. +- Representative retained-artifact replay should stay current with real GitHub seams, not only synthetic fixtures. Schema-lite summaries and legacy receipts missing optional pack fields should normalize into current contracts instead of crashing replay. +- When a live local proof is needed, the Windows-container surrogate should be checked first. The bounded local commands are `tests:windows-surface:probe`, `docker:ni:windows:bootstrap`, and `compare:docker:ni:windows:probe`. +- A reachable Windows host bridge from WSL or another Unix coordinator should be consumed before the packet emits a host-unavailable Windows-surface escalation. +- Receipt and artifact schemas are part of the control plane. Readers should validate compatible schema versions explicitly and fail closed on unsupported changes. +- `unsupported-schema` is a first-class execution or evidence outcome. Execution-post, hosted evidence, and local replay should surface that classification with contract-specific reasons such as `execution-receipt-unsupported-schema` or `pester-summary-unsupported-schema-version`. +- Evidence consumes raw execution output plus the execution receipt. It must first normalize the downloaded raw artifact into the canonical evidence workspace, then classify `context-blocked`, `readiness-blocked`, and `seam-defect` explicitly instead of collapsing them into one execution symptom. +- Evidence should also materialize `pester-operator-outcome.json` so machine consumers and humans see the same gate status, classification, reasons, and next-action guidance. +- Derived evidence and promotion artifacts should retain provenance to the exact raw artifacts, receipts, runs, and refs they were generated from. +- The canonical provenance surfaces are `pester-evidence-provenance.json`, + `release-evidence-provenance.json`, and + `promotion-dossier-provenance.json`. +- The canonical promotion-comparison surface is + `pester-service-model-promotion-comparison.json`. +- Failure detail is a versioned interface, not a convenience file. Readers must accept current and legacy payload shapes and degrade truthfully when summary counts show failures but detail entries are unavailable. +- Failure detail also has a producer-side truth requirement: execution must not emit `failed > 0` alongside an empty detail payload unless it also emits an explicit unavailable-details state that evidence can surface. +- The canonical producer payload is `pester-failures@v2`. It carries `detailStatus`, `detailCount`, optional `unavailableReason`, and normalized `results` entries so local and CI readers can surface the same truth without guessing from an empty array. +- The canonical summary fields are `failureDetailsStatus`, `failureDetailsCount`, and `failureDetailsReason`. Those fields let machine-readable consumers distinguish real zero-failure cases from degraded or unavailable detail capture. +- Dispatch now stops at test execution and immediate machine capture. Summary emission, session indexing, leak reporting, artifact manifest generation, and step-summary or publication behavior are owned by explicit finalize, postprocess, and evidence helpers so `Invoke-PesterTests.ps1` no longer owns the operator-facing side effects directly. +- Common operator workflows should be exposed through stable named entrypoints or wrappers rather than forcing maintainers to remember raw dispatcher arguments for each pack. +- Stable local-first entrypoints now include `tests:pack:comparevi`, `tests:pack:dispatcher`, and `tests:pack:workflow`, which route through the same named execution-pack contract as CI. +- The retained-artifact replay entrypoint is `tests:replay:local`. It stages a mounted raw artifact plus a retained execution receipt into a local workspace and rebuilds postprocess, totals, session index, and evidence outputs without rerunning the full workflow. +- The representative retained-artifact replay entrypoint is `tests:replay:representative`. It replays a reduced live-run fixture with truncated XML and schema-lite summary so replay compatibility regressions are caught locally before another GitHub run. +- Local replay also materializes or reuses `pester-execution-telemetry.json`, so retained artifacts can surface last-known phase and event counts without rerunning dispatch. +- When the gate fails, the operator should get an explicit classification, reason chain, and next-step context before the job exits nonzero. +- The canonical machine-readable operator surface is `pester-operator-outcome.json`. Step summary, top-failure rendering, replay, and gate propagation should consume that same outcome contract instead of inventing divergent failure wording. +- Local autonomy should also be bounded by the packet: local CI should emit a ranked requirement backlog and a selected next requirement so LLM-driven development stays inside the declared assurance surface. +- Local autonomy should be policy-driven, not improvised. The local CI loop should record active worktree matches, preferred commands, stop conditions, and escalation conditions so an agent knows when to keep iterating locally and when to seek hosted proof. +- Local autonomy should also consume proof checks, not just RTM status. The loop should reopen implemented requirements when proof checks regress, while `pester-windows-container-surface.json` records whether the Windows-container surrogate is ready, advisory, or unavailable from the current host. +- When the next truthful proof surface is unavailable from the current host, local autonomy should emit a machine-readable escalation step instead of a human-only advisory. The canonical handoff artifact is `pester-service-model-next-step.json`. +- When more than one local proof packet exists, the shared program selector should reconcile them into one next step. Requirement work still outranks escalations, and shared `windows-docker-desktop-ni-image` escalations should merge into one `comparevi-local-program-next-step.json` handoff instead of competing packet advisories. - The existing required gate remains in place until the pilot proves equivalent or better behavior. - Trusted PR proving must stay on `pull_request_target` with same-owner gating. Cross-owner fork heads are not allowed to drive self-hosted execution. @@ -60,6 +128,7 @@ The pilot can replace the monolith only after: - selection receipts resolve the declared pack and dispatcher profile without execution-side drift - execution runs the declared pack without host bootstrap - evidence produces deterministic classifications +- retained promotion evidence compares representative named packs against the current baseline - PR/release comparisons show better failure localization and lower operator ambiguity ## Promotion Packet @@ -70,6 +139,11 @@ The current upstream promotion packet for the pilot is hosted-first: - `docs/rtm-pester-service-model.csv` - `.github/workflows/pester-service-model-quality.yml` - `.github/workflows/pester-service-model-release-evidence.yml` +- `tools/priority/pester-service-model-local-ci.mjs` +- `tools/priority/pester-service-model-autonomy-policy.json` +- `tests/results/_agent/pester-service-model/local-ci/pester-service-model-next-step.json` +- `tools/priority/comparevi-local-program-ci.mjs` +- `tests/results/_agent/local-proof-program/local-ci/comparevi-local-program-next-step.json` That packet is derived from the retained fork dossier on `#2078` and is used to justify the next minimal upstream slice on `#2069`. diff --git a/docs/knowledgebase/VI-History-Local-Proof.md b/docs/knowledgebase/VI-History-Local-Proof.md new file mode 100644 index 000000000..f0b8a07fd --- /dev/null +++ b/docs/knowledgebase/VI-History-Local-Proof.md @@ -0,0 +1,103 @@ +# VI History Local Proof + +VI History should be treated as an explicit sibling proof surface beside the +Pester service model, not as an incidental side effect of Pester work. + +## Local Proof Surfaces + +- `priority:workflow:replay:windows:vi-history` + - governed Windows workflow replay for `vi-history-scenarios-windows` +- `history:local:proof` + - canonical proof profile for local VI History refinement +- `history:local:refine` + - `dev-fast` local refinement profile +- `history:local:operator:review` + - operator wrapper for `dev-fast` +- `history:local:operator:warm` + - operator wrapper for `warm-dev` +- `history:local:operator:windows-mirror:proof` + - operator wrapper for `windows-mirror-proof` + +## Design Rules + +- VI History local proof should use declared profiles instead of ad hoc Docker + commands. +- `windows-mirror-proof` is pinned to the canonical NI Windows image and should + remain proof-oriented, not become an unmanaged acceleration shortcut. +- Workflow replay, local refinement, operator session, and workflow-readiness + should remain separate contracts. +- When the next truthful proof surface is unavailable from the current host, + local autonomy should emit a machine-readable escalation step instead of a + prose-only advisory. +- The VI History packet should reuse the shared `windows-docker-desktop-ni-image` proof surface with Pester rather than inventing a second unmanaged Windows proof lane. +- When a reachable Windows Desktop is available behind WSL or another Unix coordinator, the VI History packet should consume that bridge before emitting a `windows-docker-desktop-ni-image` escalation. +- When the replay lane runs from a UNC-backed WSL checkout through the shared + Windows Docker surface, container-bound inputs and output targets should be + staged into a Windows-local mount root and synchronized back to the + requested repo paths instead of being passed straight to Docker bind mounts. +- The local VI History packet should govern one explicit clone-backed + live-history candidate instead of leaving candidate choice implicit + in maintainer memory. +- The current governed live-history candidate is + `ni/labview-icon-editor` at + `Tooling/deployment/VIP_Pre-Uninstall Custom Action.vi`. +- Public accepted corpus targets and local live-history iteration candidates + are allowed to differ; the corpus remains the public evidence catalog, while + the governed live-history candidate exists to drive truthful local iteration + against real git history. + +## Canonical Artifacts + +- `windows-workflow-replay-lane@v1` +- `comparevi/local-refinement@v1` +- `comparevi/local-operator-session@v1` +- `vi-history/workflow-readiness@v1` +- `vi-history-live-candidate.json` +- `vi-history/live-candidate-readiness@v1` +- `vi-history-local-next-step.json` + +## Local Commands + +- `npm run priority:workflow:replay:windows:vi-history` +- `npm run history:local:proof` +- `npm run history:local:refine` +- `npm run history:local:operator:review` +- `npm run history:local:operator:warm` +- `npm run history:local:operator:windows-mirror:proof` +- `npm run priority:vi-history:local-ci` +- `npm run priority:vi-history:next-step` +- `npm run priority:program:local-ci` +- `npm run priority:program:next-step` + +## Shared Escalation Surface + +When the current host cannot satisfy the Windows replay lane, the next truthful +step should be the shared `windows-docker-desktop-ni-image` surface. The +expected handoff packet is `vi-history-local-next-step.json`, and it should name: + +- the blocked requirement +- the governing escalation requirement +- the receipt path +- the current host state +- the exact next commands to run on the Windows host + +When more than one local proof packet exists, the shared selector should emit +`comparevi-local-program-next-step.json` and merge the shared +`windows-docker-desktop-ni-image` handoff across VI History and Pester instead +of leaving two independent advisories for a human to reconcile. + +## Governed Live-History Candidate + +The current clone-backed iteration target is governed explicitly in +`tools/priority/vi-history-live-candidate.json`: + +- repo: `ni/labview-icon-editor` +- default branch: `develop` +- target VI: + `Tooling/deployment/VIP_Pre-Uninstall Custom Action.vi` +- expected clone root: `/tmp/labview-icon-editor` + +That target was chosen because the downstream repo contains the VI on disk and +its git history is live and non-trivial. The local packet should verify the +clone, target path, and history before asking for Windows replay or hosted +proof. diff --git a/docs/knowledgebase/Windows-Docker-Shared-Surface.md b/docs/knowledgebase/Windows-Docker-Shared-Surface.md new file mode 100644 index 000000000..ca439c682 --- /dev/null +++ b/docs/knowledgebase/Windows-Docker-Shared-Surface.md @@ -0,0 +1,65 @@ +# Windows Docker Shared Surface + +The shared Windows Docker Desktop + pinned NI Windows image surface should be +treated as its own local proof packet, not as an incidental advisory merged +from sibling packets. + +## Local Proof Surfaces + +- `tests:windows-surface:probe` + - bounded readiness probe for Docker Desktop Windows engine and the pinned NI + image +- `docker:ni:windows:bootstrap` + - deterministic Windows host bootstrap and preflight +- `compare:docker:ni:windows:probe` + - bounded container compare probe on the shared Windows surface +- `priority:windows-surface:local-ci` + - machine-readable local assurance loop for the shared Windows surface + +## Design Rules + +- The shared Windows surface should govern itself explicitly instead of being + inferred only through Pester or VI History packet advisories. +- Shared-surface path hygiene should be checked before live Windows proof is + recommended. OneDrive-like managed roots are risk until a safe local root is + used. +- Bootstrap, probe, and packet-local proof should remain separate contracts. +- The shared Windows surface should participate in the same local proof program + selector as Pester and VI History. +- When the coordinator is running under WSL or another Unix host but a + reachable Windows Desktop is available, the packet should bridge into that + Windows host before emitting a `windows-docker-desktop-ni-image` escalation. +- Windows bridge invocations should use `ExecutionPolicy Bypass` for UNC-backed + repo scripts and keep results under repo-local or other safe non-OneDrive + roots. +- When Windows Docker cannot bind UNC-backed WSL paths directly, the compare + surface should stage container-bound inputs and output targets into a + Windows-local mount root, synchronize results back to the requested repo + paths, and record the staging lifecycle in the capture artifact. + +## Canonical Artifacts + +- `pester-windows-container-surface.json` +- `comparevi/windows-host-preflight@v1` +- `windows-docker-shared-surface-path-hygiene.json` +- `windows-docker-shared-surface-local-ci-report.json` +- `windows-docker-shared-surface-next-step.json` +- `ni-windows-container-capture.json` + +## Local Commands + +- `npm run tests:windows-surface:probe` +- `npm run docker:ni:windows:bootstrap` +- `npm run compare:docker:ni:windows:probe` +- `npm run priority:windows-surface:local-ci` +- `npm run priority:windows-surface:next-step` +- `npm run priority:program:local-ci` +- `npm run priority:program:next-step` + +## Shared Program Role + +This packet is the authoritative home for the shared +`windows-docker-desktop-ni-image` surface. Pester and VI History may still +escalate to that surface, but the surface itself should also be governable as a +packet with its own requirements, proof checks, bounded next-step handoff, and +reachable Windows host bridge rules. diff --git a/docs/pester-service-model-promotion-comparison.json b/docs/pester-service-model-promotion-comparison.json new file mode 100644 index 000000000..b73c496e7 --- /dev/null +++ b/docs/pester-service-model-promotion-comparison.json @@ -0,0 +1,48 @@ +{ + "schema": "pester-promotion-comparison@v1", + "schemaVersion": "1.0.0", + "generatedAtUtc": "2026-03-31T23:59:59Z", + "baselineLabel": "legacy-required-gate", + "candidateLabel": "service-model-integration-rail", + "decisionState": "blocked-pending-representative-expansion", + "summary": "The first governed comparison slice shows the service model localizing failure at evidence after execution, while the current baseline still owns release truth. Promotion stays blocked until representative passing and failure-localization comparisons expand beyond this first slice.", + "comparisons": [ + { + "comparisonId": "dispatcher-first-slice-baseline-vs-service-model", + "packId": "dispatcher", + "representativeness": "first-governed-slice", + "requirementCoverage": [ + "REQ-PSM-004", + "REQ-PSM-005", + "REQ-PSM-006", + "REQ-PSM-020", + "REQ-PSM-021" + ], + "baseline": { + "workflow": "test-pester.yml", + "runId": "23795198442", + "runUrl": "https://github.com/LabVIEW-Community-CI-CD/compare-vi-cli-action/actions/runs/23795198442", + "headSha": "c472531eb586da3ad9bff45e9385d42e38b34a2e", + "ref": "release/v0.6.10", + "conclusion": "failure", + "packIdentity": "legacy-monolith" + }, + "candidate": { + "workflow": "pester-service-model-on-label.yml", + "runId": "23818978524", + "runUrl": "https://github.com/LabVIEW-Community-CI-CD/compare-vi-cli-action/actions/runs/23818978524", + "headSha": "077ceaa05453b505b1fd6a722d488cdcd9fc781b", + "ref": "integration/pester-service-model", + "conclusion": "failure", + "packIdentity": "dispatcher" + }, + "observedDeltas": [ + "The service model retained trusted-router, context, selection, and readiness receipts before the active seam surfaced.", + "The service model isolated the failure at evidence after execution dispatch instead of collapsing the outcome into one monolithic red gate.", + "The current baseline remains the required gate and therefore still defines release truth despite lower failure localization." + ], + "decision": "not-ready", + "nextAction": "Collect at least one passing representative comparison and one additional failure-localization comparison before promotion beyond the integration rail." + } + ] +} diff --git a/docs/requirements-local-proof-autonomy-program-srs.md b/docs/requirements-local-proof-autonomy-program-srs.md new file mode 100644 index 000000000..8a8095664 --- /dev/null +++ b/docs/requirements-local-proof-autonomy-program-srs.md @@ -0,0 +1,37 @@ +# CompareVI Local Proof Autonomy Program SRS + +## Document Control + +- System: CompareVI local proof autonomy program +- Version: `v0.1.1` +- Owner: `#2069` +- Status: Active + +## Scope + +- Purpose: + Specify the shared local control plane that ranks sibling packet work and + emits the next truthful step for autonomous development. +- In scope: + Packet aggregation, requirement ranking, shared-surface escalation merging, + and post-local promotion escalation. +- Out of scope: + Packet-internal execution semantics for Pester, VI History, or Windows host + bootstrap. + +## Stakeholders + +| Role | Need | Priority | +| --- | --- | --- | +| Product | One machine-readable next step across local proof packets instead of manual interpretation | High | +| Engineering | Autonomous development should keep moving from packet work to proof promotion without a human translation layer | High | +| QA | Trace program-level ranking and escalation behavior to explicit requirements and tests | High | + +## Requirements + +| ID | Requirement | Rationale | Fit Criterion | Verification | +| --- | --- | --- | --- | --- | +| REQ-LPAP-001 | The local proof autonomy program shall consume sibling packet-local next-step artifacts and rank requirement work ahead of escalation work. | Autonomous development needs one machine-readable next step across packets, and local requirement closure should happen before external escalation. | `comparevi-local-program-ci.mjs` consumes packet-local reports and next-step artifacts from Pester, VI History, and Windows shared-surface packets, and requirement work outranks escalation work. | `TEST-LPAP-001` | +| REQ-LPAP-002 | The local proof autonomy program shall merge shared escalations to the same external proof surface into one bounded handoff packet. | Duplicate escalations to the same Windows or hosted surface create ambiguity and wasted operator cycles. | When multiple packets require the same external surface, the program emits one merged escalation packet preserving packet identities, governing requirements, proof checks, and recommended commands. | `TEST-LPAP-002` | +| REQ-LPAP-003 | When all tracked packet-local requirements are implemented and no packet-local escalation remains, the local proof autonomy program shall emit a machine-readable promotion escalation instead of `null`. | Autonomous development stalls if the selector stops at “nothing left locally” without identifying the next proof plane. | `comparevi-local-program-next-step.json` emits a bounded escalation naming the post-local proof surface, governing requirement, packet set, suggested loop, and stop conditions when local packet work is complete. | `TEST-LPAP-003` | +| REQ-LPAP-004 | Packet-local CI and the shared program selector shall use run-scoped audit-surface bundle workspaces so concurrent local invocations do not corrupt or delete each other’s evidence bundles. | Autonomous development should tolerate overlapping packet and program invocations without surfacing `ENOTEMPTY` or bundle-deletion races. | Packet-local CI scripts materialize audit surfaces beneath run-scoped `surface-bundle/run-*` roots, and concurrent packet/program invocations can overlap without deleting a shared bundle directory. | `TEST-LPAP-004` | diff --git a/docs/requirements-pester-service-model-srs.md b/docs/requirements-pester-service-model-srs.md index 12c22f64f..971140e25 100644 --- a/docs/requirements-pester-service-model-srs.md +++ b/docs/requirements-pester-service-model-srs.md @@ -3,7 +3,7 @@ ## Document Control - System: Pester service-model control plane -- Version: `v0.1.0` +- Version: `v0.1.20` - Owner: `#2069` - Basis: retained fork promotion dossier under `#2078` - Status: Active @@ -12,11 +12,17 @@ - Purpose: Specify the trusted Pester control plane that separates context, host - readiness, selection, execution, and evidence into auditable workflow surfaces. + readiness, selection, execution, finalize or postprocess, and evidence into + auditable workflow surfaces, while keeping local-first reproducibility and + named execution packs explicit. - In scope: Trusted pilot routing, repo/control-plane context certification, self-hosted - readiness receipts, selection receipts, execution-only dispatcher runs, and evidence - classification. + readiness receipts, selection receipts, named execution packs, execution-only + dispatcher runs, long-running execution observability, schema-governed + retained artifacts, local replay surfaces, evidence provenance, + machine-readable next-requirement guidance, representative retained-artifact + replay, local Windows-container surrogate proof, autonomy policy, + machine-readable next-step escalation, and evidence classification. - Out of scope: Legacy monolithic `test-pester.yml` behavior except where it remains the current baseline to compare against. @@ -41,6 +47,27 @@ | REQ-PSM-005 | The execution layer shall validate context, readiness, and selection receipts before dispatch, run the declared Pester pack without bootstrapping Docker runtimes or core toolchains, and shall emit an execution receipt even when execution is skipped. Execution-post shall run as a separate contract that can repair machine-readable summaries from raw results without re-entering dispatch. | Execution should only execute; it must not absorb context, selection, or readiness responsibilities, and XML/result postprocess debt must be classifiable separately from dispatch. | `pester-run.yml` refuses to start unless all upstream receipts are ready, calls `Invoke-PesterTests.ps1`, runs `Invoke-PesterExecutionPostprocess.ps1`, uploads raw results when produced, and always uploads `pester-run-receipt.json`; `tools/Run-PesterExecutionOnly.Local.ps1` mirrors that slice locally without the workflow shell. | `TEST-PSM-005` | | REQ-PSM-006 | The evidence layer shall classify `context-blocked`, `selection-blocked`, `readiness-blocked`, `results-xml-truncated`, `invalid-results-xml`, `missing-results-xml`, `test-failures`, and `seam-defect` explicitly from execution receipts and raw artifacts. | Operators need precise failure classes instead of `missing-summary` ambiguity. | `pester-evidence.yml` reads the execution contract and emits the explicit classification when raw artifacts are missing, execution is skipped, or result XML is truncated, invalid, or missing. | `TEST-PSM-006` | | REQ-PSM-007 | The pilot shall remain additive until it proves equivalent or better behavior than the monolithic required gate. | Promotion must follow evidence, not preference. | The service-model knowledgebase and promotion rule state that the legacy required gate remains in place until the pilot is proven. | `TEST-PSM-007` | +| REQ-PSM-008 | The execution slice shall preserve control-plane outputs across in-process environment mutation and expose a durable execution trace that receipts can recover from when step outputs collapse. | In-process Pester tests run inside the same process model as the dispatcher and can mutate process-scoped environment variables; execution receipts must not silently degrade when the dispatcher already produced authoritative output. | `pester-run.yml` pins the original workflow output path before dispatch, writes a durable execution trace, and recovers `dispatcherExitCode` from that trace before emitting `pester-run-receipt.json`; the local harness mirrors the durable trace behavior. | `TEST-PSM-008` | +| REQ-PSM-009 | The evidence slice shall normalize raw execution artifacts into the canonical evidence workspace before hosted summary, session-index, failure-detail, and dashboard analysis runs. | Hosted evidence consumers must analyze the downloaded execution outputs, not an empty workspace that only contains newly derived evidence files. | `pester-evidence.yml` stages the raw execution artifact into the canonical `tests/results` workspace before hosted evidence scripts execute, and those scripts consume the staged summary, XML, failure JSON, and session-index files from that workspace. | `TEST-PSM-009` | +| REQ-PSM-010 | Failure detail shall be treated as a versioned interface with backward-compatible readers, and when detail payloads are absent or unusable the subsystem shall degrade truthfully instead of claiming there were no failures. | Failure-detail drift between producers and consumers creates misleading CI summaries and hides defects behind false `(none)` reports. | `Write-PesterSummaryToStepSummary.ps1`, `Write-PesterTopFailures.ps1`, and `Print-PesterTopFailures.ps1` accept both current and legacy payload shapes; top-failure reporting emits an unavailable-details message when summary counts are nonzero but detail entries cannot be recovered. | `TEST-PSM-010` | +| REQ-PSM-011 | The execution layer shall keep failure-detail artifacts semantically consistent with summary counts: when summary totals report failed or errored tests, it shall emit non-empty failure detail or an explicit machine-readable unavailable-details state. | Empty failure-detail payloads alongside nonzero summary failures create silent observability loss and invalidate downstream failure summaries. | When `pester-summary.json` reports `failed + errors > 0`, execution emits `failureDetailsStatus=available` with populated `pester-failures.json` entries, or `failureDetailsStatus=unavailable` with an explicit reason in both the summary and the canonical `pester-failures@v2` payload. | `TEST-PSM-011` | +| REQ-PSM-012 | The selection and execution contract shall resolve a named execution pack or test group and preserve that identity through receipts, local entrypoints, and evidence; ad hoc `IncludePatterns` may refine a declared pack but shall not be the only externally visible control surface. | Generic dispatcher invocations hide policy intent and make promotion comparisons, local replay, and operator reasoning harder than they need to be. | The selection receipt carries a named execution-pack or test-group identifier, local and CI entrypoints expose supported pack or group names explicitly, and evidence artifacts retain the declared pack identity alongside any refining include patterns. | `TEST-PSM-012` | +| REQ-PSM-013 | The local execution harness shall detect unsafe synchronized or externally managed roots for results and session-lock state, such as OneDrive-managed directories, before it acquires locks or writes receipts, and shall relocate or block with an explicit path-hygiene state. | Sync agents and externally managed roots can mutate locks and result files outside dispatcher control, creating false session-lock conflicts and non-deterministic artifact corruption. | The local harness resolves path hygiene before directory creation or lock acquire, records both requested and effective roots in its execution receipt, relocates to a safe local root in `auto` or `relocate` mode, and emits `status=path-hygiene-blocked` from a safe receipt root in `block` mode before dispatch starts. | `TEST-PSM-013` | +| REQ-PSM-014 | Every non-host-dependent service-model layer shall be reproducible locally from retained artifacts without rerunning the full GitHub workflow. | Promotion and failure analysis are cheaper when retained artifacts can be replayed locally instead of burning hosted or self-hosted CI time. | `tools/Replay-PesterServiceModelArtifacts.Local.ps1` and `tests:replay:local` rebuild postprocess, summary, totals, session index, and evidence classification from mounted raw artifacts plus a retained execution receipt, without re-entering dispatch or rerunning GitHub workflows. | `TEST-PSM-014` | +| REQ-PSM-015 | Dispatcher responsibilities shall stop at declared test execution and immediate machine capture; finalize, postprocess, and evidence layers shall own summary, session-index, dashboard, leak-report, and publication side effects through dedicated helpers. | `Invoke-PesterTests.ps1` is still too generic; leaving operator-facing side effects inside the dispatcher keeps the execution lane monolithic and obscures the real failure surface. | `Invoke-PesterTests.ps1` emits raw execution outputs and delegates summary, session-index, artifact manifest, leak reporting, and publication side effects to `Invoke-PesterExecutionFinalize.ps1`, `Invoke-PesterExecutionPostprocess.ps1`, and `Invoke-PesterExecutionPublication.ps1`, which are traced separately in requirements and tests. | `TEST-PSM-015` | +| REQ-PSM-016 | Long-running execution shall emit durable progress telemetry that survives live-log unavailability and lets operators distinguish forward progress from deadlock. | A 30-minute self-hosted execution step with no durable progress signal is operationally indistinguishable from a stalled dispatcher until the job ends. | Execution retains `dispatcher-events.ndjson` plus `pester-execution-telemetry.json` with timestamps, pack identity, event counts, and last-known phase in the raw artifact set, and local replay surfaces can inspect or regenerate that telemetry without rerunning the workflow. | `TEST-PSM-016` | +| REQ-PSM-017 | All retained service-model receipts and derived artifacts shall be explicit schema contracts with version-governed readers; incompatible schema changes shall fail closed with explicit classification rather than silent misparse. | Local replay, evidence, and promotion all depend on file contracts; silent schema drift would invalidate assurance evidence without obvious runtime failure. | Receipts and derived artifacts carry stable schema identifiers or schema versions, and local or CI consumers validate those contracts before use or emit an explicit `unsupported-schema` classification with contract-specific reasons. | `TEST-PSM-017` | +| REQ-PSM-018 | Promotion beyond the integration rail shall be justified by retained requirement-to-run evidence that compares service-model behavior against the current baseline on representative named execution packs. | Replacing the monolith without comparison evidence would turn the pilot into an opinion instead of a governed promotion decision. | The release-evidence bundle retains `pester-service-model-promotion-comparison.json`, and the promotion dossier renders representative run links, named pack identities, requirement coverage, and observed deltas between the service model and the current baseline before the service model advances beyond the integration rail. | `TEST-PSM-018` | +| REQ-PSM-019 | Stable operator-facing entrypoints shall exist for common named execution packs or test groups so maintainers do not have to drive common workflows through the generic dispatcher surface directly. | A single generic dispatcher script is too broad an operator surface and encourages ad hoc `IncludePatterns` usage that is hard to trace and compare. | Supported named packs or groups are documented and exposed through stable local or CI entrypoints, such as dedicated scripts, wrappers, or package commands, and those entrypoints map back to the declared selection contract. | `TEST-PSM-019` | +| REQ-PSM-020 | Derived evidence and promotion artifacts shall retain provenance to the exact raw artifacts, receipts, run identities, and refs they were built from. | Without provenance, a correct-looking dashboard or promotion dossier can silently drift away from the execution evidence it is supposed to represent. | Evidence emits `pester-evidence-provenance.json`, local replay retains the same provenance contract, and hosted promotion bundle generation emits `release-evidence-provenance.json` plus `promotion-dossier-provenance.json` with source artifact identities, receipt identities, hashes, run IDs or refs, and generation timestamps. | `TEST-PSM-020` | +| REQ-PSM-021 | Gate failures shall be explainable as machine-readable operator outcomes with classification, reasons, and next-action context instead of opaque red-job signaling alone. | A failing gate that only exits nonzero still forces operators into log archaeology and weakens the value of the layered control plane. | Evidence emits `pester-operator-outcome.json` with classification, reasons, gate status, and next-action guidance, and the gate propagation path surfaces that contract before failing the job. | `TEST-PSM-021` | +| REQ-PSM-022 | Local assurance CI shall synthesize standards audit results, RTM gap status, and concrete code references into a machine-readable ranked backlog and a selected next requirement for local-first development. | Autonomous or semi-autonomous development needs more than a binary pass or fail; it needs a bounded next step that is grounded in the current assurance packet. | A local CI entrypoint emits a report, summary, and `next-requirement` artifact that rank unresolved requirements and identify the next requirement to tackle, including rationale, code refs, and suggested local loop steps. | `TEST-PSM-022` | +| REQ-PSM-023 | The local autonomy loop shall apply explicit policy, active-worktree signals, and stop conditions so an agent knows when to keep iterating locally and when to escalate to hosted proof. | Autonomous development needs bounded authority; otherwise it will either escalate too early to GitHub CI or keep editing locally after the packet says the next step is hosted evidence. | The local CI report records active worktree matches, preferred local commands, stop conditions, and escalation conditions derived from a versioned autonomy policy. | `TEST-PSM-023` | +| REQ-PSM-024 | Representative retained-artifact replay shall normalize backward-compatible legacy retained runs, including schema-lite summaries and execution receipts missing optional pack fields, without hiding the underlying evidence classification. | A local-first assurance loop is not credible if it only handles synthetic fixtures and crashes on the first real retained run from GitHub CI. | `tools/Replay-PesterServiceModelArtifacts.Local.ps1` can replay a representative schema-lite truncated-XML run, emit current summary and operator-outcome contracts, and preserve `results-xml-truncated` instead of throwing or collapsing into compatibility debt. | `TEST-PSM-024` | +| REQ-PSM-025 | A local Windows-container surrogate proof surface shall exist for Docker Desktop Windows engine and the pinned NI Windows image, and it shall emit an explicit bounded receipt before hosted reruns are chosen. | When a live proof is needed, the cheapest truthful next step is often a local Windows-container replay on the same Docker Desktop + NI image surface rather than another 30-minute GitHub run. | `Invoke-PesterWindowsContainerSurfaceProbe.ps1` emits `pester-windows-container-surface.json` with explicit statuses such as `ready`, `not-windows-host`, `docker-engine-not-windows`, or `ni-image-missing`; when a reachable Windows host bridge exists behind a Unix or WSL coordinator, the packet consumes that bridge before treating the surface as host-unavailable. | `TEST-PSM-025` | +| REQ-PSM-026 | The local autonomy loop shall consume representative proof checks, including retained-artifact replay and Windows-container surface status, and shall reopen implemented requirements when representative proof checks regress. | Static RTM status alone can overstate maturity; an autonomous loop must reopen the owning requirement when real local proof contradicts the packet. | `priority:pester:local-ci` records proof checks, refuses to report a clean `pass` when representative replay regresses, and selects the owning requirement as the next local target even when the RTM row is already marked implemented. | `TEST-PSM-026` | +| REQ-PSM-027 | When the next truthful proof surface is unavailable from the current host, local assurance CI shall emit a machine-readable escalation step that names the blocked requirement, governing requirement, required proof surface, current host state, and exact next commands. | An autonomous loop that stops at an advisory without an explicit escalation packet still depends on human interpretation and cannot hand off cleanly to another agent or host. | `priority:pester:local-ci` emits `pester-service-model-next-step.json`; when no locally actionable requirement remains and the Windows-container surface is unavailable, the next step is an escalation packet rather than `null`, with receipt path, required surface `windows-docker-desktop-ni-image`, and recommended commands. | `TEST-PSM-027` | +| REQ-PSM-028 | The Pester packet shall participate in the shared local proof program selector so autonomous development can choose between sibling packets and merge shared-surface escalations into one bounded handoff. | Once more than one local proof packet exists, two independent packet advisories force a human to reconcile them and weaken the value of the autonomy loop. | `priority:program:local-ci` consumes the Pester packet next-step artifact, can select a Pester requirement ahead of sibling packet escalations, and merges shared `windows-docker-desktop-ni-image` escalations from Pester and VI History into one `comparevi-local-program-next-step.json` handoff. | `TEST-PSM-028` | ## Assumptions diff --git a/docs/requirements-vi-history-local-proof-srs.md b/docs/requirements-vi-history-local-proof-srs.md new file mode 100644 index 000000000..24aee3340 --- /dev/null +++ b/docs/requirements-vi-history-local-proof-srs.md @@ -0,0 +1,44 @@ +# VI History Local Proof SRS + +## Document Control + +- System: VI History local proof control plane +- Version: `v0.1.3` +- Owner: `#2069` +- Status: Active + +## Scope + +- Purpose: + Specify the local-first VI History proof surfaces that should be used before + another GitHub run is chosen. +- In scope: + Windows workflow replay, local refinement profiles, local operator-session + wrappers, workflow-readiness envelopes, and local autonomy or escalation + selection for VI History proof. +- Out of scope: + Downstream release pinning, hosted PR comment publication, and release-train + promotion beyond the local proof surfaces. + +## Stakeholders + +| Role | Need | Priority | +| --- | --- | --- | +| Product | Treat VI History as an explicit sibling proof surface, not a side effect of Pester work | High | +| Engineering | Reproduce VI History seams locally before spending GitHub CI time | High | +| QA | Trace local proof surfaces to requirements, receipts, and replay contracts | High | +| Operations | Reuse the same Windows Docker Desktop + NI image surface intentionally across adjacent local proof lanes | High | + +## Requirements + +| ID | Requirement | Rationale | Fit Criterion | Verification | +| --- | --- | --- | --- | --- | +| REQ-VHLP-001 | A stable local Windows workflow-replay lane shall exist for `vi-history-scenarios-windows`, and it shall emit a workflow-grade receipt plus compare artifacts before another hosted rerun is chosen. | The Windows VI History lane is one of the most expensive places to discover breakage, so it needs a bounded local replay surface first. | `priority:workflow:replay:windows:vi-history` drives `windows-workflow-replay-lane.mjs --mode vi-history-scenarios-windows`, emits `windows-workflow-replay-lane@v1`, retains compare capture, report, runtime snapshot, stdout, and stderr paths, and can be launched through a reachable Windows host bridge from a Unix or WSL coordinator. | `TEST-VHLP-001` | +| REQ-VHLP-002 | Stable VI History local refinement profiles shall exist for `proof`, `dev-fast`, `warm-dev`, and `windows-mirror-proof`, and those profiles shall emit canonical local receipts and benchmarks. | Local VI History work should use declared profiles instead of ad hoc Docker invocations so the proof plane stays repeatable and comparable. | `Invoke-VIHistoryLocalRefinement.ps1` exposes the declared profiles, pins `windows-mirror-proof` to `nationalinstruments/labview:2026q1-windows`, and writes `local-refinement.json`, `local-refinement-benchmark.json`, and `vi-history-review-loop-receipt.json` under `tests/results/local-vi-history//`. | `TEST-VHLP-002` | +| REQ-VHLP-003 | Stable VI History local operator-session wrappers shall exist for common refinement profiles so maintainers do not need to drive the refinement helper directly for common review loops. | Operator-facing wrappers make the local proof plane easier to use consistently and cheaper to automate. | `history:local:operator:review`, `history:local:operator:warm`, and `history:local:operator:windows-mirror:proof` invoke `Invoke-VIHistoryLocalOperatorSession.ps1`, which returns the canonical local operator-session contract. | `TEST-VHLP-003` | +| REQ-VHLP-004 | The VI History lane state shall be normalized into a workflow-readiness envelope that records Windows and Linux lane status, failure class, lifecycle, and a bounded recommendation. | Multi-lane VI History proof should end in an explicit machine-readable decision surface instead of free-form log archaeology. | `Write-VIHistoryWorkflowReadiness.ps1` writes `vi-history/workflow-readiness@v1` with lane lifecycle, verdict, recommendation, and source artifact paths. | `TEST-VHLP-004` | +| REQ-VHLP-005 | Local assurance CI shall synthesize the VI History local-proof packet into a machine-readable report, ranked backlog, and next step for local-first development. | VI History needs the same bounded local guidance as the Pester packet instead of remaining outside the autonomy control plane. | A local VI History CI entrypoint emits a report, summary, and next-step artifact grounded in the packet, code refs, and proof checks. | `TEST-VHLP-005` | +| REQ-VHLP-006 | When the next truthful VI History proof surface is unavailable from the current host, local assurance CI shall emit a machine-readable escalation step to the shared `windows-docker-desktop-ni-image` surface instead of a human-only advisory. | VI History and Pester share the same Windows Docker Desktop + NI image infrastructure; the local loop should hand off to that shared surface explicitly. | When the Windows workflow replay surface is unavailable after native or bridge-backed Windows surface checks, the local VI History CI emits `vi-history-local-next-step.json` with the blocked requirement, governing requirement, required surface `windows-docker-desktop-ni-image`, current host state, receipt path, and recommended commands. | `TEST-VHLP-006` | +| REQ-VHLP-007 | The VI History packet shall participate in the shared local proof program selector so it can be chosen explicitly against sibling packets and share merged escalation handoffs when the required surface is common. | VI History should be an explicit sibling proof surface, not an orphan packet that a human has to reconcile manually against Pester. | `priority:program:local-ci` consumes `vi-history-local-next-step.json`, can select a VI History requirement ahead of sibling packet escalations, and merges shared `windows-docker-desktop-ni-image` escalations from VI History and Pester into one `comparevi-local-program-next-step.json` handoff. | `TEST-VHLP-007` | +| REQ-VHLP-008 | The VI History packet shall govern a clone-backed live-history iteration candidate, and the initial candidate shall be `ni/labview-icon-editor:Tooling/deployment/VIP_Pre-Uninstall Custom Action.vi`. | Retained fixtures and public proof seeds are useful, but they do not replace a real repo clone with actual git lineage when local maintainers need to iterate on VI History behavior. | `tools/priority/vi-history-live-candidate.json` declares the candidate id, repo slug, repo URL, default branch, clone-root override contract, target VI path, and minimum git-history expectation for `VIP_Pre-Uninstall Custom Action.vi`. | `TEST-VHLP-008` | +| REQ-VHLP-009 | Local assurance CI shall validate that the governed live-history candidate clone exists locally, contains the target VI, and exposes real git history before Windows replay or hosted proof is chosen as the next step. | There is no truthful VI History local proof for a live-history target if the repo clone, target VI, or commit history are missing. | `vi-history-local-ci.mjs` emits `vi-history-live-candidate-readiness.json` with `ready`, `missing-clone`, `missing-target`, `missing-history`, or `git-failed`, records clone and history facts, and emits a machine-readable clone-preparation escalation when the governed candidate is unavailable. | `TEST-VHLP-009` | diff --git a/docs/requirements-windows-docker-shared-surface-srs.md b/docs/requirements-windows-docker-shared-surface-srs.md new file mode 100644 index 000000000..6e024787d --- /dev/null +++ b/docs/requirements-windows-docker-shared-surface-srs.md @@ -0,0 +1,44 @@ +# Windows Docker Shared Surface SRS + +## Document Control + +- System: Windows Docker shared local proof surface +- Version: `v0.2.1` +- Owner: `#2069` +- Status: Active + +## Scope + +- Purpose: + Specify the shared Windows Docker Desktop + pinned NI Windows image proof + surface that adjacent packets should use before another hosted rerun is + chosen. +- In scope: + Bounded readiness probes, deterministic host bootstrap and preflight, + OneDrive-safe local path hygiene, local assurance CI, and shared local proof + program integration. +- Out of scope: + Product-layer Pester execution, VI History replay semantics, and hosted trust + routing. + +## Stakeholders + +| Role | Need | Priority | +| --- | --- | --- | +| Product | Avoid spending GitHub CI on defects that should be found on the shared Windows surface first | High | +| Engineering | Use one explicit Windows Docker Desktop + NI image packet across Pester and VI History instead of duplicating surface logic | High | +| QA | Trace shared-surface readiness, path hygiene, and escalation contracts to requirements and tests | High | +| Operations | Keep Windows Docker host preparation bounded, machine-readable, and safe from synced-root hazards | High | + +## Requirements + +| ID | Requirement | Rationale | Fit Criterion | Verification | +| --- | --- | --- | --- | --- | +| REQ-WDSS-001 | A stable shared-surface readiness probe shall exist for Docker Desktop Windows engine plus the pinned NI Windows image, and it shall emit a machine-readable receipt with explicit readiness and host-unavailable states. | The cheapest truthful next proof often depends on knowing whether the shared Windows Docker surface is actually ready before any packet tries to use it. | `Invoke-PesterWindowsContainerSurfaceProbe.ps1` emits `pester-windows-container-surface.json` with bounded statuses such as `ready`, `not-windows-host`, `docker-cli-missing`, `docker-engine-not-windows`, and `ni-image-missing`. | `TEST-WDSS-001` | +| REQ-WDSS-002 | A deterministic Windows host bootstrap and preflight contract shall exist for the pinned NI image surface, separate from packet-specific execution. | Shared infrastructure should be certified once and then reused by adjacent proof packets instead of each packet improvising its own host preparation. | `Test-WindowsNI2026q1HostPreflight.ps1` emits `comparevi/windows-host-preflight@v1`, and the packet documents the bounded bootstrap plus probe command order. | `TEST-WDSS-002` | +| REQ-WDSS-003 | The shared Windows proof packet shall detect unsafe synchronized or externally managed local roots, such as OneDrive-managed paths, before it recommends live Windows proof work. | Synced roots can introduce background file mutation and non-deterministic behavior during local container proof, so path hygiene must be explicit instead of assumed. | Local CI emits `windows-docker-shared-surface-path-hygiene.json`, records repo and results-root risk assessment, and emits a machine-readable relocation escalation when a managed root is detected. | `TEST-WDSS-003` | +| REQ-WDSS-004 | Local assurance CI shall synthesize the shared Windows surface packet into a machine-readable report, ranked backlog, proof checks, and next step for local-first development. | The shared surface should govern itself explicitly instead of existing only as a merged advisory produced by sibling packets. | A dedicated Windows shared-surface local CI entrypoint emits a report, summary, and next-step artifact grounded in the packet, code refs, and proof checks. | `TEST-WDSS-004` | +| REQ-WDSS-005 | When the next truthful shared-surface proof is unavailable from the current host, local assurance CI shall emit a machine-readable escalation to `windows-docker-desktop-ni-image` with exact next commands. | Autonomous development needs a bounded handoff packet instead of a prose-only note when the current host cannot satisfy the shared Windows surface. | When the probe or host state prevents live proof, local CI emits `windows-docker-shared-surface-next-step.json` naming the blocked requirement, governing requirement, required surface, host state, receipt path, and recommended commands. | `TEST-WDSS-005` | +| REQ-WDSS-006 | The shared Windows surface packet shall participate in the shared local proof program selector so it can be chosen explicitly beside Pester and VI History. | Once the Windows surface becomes a first-class packet, the autonomy loop should be able to reason about it directly rather than only through merged packet advisories. | `priority:program:local-ci` consumes the shared-surface next-step artifact, can select a shared-surface requirement ahead of sibling packet escalations, and merges common `windows-docker-desktop-ni-image` handoffs across all packets. | `TEST-WDSS-006` | +| REQ-WDSS-007 | When a reachable Windows host bridge exists behind a Unix or WSL coordinator, the shared Windows surface packet shall use that governed bridge to execute Windows-local probe and preflight work before emitting a host-unavailable escalation. | A WSL-based operator should not stop at a human handoff if the actual Windows Docker Desktop + NI image surface is already reachable from the same session. | `windows-host-bridge.mjs` resolves the reachable Windows PowerShell and Node surfaces, the shared-surface local CI uses that bridge to run `Invoke-PesterWindowsContainerSurfaceProbe.ps1` and `Test-WindowsNI2026q1HostPreflight.ps1`, and host-unavailable escalation is only emitted when that bridge is absent or the Windows surface still reports non-ready. | `TEST-WDSS-007` | +| REQ-WDSS-008 | When the coordinator is running from a UNC-backed WSL or other non-bindable Windows path, the shared Windows Docker surface shall stage container-bound inputs and output targets into a governed Windows-local mount root and synchronize artifacts back to the requested repo paths. | Windows Docker bind mounts do not reliably accept UNC-backed WSL paths, so local proof needs an explicit staging contract instead of rediscovering mount-spec failures at runtime. | `Run-NIWindowsContainerCompare.ps1` detects UNC-backed container-bound inputs or report paths, emits staging metadata in `ni-windows-container-capture.json`, uses a Windows-local stage root for Docker bind mounts, synchronizes report artifacts back to the requested repo paths, and records cleanup status. | `TEST-WDSS-008` | diff --git a/docs/rtm-local-proof-autonomy-program.csv b/docs/rtm-local-proof-autonomy-program.csv new file mode 100644 index 000000000..b8c78015c --- /dev/null +++ b/docs/rtm-local-proof-autonomy-program.csv @@ -0,0 +1,5 @@ +ReqID,Requirement,Source,Priority,TestID,TestArtifact,CodeRef,Status +REQ-LPAP-001,"The local proof autonomy program consumes sibling packet-local next-step artifacts and ranks requirement work ahead of escalation work","docs/requirements-local-proof-autonomy-program-srs.md",High,TEST-LPAP-001,"tools/priority/__tests__/comparevi-local-program-ci.test.mjs","tools/priority/comparevi-local-program-ci.mjs;docs/knowledgebase/Local-Proof-Autonomy-Program.md;package.json",Implemented +REQ-LPAP-002,"The local proof autonomy program merges shared escalations to the same external proof surface into one bounded handoff packet","docs/requirements-local-proof-autonomy-program-srs.md",High,TEST-LPAP-002,"tools/priority/__tests__/comparevi-local-program-ci.test.mjs","tools/priority/comparevi-local-program-ci.mjs;docs/knowledgebase/Local-Proof-Autonomy-Program.md;docs/schemas/comparevi-local-program-next-step-v1.schema.json",Implemented +REQ-LPAP-003,"When all tracked packet-local requirements are implemented and no packet-local escalation remains, the local proof autonomy program emits a machine-readable promotion escalation instead of null","docs/requirements-local-proof-autonomy-program-srs.md",High,TEST-LPAP-003,"tools/priority/__tests__/comparevi-local-program-ci.test.mjs","tools/priority/comparevi-local-program-ci.mjs;docs/knowledgebase/Local-Proof-Autonomy-Program.md;docs/schemas/comparevi-local-program-next-step-v1.schema.json;docs/schemas/comparevi-local-program-ci-report-v1.schema.json",Implemented +REQ-LPAP-004,"Packet-local CI and the shared program selector use run-scoped audit-surface bundle workspaces so concurrent local invocations do not corrupt each other","docs/requirements-local-proof-autonomy-program-srs.md",High,TEST-LPAP-004,"tools/priority/__tests__/comparevi-local-program-ci.test.mjs;tools/priority/__tests__/pester-service-model-local-ci.test.mjs;tools/priority/__tests__/vi-history-local-ci.test.mjs;tools/priority/__tests__/windows-docker-shared-surface-local-ci.test.mjs","tools/priority/comparevi-local-program-ci.mjs;tools/priority/pester-service-model-local-ci.mjs;tools/priority/vi-history-local-ci.mjs;tools/priority/windows-docker-shared-surface-local-ci.mjs",Implemented diff --git a/docs/rtm-pester-service-model.csv b/docs/rtm-pester-service-model.csv index 3ed84be37..0094af634 100644 --- a/docs/rtm-pester-service-model.csv +++ b/docs/rtm-pester-service-model.csv @@ -6,3 +6,24 @@ REQ-PSM-004,"Selection resolves integration mode, include patterns, and dispatch REQ-PSM-005,"Execution validates context, readiness, and selection receipts, runs Invoke-PesterTests without host bootstrap, invokes execution-post for XML/result postprocess, and emits an execution receipt even when skipped","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-005,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs;tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs;tests/Invoke-PesterExecutionPostprocess.Tests.ps1",".github/workflows/pester-run.yml;Invoke-PesterTests.ps1;tools/Invoke-PesterExecutionPostprocess.ps1;tools/Run-PesterExecutionOnly.Local.ps1",Implemented REQ-PSM-006,"Evidence classifies context-blocked, selection-blocked, readiness-blocked, results-xml-truncated, invalid-results-xml, missing-results-xml, test-failures, and seam-defect explicitly from the execution contract","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-006,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs;tests/Invoke-PesterExecutionPostprocess.Tests.ps1",".github/workflows/pester-evidence.yml;tools/Invoke-PesterExecutionPostprocess.ps1",Implemented REQ-PSM-007,"The pilot remains additive until it proves equivalent or better than the monolith","docs/requirements-pester-service-model-srs.md",Medium,TEST-PSM-007,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs","docs/knowledgebase/Pester-Service-Model.md;.github/workflows/pester-gate.yml",Implemented +REQ-PSM-008,"Execution preserves control-plane outputs across in-process environment mutation and recovers dispatcher exit code from a durable trace when step outputs collapse","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-008,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs;tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs",".github/workflows/pester-run.yml;tools/Run-PesterExecutionOnly.Local.ps1",Implemented +REQ-PSM-009,"Evidence normalizes raw execution artifacts into the canonical evidence workspace before hosted analysis runs","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-009,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs",".github/workflows/pester-evidence.yml",Implemented +REQ-PSM-010,"Failure detail is governed as a versioned interface with backward-compatible readers and truthful unavailable-details degradation","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-010,"tests/Write-PesterSummaryToStepSummary.Tests.ps1;tests/Write-PesterSummaryToStepSummary.CompactMode.Tests.ps1;tests/PesterFailurePayloadShape.Tests.ps1","scripts/Write-PesterSummaryToStepSummary.ps1;tools/Write-PesterTopFailures.ps1;tools/Print-PesterTopFailures.ps1",Implemented +REQ-PSM-011,"Execution keeps failure-detail artifacts semantically consistent with summary counts by emitting populated failure detail or an explicit unavailable-details state","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-011,"tests/PesterFailureProducerConsistency.Tests.ps1;tests/Invoke-PesterExecutionFinalize.Tests.ps1;tests/PesterFailurePayloadShape.Tests.ps1;tests/Write-PesterSummaryToStepSummary.Tests.ps1;tests/Write-PesterSummaryToStepSummary.CompactMode.Tests.ps1;tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs","Invoke-PesterTests.ps1;tools/Invoke-PesterExecutionFinalize.ps1;tools/PesterFailurePayload.ps1;scripts/Write-PesterSummaryToStepSummary.ps1;tools/Write-PesterTopFailures.ps1;tools/Print-PesterTopFailures.ps1",Implemented +REQ-PSM-012,"Selection and execution preserve a named execution pack or test group through receipts, local entrypoints, and evidence; IncludePatterns are only refinements","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-012,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs;tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs;tests/PesterExecutionPacks.Tests.ps1",".github/workflows/pester-selection.yml;package.json;Invoke-PesterTests.ps1;tools/Run-PesterExecutionOnly.Local.ps1;tools/PesterExecutionPacks.ps1",Implemented +REQ-PSM-013,"The local harness detects unsafe synchronized or externally managed results and session-lock roots, such as OneDrive-managed directories, and relocates or blocks explicitly","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-013,"PesterPathHygiene.Tests.ps1;Run-PesterExecutionOnly.Local.PathHygiene.Tests.ps1","tools/PesterPathHygiene.ps1;tools/Run-PesterExecutionOnly.Local.ps1;tools/Session-Lock.ps1",Implemented +REQ-PSM-014,"Non-host-dependent service-model layers are reproducible locally from retained artifacts without rerunning the full workflow","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-014,"Invoke-PesterEvidenceClassification.Tests.ps1;Replay-PesterServiceModelArtifacts.Local.Tests.ps1",".github/workflows/pester-evidence.yml;tools/Invoke-PesterExecutionPostprocess.ps1;tools/Invoke-PesterEvidenceClassification.ps1;tools/Replay-PesterServiceModelArtifacts.Local.ps1;tools/Write-PesterTotals.ps1",Implemented +REQ-PSM-015,"Dispatcher side effects are decomposed so finalize, postprocess, and evidence own summary, session-index, dashboard, leak-report, and publication behavior","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-015,"tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs;tests/Invoke-PesterExecutionFinalize.Tests.ps1;tests/Invoke-PesterExecutionPublication.Tests.ps1;tests/Invoke-PesterTests.Summary.Tests.ps1","Invoke-PesterTests.ps1;tools/Invoke-PesterExecutionFinalize.ps1;tools/Invoke-PesterExecutionPostprocess.ps1;tools/Invoke-PesterExecutionPublication.ps1;scripts/Write-PesterSummaryToStepSummary.ps1;tools/Write-SessionIndexSummary.ps1",Implemented +REQ-PSM-016,"Long-running execution emits durable progress telemetry that survives live-log unavailability and distinguishes forward progress from deadlock","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-016,"tests/Invoke-PesterExecutionTelemetry.Tests.ps1;tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs;tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs;tests/Replay-PesterServiceModelArtifacts.Local.Tests.ps1","Invoke-PesterTests.ps1;.github/workflows/pester-run.yml;tools/Invoke-PesterExecutionTelemetry.ps1;tools/Run-PesterExecutionOnly.Local.ps1;tools/Replay-PesterServiceModelArtifacts.Local.ps1",Implemented +REQ-PSM-017,"Receipts and derived artifacts are schema contracts with version-governed readers that fail closed on incompatible changes","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-017,"tests/PesterServiceModelSchema.Tests.ps1;tests/Invoke-PesterExecutionPostprocess.Tests.ps1;tests/Invoke-PesterEvidenceClassification.Tests.ps1;tests/Replay-PesterServiceModelArtifacts.Local.Tests.ps1;tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs","tools/PesterServiceModelSchema.ps1;tools/Invoke-PesterExecutionPostprocess.ps1;tools/Invoke-PesterEvidenceClassification.ps1;tools/Replay-PesterServiceModelArtifacts.Local.ps1;.github/workflows/pester-run.yml;.github/workflows/pester-evidence.yml",Implemented +REQ-PSM-018,"Promotion beyond the integration rail requires retained requirement-to-run comparison evidence against the current baseline on representative named packs","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-018,"tools/priority/__tests__/pester-service-model-release-evidence-provenance.test.mjs;tools/priority/__tests__/pester-service-model-release-evidence-workflow-contract.test.mjs;tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs","docs/pester-service-model-promotion-comparison.json;docs/knowledgebase/Pester-Service-Model.md;docs/rtm-pester-service-model.csv;.github/workflows/pester-service-model-release-evidence.yml;tools/priority/render-pester-service-model-promotion-dossier.mjs",Implemented +REQ-PSM-019,"Stable operator-facing entrypoints exist for common named execution packs or test groups so common workflows do not depend on the generic dispatcher alone","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-019,"tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs;tests/PesterExecutionPacks.Tests.ps1","package.json;Invoke-PesterTests.ps1;tools/Run-PesterExecutionOnly.Local.ps1;tools/Invoke-CompareCli.ps1;tools/PesterExecutionPacks.ps1",Implemented +REQ-PSM-020,"Derived evidence and promotion artifacts retain provenance to the exact raw artifacts, receipts, run identities, and refs they were built from","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-020,"tests/Invoke-PesterEvidenceProvenance.Tests.ps1;tests/Replay-PesterServiceModelArtifacts.Local.Tests.ps1;tools/priority/__tests__/pester-service-model-release-evidence-provenance.test.mjs;tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs",".github/workflows/pester-evidence.yml;.github/workflows/pester-service-model-release-evidence.yml;tools/Invoke-PesterEvidenceProvenance.ps1;tools/Replay-PesterServiceModelArtifacts.Local.ps1;tools/priority/materialize-pester-service-model-release-evidence.mjs;tools/priority/render-pester-service-model-promotion-dossier.mjs;tests/results",Implemented +REQ-PSM-021,"Gate failures are surfaced as machine-readable operator outcomes with classification, reasons, and next-action context instead of opaque red-job signaling alone","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-021,"tests/Invoke-PesterOperatorOutcome.Tests.ps1;tests/Write-PesterSummaryToStepSummary.Tests.ps1;tests/Write-PesterSummaryToStepSummary.CompactMode.Tests.ps1;tests/PesterFailurePayloadShape.Tests.ps1;tests/Replay-PesterServiceModelArtifacts.Local.Tests.ps1;tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs",".github/workflows/pester-evidence.yml;tools/Invoke-PesterOperatorOutcome.ps1;scripts/Write-PesterSummaryToStepSummary.ps1;tools/Write-PesterTopFailures.ps1;tools/Replay-PesterServiceModelArtifacts.Local.ps1",Implemented +REQ-PSM-022,"Local assurance CI synthesizes standards audit results, RTM gaps, and code refs into a ranked backlog and selected next requirement for local-first development","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-022,"tools/priority/__tests__/pester-service-model-local-ci.test.mjs;tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs","tools/priority/pester-service-model-local-ci.mjs;tools/priority/pester-service-model-audit-surface.yaml;package.json",Implemented +REQ-PSM-023,"The local autonomy loop applies explicit policy, active-worktree signals, and stop conditions so an agent knows when to stay local and when to escalate","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-023,"tools/priority/__tests__/pester-service-model-local-ci.test.mjs;tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs","tools/priority/pester-service-model-local-ci.mjs;tools/priority/pester-service-model-autonomy-policy.json",Implemented +REQ-PSM-024,"Representative retained-artifact replay normalizes backward-compatible legacy runs without hiding the underlying evidence classification","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-024,"tests/Replay-PesterServiceModelRepresentativeArtifact.Tests.ps1;tests/Invoke-PesterExecutionPostprocess.Tests.ps1;tests/Invoke-PesterEvidenceClassification.Tests.ps1","tools/Replay-PesterServiceModelArtifacts.Local.ps1;tools/Invoke-PesterExecutionPostprocess.ps1;tools/Invoke-PesterEvidenceClassification.ps1;tools/Invoke-PesterOperatorOutcome.ps1;tests/fixtures/pester-service-model/legacy-results-xml-truncated",Implemented +REQ-PSM-025,"A local Windows-container surrogate proof surface emits an explicit receipt for Docker Desktop Windows engine plus the pinned NI Windows image, including reachable Windows host bridge use from Unix or WSL coordinators","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-025,"tests/Invoke-PesterWindowsContainerSurfaceProbe.Tests.ps1;tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs","tools/Invoke-PesterWindowsContainerSurfaceProbe.ps1;tools/priority/windows-host-bridge.mjs;package.json;tools/Test-WindowsNI2026q1HostPreflight.ps1;tools/Run-NIWindowsContainerCompare.ps1",Implemented +REQ-PSM-026,"The local autonomy loop consumes proof checks and reopens implemented requirements when representative local proof regresses","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-026,"tools/priority/__tests__/pester-service-model-local-ci.test.mjs;tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs","tools/priority/pester-service-model-local-ci.mjs;docs/schemas/pester-service-model-local-ci-report-v1.schema.json;tools/priority/pester-service-model-autonomy-policy.json",Implemented +REQ-PSM-027,"When the next truthful proof surface is unavailable from the current host, local assurance CI emits a machine-readable escalation step instead of a human-only advisory","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-027,"tools/priority/__tests__/pester-service-model-local-ci.test.mjs;tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs","tools/priority/pester-service-model-local-ci.mjs;docs/schemas/pester-service-model-local-ci-report-v1.schema.json;docs/schemas/pester-service-model-next-step-v1.schema.json;package.json",Implemented +REQ-PSM-028,"The Pester packet participates in the shared local proof program selector so requirement choice and shared-surface escalations are reconciled once for the whole local loop","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-028,"tools/priority/__tests__/comparevi-local-program-ci.test.mjs;tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs","tools/priority/comparevi-local-program-ci.mjs;tools/priority/pester-service-model-local-ci.mjs;tools/priority/vi-history-local-ci.mjs;docs/schemas/comparevi-local-program-ci-report-v1.schema.json;docs/schemas/comparevi-local-program-next-step-v1.schema.json;package.json",Implemented diff --git a/docs/rtm-vi-history-local-proof.csv b/docs/rtm-vi-history-local-proof.csv new file mode 100644 index 000000000..c8b571c53 --- /dev/null +++ b/docs/rtm-vi-history-local-proof.csv @@ -0,0 +1,10 @@ +ReqID,Requirement,Source,Priority,TestID,TestArtifact,CodeRef,Status +REQ-VHLP-001,"Stable local Windows workflow replay exists for vi-history-scenarios-windows, emits a workflow-grade receipt plus compare artifacts, and can be launched through a reachable Windows host bridge","docs/requirements-vi-history-local-proof-srs.md",High,TEST-VHLP-001,"tools/priority/__tests__/windows-workflow-replay-lane.test.mjs;tools/priority/__tests__/vi-history-local-proof-contract.test.mjs","package.json;tools/priority/windows-workflow-replay-lane.mjs;tools/priority/windows-host-bridge.mjs;docs/schemas/windows-workflow-replay-lane-v1.schema.json",Implemented +REQ-VHLP-002,"Stable VI History local refinement profiles exist for proof, dev-fast, warm-dev, and windows-mirror-proof and emit canonical receipts and benchmarks","docs/requirements-vi-history-local-proof-srs.md",High,TEST-VHLP-002,"tests/VIHistoryLocalAcceleration.Tests.ps1;tools/priority/__tests__/vi-history-local-proof-contract.test.mjs","package.json;tools/Invoke-VIHistoryLocalRefinement.ps1;tools/docker/Dockerfile.vi-history-dev",Implemented +REQ-VHLP-003,"Stable VI History local operator-session wrappers exist for common refinement profiles","docs/requirements-vi-history-local-proof-srs.md",High,TEST-VHLP-003,"tests/VIHistoryLocalOperatorSession.Tests.ps1;tools/priority/__tests__/vi-history-local-proof-contract.test.mjs","package.json;tools/Invoke-VIHistoryLocalOperatorSession.ps1",Implemented +REQ-VHLP-004,"VI History lane state is normalized into a workflow-readiness envelope with lifecycle, verdict, and recommendation","docs/requirements-vi-history-local-proof-srs.md",High,TEST-VHLP-004,"tests/Write-VIHistoryWorkflowReadiness.Tests.ps1;tools/priority/__tests__/vi-history-local-proof-contract.test.mjs","tools/Write-VIHistoryWorkflowReadiness.ps1",Implemented +REQ-VHLP-005,"Local assurance CI synthesizes the VI History local-proof packet into a report, ranked backlog, and next step","docs/requirements-vi-history-local-proof-srs.md",High,TEST-VHLP-005,"tools/priority/__tests__/vi-history-local-ci.test.mjs;tools/priority/__tests__/vi-history-local-proof-contract.test.mjs","tools/priority/vi-history-local-ci.mjs;tools/priority/vi-history-local-proof-audit-surface.yaml;tools/priority/vi-history-local-proof-autonomy-policy.json;docs/schemas/vi-history-local-ci-report-v1.schema.json",Implemented +REQ-VHLP-006,"When the next truthful VI History proof surface is unavailable from the current host, local assurance CI emits a machine-readable escalation to the shared windows-docker-desktop-ni-image surface after native or bridge-backed Windows checks","docs/requirements-vi-history-local-proof-srs.md",High,TEST-VHLP-006,"tools/priority/__tests__/vi-history-local-ci.test.mjs;tools/priority/__tests__/vi-history-local-proof-contract.test.mjs","tools/priority/vi-history-local-ci.mjs;tools/priority/windows-host-bridge.mjs;docs/schemas/vi-history-local-ci-report-v1.schema.json;docs/schemas/vi-history-local-next-step-v1.schema.json;package.json",Implemented +REQ-VHLP-007,"The VI History packet participates in the shared local proof program selector so it can be chosen explicitly against sibling packets and share merged shared-surface handoffs","docs/requirements-vi-history-local-proof-srs.md",High,TEST-VHLP-007,"tools/priority/__tests__/comparevi-local-program-ci.test.mjs;tools/priority/__tests__/vi-history-local-proof-contract.test.mjs","tools/priority/comparevi-local-program-ci.mjs;tools/priority/vi-history-local-ci.mjs;tools/priority/pester-service-model-local-ci.mjs;docs/schemas/comparevi-local-program-ci-report-v1.schema.json;docs/schemas/comparevi-local-program-next-step-v1.schema.json;package.json",Implemented +REQ-VHLP-008,"The VI History packet governs a clone-backed live-history iteration candidate, initially ni/labview-icon-editor plus Tooling/deployment/VIP_Pre-Uninstall Custom Action.vi","docs/requirements-vi-history-local-proof-srs.md",High,TEST-VHLP-008,"tools/priority/__tests__/vi-history-local-proof-contract.test.mjs","tools/priority/vi-history-live-candidate.json;docs/schemas/vi-history-live-candidate-v1.schema.json;docs/knowledgebase/VI-History-Local-Proof.md",Implemented +REQ-VHLP-009,"Local assurance CI validates clone presence, target path presence, and git history for the governed live-history candidate before Windows replay or hosted proof is chosen","docs/requirements-vi-history-local-proof-srs.md",High,TEST-VHLP-009,"tools/priority/__tests__/vi-history-local-ci.test.mjs;tools/priority/__tests__/vi-history-local-proof-contract.test.mjs","tools/priority/vi-history-local-ci.mjs;tools/priority/vi-history-live-candidate.json;docs/schemas/vi-history-live-candidate-v1.schema.json;docs/schemas/vi-history-live-candidate-readiness-v1.schema.json",Implemented diff --git a/docs/rtm-windows-docker-shared-surface.csv b/docs/rtm-windows-docker-shared-surface.csv new file mode 100644 index 000000000..a8e6bcca9 --- /dev/null +++ b/docs/rtm-windows-docker-shared-surface.csv @@ -0,0 +1,9 @@ +ReqID,Requirement,Source,Priority,TestID,TestArtifact,CodeRef,Status +REQ-WDSS-001,"Stable shared-surface readiness probe exists for Docker Desktop Windows engine plus the pinned NI Windows image and emits explicit readiness states","docs/requirements-windows-docker-shared-surface-srs.md",High,TEST-WDSS-001,"tests/Invoke-PesterWindowsContainerSurfaceProbe.Tests.ps1;tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs","tools/Invoke-PesterWindowsContainerSurfaceProbe.ps1;package.json",Implemented +REQ-WDSS-002,"Deterministic Windows host bootstrap and preflight contract exists for the pinned NI image surface","docs/requirements-windows-docker-shared-surface-srs.md",High,TEST-WDSS-002,"tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs","tools/Test-WindowsNI2026q1HostPreflight.ps1;tools/Run-NIWindowsContainerCompare.ps1;package.json",Implemented +REQ-WDSS-003,"The shared Windows proof packet detects unsafe synchronized or externally managed roots, such as OneDrive-managed paths, before recommending live Windows proof","docs/requirements-windows-docker-shared-surface-srs.md",High,TEST-WDSS-003,"tools/priority/__tests__/windows-docker-shared-surface-local-ci.test.mjs;tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs","tools/priority/windows-docker-shared-surface-local-ci.mjs;docs/schemas/windows-docker-shared-surface-local-ci-report-v1.schema.json;docs/schemas/windows-docker-shared-surface-next-step-v1.schema.json",Implemented +REQ-WDSS-004,"Local assurance CI synthesizes the shared Windows surface packet into a machine-readable report, ranked backlog, proof checks, and next step","docs/requirements-windows-docker-shared-surface-srs.md",High,TEST-WDSS-004,"tools/priority/__tests__/windows-docker-shared-surface-local-ci.test.mjs;tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs","tools/priority/windows-docker-shared-surface-local-ci.mjs;tools/priority/windows-docker-shared-surface-audit-surface.yaml;tools/priority/windows-docker-shared-surface-autonomy-policy.json",Implemented +REQ-WDSS-005,"When the next truthful shared-surface proof is unavailable from the current host, local assurance CI emits a machine-readable escalation to windows-docker-desktop-ni-image","docs/requirements-windows-docker-shared-surface-srs.md",High,TEST-WDSS-005,"tools/priority/__tests__/windows-docker-shared-surface-local-ci.test.mjs;tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs","tools/priority/windows-docker-shared-surface-local-ci.mjs;docs/schemas/windows-docker-shared-surface-next-step-v1.schema.json;package.json",Implemented +REQ-WDSS-006,"The shared Windows surface packet participates in the shared local proof program selector beside Pester and VI History","docs/requirements-windows-docker-shared-surface-srs.md",High,TEST-WDSS-006,"tools/priority/__tests__/comparevi-local-program-ci.test.mjs;tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs","tools/priority/comparevi-local-program-ci.mjs;tools/priority/windows-docker-shared-surface-local-ci.mjs;docs/knowledgebase/Local-Proof-Autonomy-Program.md;package.json",Implemented +REQ-WDSS-007,"When a reachable Windows host exists behind a Unix or WSL coordinator, the shared Windows surface packet uses a governed bridge before emitting host-unavailable escalation","docs/requirements-windows-docker-shared-surface-srs.md",High,TEST-WDSS-007,"tools/priority/__tests__/windows-host-bridge.test.mjs;tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs","tools/priority/windows-host-bridge.mjs;tools/priority/windows-docker-shared-surface-local-ci.mjs;tools/Test-WindowsNI2026q1HostPreflight.ps1;tools/Run-NIWindowsContainerCompare.ps1",Implemented +REQ-WDSS-008,"When the coordinator is running from a UNC-backed WSL or other non-bindable Windows path, the shared Windows Docker surface stages container-bound inputs and output targets into a governed Windows-local mount root and synchronizes artifacts back","docs/requirements-windows-docker-shared-surface-srs.md",High,TEST-WDSS-008,"tests/Run-NIWindowsContainerCompare.Tests.ps1;tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs;tools/priority/__tests__/vi-history-local-proof-contract.test.mjs","tools/Run-NIWindowsContainerCompare.ps1;tools/priority/windows-workflow-replay-lane.mjs;docs/knowledgebase/Windows-Docker-Shared-Surface.md",Implemented diff --git a/docs/schemas/comparevi-local-program-ci-report-v1.schema.json b/docs/schemas/comparevi-local-program-ci-report-v1.schema.json new file mode 100644 index 000000000..9aba9f2ab --- /dev/null +++ b/docs/schemas/comparevi-local-program-ci-report-v1.schema.json @@ -0,0 +1,218 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://comparevi.dev/schemas/comparevi-local-program-ci-report-v1.schema.json", + "title": "CompareVI Local Program CI Report", + "type": "object", + "required": [ + "schema_version", + "generated_at", + "repo_root", + "packets", + "ranked_requirements", + "escalations", + "overall" + ], + "properties": { + "schema_version": { "type": "string" }, + "generated_at": { "type": "string", "format": "date-time" }, + "repo_root": { "type": "string" }, + "packets": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "label", + "report_path", + "next_step_path", + "overall_status", + "overall_reason" + ], + "properties": { + "id": { "type": "string" }, + "label": { "type": "string" }, + "report_path": { "type": "string" }, + "next_step_path": { "type": "string" }, + "overall_status": { "type": "string" }, + "overall_reason": { "type": "string" }, + "next_step_type": { + "anyOf": [ + { "type": "string" }, + { "type": "null" } + ] + }, + "next_requirement_id": { + "anyOf": [ + { "type": "string" }, + { "type": "null" } + ] + }, + "required_surface": { + "anyOf": [ + { "type": "string" }, + { "type": "null" } + ] + } + }, + "additionalProperties": true + } + }, + "ranked_requirements": { + "type": "array", + "items": { + "$ref": "#/$defs/requirement" + } + }, + "escalations": { + "type": "array", + "items": { + "$ref": "#/$defs/escalation" + } + }, + "next_step": { + "anyOf": [ + { "$ref": "#/$defs/requirement" }, + { "$ref": "#/$defs/escalation" }, + { "type": "null" } + ] + }, + "overall": { + "type": "object", + "required": [ + "status", + "reason" + ], + "properties": { + "status": { "type": "string" }, + "reason": { "type": "string" } + }, + "additionalProperties": true + } + }, + "$defs": { + "requirement": { + "type": "object", + "required": [ + "type", + "packet_id", + "packet_label", + "source_report_path", + "source_next_step_path", + "req_id", + "priority", + "status", + "phase", + "score", + "why_now", + "requirement", + "test_id", + "code_refs", + "suggested_loop" + ], + "properties": { + "type": { "const": "requirement" }, + "packet_id": { "type": "string" }, + "packet_label": { "type": "string" }, + "source_report_path": { "type": "string" }, + "source_next_step_path": { "type": "string" }, + "req_id": { "type": "string" }, + "priority": { "type": "string" }, + "status": { "type": "string" }, + "phase": { "type": "string" }, + "score": { "type": "number" }, + "program_score": { "type": "number" }, + "why_now": { "type": "string" }, + "requirement": { "type": "string" }, + "test_id": { "type": "string" }, + "test_artifact": { "type": "string" }, + "code_refs": { + "type": "array", + "items": { "type": "string" } + }, + "suggested_loop": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": true + }, + "escalation": { + "type": "object", + "required": [ + "type", + "escalation_id", + "status", + "mode", + "why_now", + "reason", + "required_surface", + "current_surface_status", + "current_host_platform", + "packet_ids", + "packet_labels", + "governing_requirements", + "blocked_requirements", + "proof_check_ids", + "receipt_paths", + "source_next_step_paths", + "suggested_loop", + "recommended_commands", + "stop_conditions" + ], + "properties": { + "type": { "const": "escalation" }, + "escalation_id": { "type": "string" }, + "status": { "type": "string" }, + "mode": { "type": "string" }, + "why_now": { "type": "string" }, + "reason": { "type": "string" }, + "required_surface": { "type": "string" }, + "current_surface_status": { "type": "string" }, + "current_host_platform": { "type": "string" }, + "packet_count": { "type": "number" }, + "packet_ids": { + "type": "array", + "items": { "type": "string" } + }, + "packet_labels": { + "type": "array", + "items": { "type": "string" } + }, + "governing_requirements": { + "type": "array", + "items": { "type": "string" } + }, + "blocked_requirements": { + "type": "array", + "items": { "type": "string" } + }, + "proof_check_ids": { + "type": "array", + "items": { "type": "string" } + }, + "receipt_paths": { + "type": "array", + "items": { "type": "string" } + }, + "source_next_step_paths": { + "type": "array", + "items": { "type": "string" } + }, + "suggested_loop": { + "type": "array", + "items": { "type": "string" } + }, + "recommended_commands": { + "type": "array", + "items": { "type": "string" } + }, + "stop_conditions": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true +} diff --git a/docs/schemas/comparevi-local-program-next-step-v1.schema.json b/docs/schemas/comparevi-local-program-next-step-v1.schema.json new file mode 100644 index 000000000..999eb2ce6 --- /dev/null +++ b/docs/schemas/comparevi-local-program-next-step-v1.schema.json @@ -0,0 +1,141 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://comparevi.dev/schemas/comparevi-local-program-next-step-v1.schema.json", + "title": "CompareVI Local Program Next Step", + "anyOf": [ + { + "$ref": "#/$defs/requirement" + }, + { + "$ref": "#/$defs/escalation" + }, + { + "type": "null" + } + ], + "$defs": { + "requirement": { + "type": "object", + "required": [ + "type", + "packet_id", + "packet_label", + "source_report_path", + "source_next_step_path", + "req_id", + "priority", + "status", + "phase", + "score", + "why_now", + "requirement", + "test_id", + "code_refs", + "suggested_loop" + ], + "properties": { + "type": { "const": "requirement" }, + "packet_id": { "type": "string" }, + "packet_label": { "type": "string" }, + "source_report_path": { "type": "string" }, + "source_next_step_path": { "type": "string" }, + "req_id": { "type": "string" }, + "priority": { "type": "string" }, + "status": { "type": "string" }, + "phase": { "type": "string" }, + "score": { "type": "number" }, + "program_score": { "type": "number" }, + "why_now": { "type": "string" }, + "requirement": { "type": "string" }, + "test_id": { "type": "string" }, + "test_artifact": { "type": "string" }, + "code_refs": { + "type": "array", + "items": { "type": "string" } + }, + "suggested_loop": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": true + }, + "escalation": { + "type": "object", + "required": [ + "type", + "escalation_id", + "status", + "mode", + "why_now", + "reason", + "required_surface", + "current_surface_status", + "current_host_platform", + "packet_ids", + "packet_labels", + "governing_requirements", + "blocked_requirements", + "proof_check_ids", + "receipt_paths", + "source_next_step_paths", + "suggested_loop", + "recommended_commands", + "stop_conditions" + ], + "properties": { + "type": { "const": "escalation" }, + "escalation_id": { "type": "string" }, + "status": { "type": "string" }, + "mode": { "type": "string" }, + "why_now": { "type": "string" }, + "reason": { "type": "string" }, + "required_surface": { "type": "string" }, + "current_surface_status": { "type": "string" }, + "current_host_platform": { "type": "string" }, + "packet_count": { "type": "number" }, + "packet_ids": { + "type": "array", + "items": { "type": "string" } + }, + "packet_labels": { + "type": "array", + "items": { "type": "string" } + }, + "governing_requirements": { + "type": "array", + "items": { "type": "string" } + }, + "blocked_requirements": { + "type": "array", + "items": { "type": "string" } + }, + "proof_check_ids": { + "type": "array", + "items": { "type": "string" } + }, + "receipt_paths": { + "type": "array", + "items": { "type": "string" } + }, + "source_next_step_paths": { + "type": "array", + "items": { "type": "string" } + }, + "suggested_loop": { + "type": "array", + "items": { "type": "string" } + }, + "recommended_commands": { + "type": "array", + "items": { "type": "string" } + }, + "stop_conditions": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": true + } + } +} diff --git a/docs/schemas/pester-derived-provenance-v1.schema.json b/docs/schemas/pester-derived-provenance-v1.schema.json new file mode 100644 index 000000000..0a0476d0d --- /dev/null +++ b/docs/schemas/pester-derived-provenance-v1.schema.json @@ -0,0 +1,122 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "pester-derived-provenance@v1", + "type": "object", + "required": [ + "schema", + "schemaVersion", + "generatedAtUtc", + "provenanceKind", + "producer", + "subject", + "runContext", + "sourceInputs", + "derivedOutputs" + ], + "properties": { + "schema": { "const": "pester-derived-provenance@v1" }, + "schemaVersion": { "type": "string" }, + "generatedAtUtc": { "type": "string" }, + "provenanceKind": { + "enum": ["evidence", "local-replay", "release-evidence", "promotion-dossier"] + }, + "producer": { + "type": "object", + "required": ["id", "version"], + "properties": { + "id": { "type": "string" }, + "version": { "type": "string" } + }, + "additionalProperties": true + }, + "subject": { + "type": "object", + "required": ["id"], + "properties": { + "id": { "type": "string" } + }, + "additionalProperties": true + }, + "runContext": { + "type": "object", + "required": ["source", "repository", "workflow", "eventName", "headSha"], + "properties": { + "source": { "type": "string" }, + "repository": { "type": "string" }, + "workflow": { "type": "string" }, + "eventName": { "type": "string" }, + "runId": { "type": ["string", "null"] }, + "runAttempt": { "type": ["string", "null"] }, + "runUrl": { "type": ["string", "null"] }, + "ref": { "type": ["string", "null"] }, + "refName": { "type": ["string", "null"] }, + "branch": { "type": ["string", "null"] }, + "headRef": { "type": ["string", "null"] }, + "baseRef": { "type": ["string", "null"] }, + "headSha": { "type": ["string", "null"] } + }, + "additionalProperties": true + }, + "sourceInputs": { + "type": "array", + "items": { + "type": "object", + "required": ["kind", "role", "present"], + "properties": { + "kind": { "type": "string" }, + "role": { "type": "string" }, + "artifactName": { "type": ["string", "null"] }, + "path": { "type": ["string", "null"] }, + "repoRelativePath": { "type": ["string", "null"] }, + "present": { "type": "boolean" }, + "sizeBytes": { "type": "number" }, + "sha256": { "type": "string" }, + "lastWriteTimeUtc": { "type": "string" }, + "schema": { "type": ["string", "null"] }, + "schemaVersion": { "type": ["string", "null"] }, + "status": { "type": ["string", "null"] }, + "fileCount": { "type": "number" }, + "files": { + "type": "array", + "items": { + "type": "object", + "required": ["path", "relativePath", "sizeBytes", "sha256"], + "properties": { + "path": { "type": "string" }, + "repoRelativePath": { "type": ["string", "null"] }, + "relativePath": { "type": "string" }, + "sizeBytes": { "type": "number" }, + "sha256": { "type": "string" } + }, + "additionalProperties": true + } + } + }, + "additionalProperties": true + } + }, + "derivedOutputs": { + "type": "array", + "items": { + "type": "object", + "required": ["kind", "role", "present"], + "properties": { + "kind": { "type": "string" }, + "role": { "type": "string" }, + "artifactName": { "type": ["string", "null"] }, + "path": { "type": ["string", "null"] }, + "repoRelativePath": { "type": ["string", "null"] }, + "present": { "type": "boolean" }, + "sizeBytes": { "type": "number" }, + "sha256": { "type": "string" }, + "lastWriteTimeUtc": { "type": "string" }, + "schema": { "type": ["string", "null"] }, + "schemaVersion": { "type": ["string", "null"] }, + "status": { "type": ["string", "null"] } + }, + "additionalProperties": true + } + } + }, + "additionalProperties": true +} diff --git a/docs/schemas/pester-promotion-comparison-v1.schema.json b/docs/schemas/pester-promotion-comparison-v1.schema.json new file mode 100644 index 000000000..84a49e647 --- /dev/null +++ b/docs/schemas/pester-promotion-comparison-v1.schema.json @@ -0,0 +1,86 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "pester-promotion-comparison@v1", + "type": "object", + "required": [ + "schema", + "schemaVersion", + "generatedAtUtc", + "baselineLabel", + "candidateLabel", + "decisionState", + "comparisons" + ], + "properties": { + "schema": { "const": "pester-promotion-comparison@v1" }, + "schemaVersion": { "type": "string" }, + "generatedAtUtc": { "type": "string" }, + "baselineLabel": { "type": "string" }, + "candidateLabel": { "type": "string" }, + "decisionState": { "type": "string" }, + "summary": { "type": "string" }, + "comparisons": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": [ + "comparisonId", + "packId", + "representativeness", + "requirementCoverage", + "baseline", + "candidate", + "observedDeltas", + "decision", + "nextAction" + ], + "properties": { + "comparisonId": { "type": "string" }, + "packId": { "type": "string" }, + "representativeness": { "type": "string" }, + "requirementCoverage": { + "type": "array", + "items": { "type": "string" } + }, + "baseline": { + "type": "object", + "required": ["workflow", "runId", "runUrl", "headSha", "ref", "conclusion", "packIdentity"], + "properties": { + "workflow": { "type": "string" }, + "runId": { "type": "string" }, + "runUrl": { "type": "string" }, + "headSha": { "type": "string" }, + "ref": { "type": "string" }, + "conclusion": { "type": "string" }, + "packIdentity": { "type": "string" } + }, + "additionalProperties": true + }, + "candidate": { + "type": "object", + "required": ["workflow", "runId", "runUrl", "headSha", "ref", "conclusion", "packIdentity"], + "properties": { + "workflow": { "type": "string" }, + "runId": { "type": "string" }, + "runUrl": { "type": "string" }, + "headSha": { "type": "string" }, + "ref": { "type": "string" }, + "conclusion": { "type": "string" }, + "packIdentity": { "type": "string" } + }, + "additionalProperties": true + }, + "observedDeltas": { + "type": "array", + "items": { "type": "string" } + }, + "decision": { "type": "string" }, + "nextAction": { "type": "string" } + }, + "additionalProperties": true + } + } + }, + "additionalProperties": true +} diff --git a/docs/schemas/pester-service-model-local-ci-report-v1.schema.json b/docs/schemas/pester-service-model-local-ci-report-v1.schema.json new file mode 100644 index 000000000..28ffa1969 --- /dev/null +++ b/docs/schemas/pester-service-model-local-ci-report-v1.schema.json @@ -0,0 +1,297 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://comparevi.dev/schemas/pester-service-model-local-ci-report-v1.schema.json", + "title": "Pester Service Model Local CI Report", + "type": "object", + "required": [ + "schema_version", + "generated_at", + "repo_root", + "audit_surface", + "standards_audit", + "requirement_summary", + "proof_checks", + "ranked_requirements", + "escalations", + "next_requirement", + "next_step", + "overall" + ], + "properties": { + "schema_version": { + "type": "string", + "const": "1.2.0" + }, + "generated_at": { + "type": "string", + "format": "date-time" + }, + "repo_root": { + "type": "string" + }, + "audit_surface": { + "type": "object", + "required": [ + "id", + "description", + "manifest_path", + "bundle_root", + "included_paths" + ], + "properties": { + "id": { "type": "string" }, + "description": { "type": "string" }, + "manifest_path": { "type": "string" }, + "bundle_root": { "type": "string" }, + "included_paths": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": true + }, + "worktree_status": { + "type": "object", + "required": [ + "branch", + "modified_paths", + "active_requirement_refs" + ], + "properties": { + "branch": { "type": "string" }, + "modified_paths": { + "type": "array", + "items": { "type": "string" } + }, + "active_requirement_refs": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": true + }, + "standards_audit": { + "type": "object", + "required": [ + "status", + "evidence_path", + "score_path" + ], + "properties": { + "status": { "type": "string" }, + "evidence_path": { "type": "string" }, + "score_path": { "type": "string" }, + "weak_areas": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": true + }, + "requirement_summary": { + "type": "object", + "required": [ + "total", + "implemented", + "gaps" + ], + "properties": { + "total": { "type": "integer", "minimum": 0 }, + "implemented": { "type": "integer", "minimum": 0 }, + "gaps": { "type": "integer", "minimum": 0 } + }, + "additionalProperties": true + }, + "proof_checks": { + "type": "object", + "required": [ + "blocking_failures", + "advisories", + "checks" + ], + "properties": { + "blocking_failures": { "type": "integer", "minimum": 0 }, + "advisories": { "type": "integer", "minimum": 0 }, + "checks": { + "type": "array", + "items": { + "$ref": "#/$defs/proofCheck" + } + } + }, + "additionalProperties": true + }, + "ranked_requirements": { + "type": "array", + "items": { + "$ref": "#/$defs/rankedRequirement" + } + }, + "escalations": { + "type": "array", + "items": { + "$ref": "#/$defs/nextStepEscalation" + } + }, + "next_requirement": { + "anyOf": [ + { "$ref": "#/$defs/rankedRequirement" }, + { "type": "null" } + ] + }, + "next_step": { + "anyOf": [ + { "$ref": "#/$defs/nextStepRequirement" }, + { "$ref": "#/$defs/nextStepEscalation" }, + { "type": "null" } + ] + }, + "overall": { + "type": "object", + "required": [ + "status", + "reason" + ], + "properties": { + "status": { "type": "string" }, + "reason": { "type": "string" } + }, + "additionalProperties": true + } + }, + "$defs": { + "rankedRequirement": { + "type": "object", + "required": [ + "req_id", + "priority", + "status", + "phase", + "score", + "why_now", + "requirement", + "test_id", + "code_refs", + "suggested_loop" + ], + "properties": { + "req_id": { "type": "string" }, + "priority": { "type": "string" }, + "status": { "type": "string" }, + "phase": { "type": "string" }, + "score": { "type": "number" }, + "why_now": { "type": "string" }, + "requirement": { "type": "string" }, + "test_id": { "type": "string" }, + "code_refs": { + "type": "array", + "items": { "type": "string" } + }, + "suggested_loop": { + "type": "array", + "items": { "type": "string" } + }, + "test_artifact": { "type": "string" }, + "active_now": { "type": "boolean" }, + "mode": { "type": "string" }, + "preferred_commands": { + "type": "array", + "items": { "type": "string" } + }, + "stop_conditions": { + "type": "array", + "items": { "type": "string" } + }, + "escalate_when": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": true + }, + "nextStepRequirement": { + "allOf": [ + { "$ref": "#/$defs/rankedRequirement" }, + { + "type": "object", + "required": [ "type" ], + "properties": { + "type": { "const": "requirement" } + } + } + ] + }, + "nextStepEscalation": { + "type": "object", + "required": [ + "type", + "escalation_id", + "governing_requirement", + "blocked_requirement", + "proof_check_id", + "status", + "mode", + "why_now", + "reason", + "required_surface", + "current_surface_status", + "current_host_platform", + "suggested_loop", + "recommended_commands", + "stop_conditions" + ], + "properties": { + "type": { "const": "escalation" }, + "escalation_id": { "type": "string" }, + "governing_requirement": { "type": "string" }, + "blocked_requirement": { "type": "string" }, + "proof_check_id": { "type": "string" }, + "status": { "type": "string" }, + "mode": { "type": "string" }, + "why_now": { "type": "string" }, + "reason": { "type": "string" }, + "required_surface": { "type": "string" }, + "current_surface_status": { "type": "string" }, + "current_host_platform": { "type": "string" }, + "receipt_path": { + "anyOf": [ + { "type": "string" }, + { "type": "null" } + ] + }, + "suggested_loop": { + "type": "array", + "items": { "type": "string" } + }, + "recommended_commands": { + "type": "array", + "items": { "type": "string" } + }, + "stop_conditions": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": true + }, + "proofCheck": { + "type": "object", + "required": [ + "id", + "owner_requirement", + "status", + "blocking", + "summary" + ], + "properties": { + "id": { "type": "string" }, + "owner_requirement": { "type": "string" }, + "status": { "type": "string" }, + "blocking": { "type": "boolean" }, + "summary": { "type": "string" } + }, + "additionalProperties": true + } + }, + "additionalProperties": true +} diff --git a/docs/schemas/pester-service-model-next-step-v1.schema.json b/docs/schemas/pester-service-model-next-step-v1.schema.json new file mode 100644 index 000000000..da93e23d9 --- /dev/null +++ b/docs/schemas/pester-service-model-next-step-v1.schema.json @@ -0,0 +1,132 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://comparevi.dev/schemas/pester-service-model-next-step-v1.schema.json", + "title": "Pester Service Model Next Step", + "anyOf": [ + { + "$ref": "#/$defs/nextStepRequirement" + }, + { + "$ref": "#/$defs/nextStepEscalation" + }, + { + "type": "null" + } + ], + "$defs": { + "rankedRequirement": { + "type": "object", + "required": [ + "req_id", + "priority", + "status", + "phase", + "score", + "why_now", + "requirement", + "test_id", + "code_refs", + "suggested_loop" + ], + "properties": { + "req_id": { "type": "string" }, + "priority": { "type": "string" }, + "status": { "type": "string" }, + "phase": { "type": "string" }, + "score": { "type": "number" }, + "why_now": { "type": "string" }, + "requirement": { "type": "string" }, + "test_id": { "type": "string" }, + "code_refs": { + "type": "array", + "items": { "type": "string" } + }, + "suggested_loop": { + "type": "array", + "items": { "type": "string" } + }, + "test_artifact": { "type": "string" }, + "active_now": { "type": "boolean" }, + "mode": { "type": "string" }, + "preferred_commands": { + "type": "array", + "items": { "type": "string" } + }, + "stop_conditions": { + "type": "array", + "items": { "type": "string" } + }, + "escalate_when": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": true + }, + "nextStepRequirement": { + "allOf": [ + { "$ref": "#/$defs/rankedRequirement" }, + { + "type": "object", + "required": [ "type" ], + "properties": { + "type": { "const": "requirement" } + } + } + ] + }, + "nextStepEscalation": { + "type": "object", + "required": [ + "type", + "escalation_id", + "governing_requirement", + "blocked_requirement", + "proof_check_id", + "status", + "mode", + "why_now", + "reason", + "required_surface", + "current_surface_status", + "current_host_platform", + "suggested_loop", + "recommended_commands", + "stop_conditions" + ], + "properties": { + "type": { "const": "escalation" }, + "escalation_id": { "type": "string" }, + "governing_requirement": { "type": "string" }, + "blocked_requirement": { "type": "string" }, + "proof_check_id": { "type": "string" }, + "status": { "type": "string" }, + "mode": { "type": "string" }, + "why_now": { "type": "string" }, + "reason": { "type": "string" }, + "required_surface": { "type": "string" }, + "current_surface_status": { "type": "string" }, + "current_host_platform": { "type": "string" }, + "receipt_path": { + "anyOf": [ + { "type": "string" }, + { "type": "null" } + ] + }, + "suggested_loop": { + "type": "array", + "items": { "type": "string" } + }, + "recommended_commands": { + "type": "array", + "items": { "type": "string" } + }, + "stop_conditions": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": true + } + } +} diff --git a/docs/schemas/vi-history-live-candidate-readiness-v1.schema.json b/docs/schemas/vi-history-live-candidate-readiness-v1.schema.json new file mode 100644 index 000000000..14a135cd6 --- /dev/null +++ b/docs/schemas/vi-history-live-candidate-readiness-v1.schema.json @@ -0,0 +1,96 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://comparevi.dev/schemas/vi-history-live-candidate-readiness-v1.schema.json", + "title": "VI History Live Candidate Readiness", + "type": "object", + "required": [ + "schema", + "generatedAt", + "status", + "candidateId", + "repoSlug", + "repoUrl", + "defaultBranch", + "targetViPath", + "cloneRootCandidates", + "recommendedCommands" + ], + "properties": { + "schema": { + "const": "vi-history/live-candidate-readiness@v1" + }, + "generatedAt": { + "type": "string", + "format": "date-time" + }, + "status": { + "enum": [ + "ready", + "missing-clone", + "missing-target", + "missing-history", + "git-failed" + ] + }, + "candidateId": { + "type": "string" + }, + "repoSlug": { + "type": "string" + }, + "repoUrl": { + "type": "string", + "format": "uri" + }, + "defaultBranch": { + "type": "string" + }, + "cloneRootEnvVar": { + "type": "string" + }, + "cloneRootCandidates": { + "type": "array", + "items": { + "type": "string" + } + }, + "resolvedCloneRoot": { + "anyOf": [ + { "type": "string" }, + { "type": "null" } + ] + }, + "targetViPath": { + "type": "string" + }, + "historyExpectation": { + "type": "object" + }, + "history": { + "type": "object", + "properties": { + "commitCount": { + "type": "integer", + "minimum": 0 + }, + "latestCommit": { + "type": [ + "string", + "null" + ] + } + }, + "additionalProperties": true + }, + "reason": { + "type": "string" + }, + "recommendedCommands": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": true +} diff --git a/docs/schemas/vi-history-live-candidate-v1.schema.json b/docs/schemas/vi-history-live-candidate-v1.schema.json new file mode 100644 index 000000000..322bf56ca --- /dev/null +++ b/docs/schemas/vi-history-live-candidate-v1.schema.json @@ -0,0 +1,77 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://comparevi.dev/schemas/vi-history-live-candidate-v1.schema.json", + "title": "VI History Live Candidate", + "type": "object", + "required": [ + "$schema", + "schemaVersion", + "id", + "repoSlug", + "repoUrl", + "defaultBranch", + "cloneRootEnvVar", + "preferredLocalCloneRoots", + "targetViPath", + "historyExpectation", + "iterationRationale" + ], + "properties": { + "$schema": { + "type": "string" + }, + "schemaVersion": { + "type": "string", + "const": "1.0.0" + }, + "id": { + "type": "string" + }, + "repoSlug": { + "type": "string" + }, + "repoUrl": { + "type": "string", + "format": "uri" + }, + "defaultBranch": { + "type": "string" + }, + "cloneRootEnvVar": { + "type": "string" + }, + "preferredLocalCloneRoots": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "targetViPath": { + "type": "string" + }, + "historyExpectation": { + "type": "object", + "required": [ + "minCommits" + ], + "properties": { + "minCommits": { + "type": "integer", + "minimum": 1 + } + }, + "additionalProperties": false + }, + "iterationRationale": { + "type": "string" + }, + "notes": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false +} diff --git a/docs/schemas/vi-history-local-ci-report-v1.schema.json b/docs/schemas/vi-history-local-ci-report-v1.schema.json new file mode 100644 index 000000000..db18fd5da --- /dev/null +++ b/docs/schemas/vi-history-local-ci-report-v1.schema.json @@ -0,0 +1,231 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://comparevi.dev/schemas/vi-history-local-ci-report-v1.schema.json", + "title": "VI History Local CI Report", + "type": "object", + "required": [ + "schema_version", + "generated_at", + "repo_root", + "audit_surface", + "standards_audit", + "requirement_summary", + "proof_checks", + "ranked_requirements", + "escalations", + "next_requirement", + "next_step", + "overall" + ], + "properties": { + "schema_version": { + "type": "string", + "const": "1.0.0" + }, + "generated_at": { + "type": "string", + "format": "date-time" + }, + "repo_root": { "type": "string" }, + "audit_surface": { + "type": "object", + "required": ["id", "description", "manifest_path", "bundle_root", "included_paths"], + "properties": { + "id": { "type": "string" }, + "description": { "type": "string" }, + "manifest_path": { "type": "string" }, + "bundle_root": { "type": "string" }, + "included_paths": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": true + }, + "worktree_status": { + "type": "object", + "required": ["branch", "modified_paths", "active_requirement_refs"], + "properties": { + "branch": { "type": "string" }, + "modified_paths": { + "type": "array", + "items": { "type": "string" } + }, + "active_requirement_refs": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": true + }, + "standards_audit": { + "type": "object", + "required": ["status", "evidence_path", "score_path"], + "properties": { + "status": { "type": "string" }, + "evidence_path": { "type": "string" }, + "score_path": { "type": "string" }, + "weak_areas": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": true + }, + "requirement_summary": { + "type": "object", + "required": ["total", "implemented", "gaps"], + "properties": { + "total": { "type": "integer", "minimum": 0 }, + "implemented": { "type": "integer", "minimum": 0 }, + "gaps": { "type": "integer", "minimum": 0 } + }, + "additionalProperties": true + }, + "proof_checks": { + "type": "object", + "required": ["blocking_failures", "advisories", "checks"], + "properties": { + "blocking_failures": { "type": "integer", "minimum": 0 }, + "advisories": { "type": "integer", "minimum": 0 }, + "checks": { + "type": "array", + "items": { "$ref": "#/$defs/proofCheck" } + } + }, + "additionalProperties": true + }, + "ranked_requirements": { + "type": "array", + "items": { "$ref": "#/$defs/rankedRequirement" } + }, + "escalations": { + "type": "array", + "items": { "$ref": "#/$defs/nextStepEscalation" } + }, + "next_requirement": { + "anyOf": [ + { "$ref": "#/$defs/rankedRequirement" }, + { "type": "null" } + ] + }, + "next_step": { + "anyOf": [ + { "$ref": "#/$defs/nextStepRequirement" }, + { "$ref": "#/$defs/nextStepEscalation" }, + { "type": "null" } + ] + }, + "overall": { + "type": "object", + "required": ["status", "reason"], + "properties": { + "status": { "type": "string" }, + "reason": { "type": "string" } + }, + "additionalProperties": true + } + }, + "$defs": { + "rankedRequirement": { + "type": "object", + "required": [ + "req_id", + "priority", + "status", + "phase", + "score", + "why_now", + "requirement", + "test_id", + "code_refs", + "suggested_loop" + ], + "properties": { + "req_id": { "type": "string" }, + "priority": { "type": "string" }, + "status": { "type": "string" }, + "phase": { "type": "string" }, + "score": { "type": "number" }, + "why_now": { "type": "string" }, + "requirement": { "type": "string" }, + "test_id": { "type": "string" }, + "code_refs": { "type": "array", "items": { "type": "string" } }, + "suggested_loop": { "type": "array", "items": { "type": "string" } }, + "test_artifact": { "type": "string" }, + "active_now": { "type": "boolean" }, + "mode": { "type": "string" }, + "preferred_commands": { "type": "array", "items": { "type": "string" } }, + "stop_conditions": { "type": "array", "items": { "type": "string" } }, + "escalate_when": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": true + }, + "nextStepRequirement": { + "allOf": [ + { "$ref": "#/$defs/rankedRequirement" }, + { + "type": "object", + "required": ["type"], + "properties": { + "type": { "const": "requirement" } + } + } + ] + }, + "nextStepEscalation": { + "type": "object", + "required": [ + "type", + "escalation_id", + "governing_requirement", + "blocked_requirement", + "proof_check_id", + "status", + "mode", + "why_now", + "reason", + "required_surface", + "current_surface_status", + "current_host_platform", + "suggested_loop", + "recommended_commands", + "stop_conditions" + ], + "properties": { + "type": { "const": "escalation" }, + "escalation_id": { "type": "string" }, + "governing_requirement": { "type": "string" }, + "blocked_requirement": { "type": "string" }, + "proof_check_id": { "type": "string" }, + "status": { "type": "string" }, + "mode": { "type": "string" }, + "why_now": { "type": "string" }, + "reason": { "type": "string" }, + "required_surface": { "type": "string" }, + "current_surface_status": { "type": "string" }, + "current_host_platform": { "type": "string" }, + "receipt_path": { + "anyOf": [{ "type": "string" }, { "type": "null" }] + }, + "suggested_loop": { "type": "array", "items": { "type": "string" } }, + "recommended_commands": { "type": "array", "items": { "type": "string" } }, + "stop_conditions": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": true + }, + "proofCheck": { + "type": "object", + "required": ["id", "owner_requirement", "status", "blocking", "summary"], + "properties": { + "id": { "type": "string" }, + "owner_requirement": { "type": "string" }, + "status": { "type": "string" }, + "blocking": { "type": "boolean" }, + "summary": { "type": "string" } + }, + "additionalProperties": true + } + }, + "additionalProperties": true +} diff --git a/docs/schemas/vi-history-local-next-step-v1.schema.json b/docs/schemas/vi-history-local-next-step-v1.schema.json new file mode 100644 index 000000000..260a8616c --- /dev/null +++ b/docs/schemas/vi-history-local-next-step-v1.schema.json @@ -0,0 +1,132 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://comparevi.dev/schemas/vi-history-local-next-step-v1.schema.json", + "title": "VI History Local Next Step", + "anyOf": [ + { + "$ref": "#/$defs/nextStepRequirement" + }, + { + "$ref": "#/$defs/nextStepEscalation" + }, + { + "type": "null" + } + ], + "$defs": { + "rankedRequirement": { + "type": "object", + "required": [ + "req_id", + "priority", + "status", + "phase", + "score", + "why_now", + "requirement", + "test_id", + "code_refs", + "suggested_loop" + ], + "properties": { + "req_id": { "type": "string" }, + "priority": { "type": "string" }, + "status": { "type": "string" }, + "phase": { "type": "string" }, + "score": { "type": "number" }, + "why_now": { "type": "string" }, + "requirement": { "type": "string" }, + "test_id": { "type": "string" }, + "code_refs": { + "type": "array", + "items": { "type": "string" } + }, + "suggested_loop": { + "type": "array", + "items": { "type": "string" } + }, + "test_artifact": { "type": "string" }, + "active_now": { "type": "boolean" }, + "mode": { "type": "string" }, + "preferred_commands": { + "type": "array", + "items": { "type": "string" } + }, + "stop_conditions": { + "type": "array", + "items": { "type": "string" } + }, + "escalate_when": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": true + }, + "nextStepRequirement": { + "allOf": [ + { "$ref": "#/$defs/rankedRequirement" }, + { + "type": "object", + "required": [ "type" ], + "properties": { + "type": { "const": "requirement" } + } + } + ] + }, + "nextStepEscalation": { + "type": "object", + "required": [ + "type", + "escalation_id", + "governing_requirement", + "blocked_requirement", + "proof_check_id", + "status", + "mode", + "why_now", + "reason", + "required_surface", + "current_surface_status", + "current_host_platform", + "suggested_loop", + "recommended_commands", + "stop_conditions" + ], + "properties": { + "type": { "const": "escalation" }, + "escalation_id": { "type": "string" }, + "governing_requirement": { "type": "string" }, + "blocked_requirement": { "type": "string" }, + "proof_check_id": { "type": "string" }, + "status": { "type": "string" }, + "mode": { "type": "string" }, + "why_now": { "type": "string" }, + "reason": { "type": "string" }, + "required_surface": { "type": "string" }, + "current_surface_status": { "type": "string" }, + "current_host_platform": { "type": "string" }, + "receipt_path": { + "anyOf": [ + { "type": "string" }, + { "type": "null" } + ] + }, + "suggested_loop": { + "type": "array", + "items": { "type": "string" } + }, + "recommended_commands": { + "type": "array", + "items": { "type": "string" } + }, + "stop_conditions": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": true + } + } +} diff --git a/docs/schemas/windows-docker-shared-surface-local-ci-report-v1.schema.json b/docs/schemas/windows-docker-shared-surface-local-ci-report-v1.schema.json new file mode 100644 index 000000000..5fa703bb3 --- /dev/null +++ b/docs/schemas/windows-docker-shared-surface-local-ci-report-v1.schema.json @@ -0,0 +1,164 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://comparevi.dev/schemas/windows-docker-shared-surface-local-ci-report-v1.schema.json", + "title": "Windows Docker Shared Surface Local CI Report", + "type": "object", + "required": [ + "schema_version", + "generated_at", + "repo_root", + "audit_surface", + "standards_audit", + "requirement_summary", + "proof_checks", + "ranked_requirements", + "escalations", + "next_requirement", + "next_step", + "overall" + ], + "properties": { + "schema_version": { "const": "1.0.0" }, + "generated_at": { "type": "string", "format": "date-time" }, + "repo_root": { "type": "string" }, + "audit_surface": { + "type": "object", + "required": ["id", "description", "manifest_path", "bundle_root", "included_paths"], + "properties": { + "id": { "type": "string" }, + "description": { "type": "string" }, + "manifest_path": { "type": "string" }, + "bundle_root": { "type": "string" }, + "included_paths": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": true + }, + "worktree_status": { + "type": "object", + "required": ["branch", "modified_paths", "active_requirement_refs"], + "properties": { + "branch": { "type": "string" }, + "modified_paths": { "type": "array", "items": { "type": "string" } }, + "active_requirement_refs": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": true + }, + "standards_audit": { + "type": "object", + "required": ["status", "evidence_path", "score_path"], + "properties": { + "status": { "type": "string" }, + "evidence_path": { "type": "string" }, + "score_path": { "type": "string" }, + "weak_areas": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": true + }, + "requirement_summary": { + "type": "object", + "required": ["total", "implemented", "gaps"], + "properties": { + "total": { "type": "integer", "minimum": 0 }, + "implemented": { "type": "integer", "minimum": 0 }, + "gaps": { "type": "integer", "minimum": 0 } + }, + "additionalProperties": true + }, + "proof_checks": { + "type": "object", + "required": ["blocking_failures", "advisories", "checks"], + "properties": { + "blocking_failures": { "type": "integer", "minimum": 0 }, + "advisories": { "type": "integer", "minimum": 0 }, + "checks": { "type": "array", "items": { "$ref": "#/$defs/proofCheck" } } + }, + "additionalProperties": true + }, + "ranked_requirements": { "type": "array", "items": { "$ref": "#/$defs/rankedRequirement" } }, + "escalations": { "type": "array", "items": { "$ref": "#/$defs/nextStepEscalation" } }, + "next_requirement": { "anyOf": [{ "$ref": "#/$defs/rankedRequirement" }, { "type": "null" }] }, + "next_step": { + "anyOf": [ + { "$ref": "#/$defs/nextStepRequirement" }, + { "$ref": "#/$defs/nextStepEscalation" }, + { "type": "null" } + ] + }, + "overall": { + "type": "object", + "required": ["status", "reason"], + "properties": { + "status": { "type": "string" }, + "reason": { "type": "string" } + }, + "additionalProperties": true + } + }, + "$defs": { + "rankedRequirement": { + "type": "object", + "required": ["req_id", "priority", "status", "phase", "score", "why_now", "requirement", "test_id", "code_refs", "suggested_loop"], + "properties": { + "req_id": { "type": "string" }, + "priority": { "type": "string" }, + "status": { "type": "string" }, + "phase": { "type": "string" }, + "score": { "type": "number" }, + "why_now": { "type": "string" }, + "requirement": { "type": "string" }, + "test_id": { "type": "string" }, + "code_refs": { "type": "array", "items": { "type": "string" } }, + "suggested_loop": { "type": "array", "items": { "type": "string" } }, + "test_artifact": { "type": "string" }, + "active_now": { "type": "boolean" }, + "mode": { "type": "string" }, + "preferred_commands": { "type": "array", "items": { "type": "string" } }, + "stop_conditions": { "type": "array", "items": { "type": "string" } }, + "escalate_when": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": true + }, + "nextStepRequirement": { + "allOf": [ + { "$ref": "#/$defs/rankedRequirement" }, + { "type": "object", "required": ["type"], "properties": { "type": { "const": "requirement" } } } + ] + }, + "nextStepEscalation": { + "type": "object", + "required": ["type", "escalation_id", "governing_requirement", "blocked_requirement", "proof_check_id", "status", "mode", "why_now", "reason", "required_surface", "current_surface_status", "current_host_platform", "suggested_loop", "recommended_commands", "stop_conditions"], + "properties": { + "type": { "const": "escalation" }, + "escalation_id": { "type": "string" }, + "governing_requirement": { "type": "string" }, + "blocked_requirement": { "type": "string" }, + "proof_check_id": { "type": "string" }, + "status": { "type": "string" }, + "mode": { "type": "string" }, + "why_now": { "type": "string" }, + "reason": { "type": "string" }, + "required_surface": { "type": "string" }, + "current_surface_status": { "type": "string" }, + "current_host_platform": { "type": "string" }, + "receipt_path": { "anyOf": [{ "type": "string" }, { "type": "null" }] }, + "suggested_loop": { "type": "array", "items": { "type": "string" } }, + "recommended_commands": { "type": "array", "items": { "type": "string" } }, + "stop_conditions": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": true + }, + "proofCheck": { + "type": "object", + "required": ["id", "owner_requirement", "status", "blocking", "summary"], + "properties": { + "id": { "type": "string" }, + "owner_requirement": { "type": "string" }, + "status": { "type": "string" }, + "blocking": { "type": "boolean" }, + "summary": { "type": "string" } + }, + "additionalProperties": true + } + }, + "additionalProperties": true +} diff --git a/docs/schemas/windows-docker-shared-surface-next-step-v1.schema.json b/docs/schemas/windows-docker-shared-surface-next-step-v1.schema.json new file mode 100644 index 000000000..fd5b4f761 --- /dev/null +++ b/docs/schemas/windows-docker-shared-surface-next-step-v1.schema.json @@ -0,0 +1,58 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://comparevi.dev/schemas/windows-docker-shared-surface-next-step-v1.schema.json", + "title": "Windows Docker Shared Surface Next Step", + "anyOf": [ + { "$ref": "#/$defs/nextStepRequirement" }, + { "$ref": "#/$defs/nextStepEscalation" }, + { "type": "null" } + ], + "$defs": { + "rankedRequirement": { + "type": "object", + "required": ["req_id", "priority", "status", "phase", "score", "why_now", "requirement", "test_id", "code_refs", "suggested_loop"], + "properties": { + "req_id": { "type": "string" }, + "priority": { "type": "string" }, + "status": { "type": "string" }, + "phase": { "type": "string" }, + "score": { "type": "number" }, + "why_now": { "type": "string" }, + "requirement": { "type": "string" }, + "test_id": { "type": "string" }, + "code_refs": { "type": "array", "items": { "type": "string" } }, + "suggested_loop": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": true + }, + "nextStepRequirement": { + "allOf": [ + { "$ref": "#/$defs/rankedRequirement" }, + { "type": "object", "required": ["type"], "properties": { "type": { "const": "requirement" } } } + ] + }, + "nextStepEscalation": { + "type": "object", + "required": ["type", "escalation_id", "governing_requirement", "blocked_requirement", "proof_check_id", "status", "mode", "why_now", "reason", "required_surface", "current_surface_status", "current_host_platform", "suggested_loop", "recommended_commands", "stop_conditions"], + "properties": { + "type": { "const": "escalation" }, + "escalation_id": { "type": "string" }, + "governing_requirement": { "type": "string" }, + "blocked_requirement": { "type": "string" }, + "proof_check_id": { "type": "string" }, + "status": { "type": "string" }, + "mode": { "type": "string" }, + "why_now": { "type": "string" }, + "reason": { "type": "string" }, + "required_surface": { "type": "string" }, + "current_surface_status": { "type": "string" }, + "current_host_platform": { "type": "string" }, + "receipt_path": { "anyOf": [{ "type": "string" }, { "type": "null" }] }, + "suggested_loop": { "type": "array", "items": { "type": "string" } }, + "recommended_commands": { "type": "array", "items": { "type": "string" } }, + "stop_conditions": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": true + } + } +} diff --git a/docs/testing/local-proof-autonomy-program-test-plan.md b/docs/testing/local-proof-autonomy-program-test-plan.md new file mode 100644 index 000000000..a35f4217b --- /dev/null +++ b/docs/testing/local-proof-autonomy-program-test-plan.md @@ -0,0 +1,27 @@ +# CompareVI Local Proof Autonomy Program Test Plan + +## Document Control + +- System: CompareVI local proof autonomy program +- Version: `v0.1.1` +- Status: Active + +## Verification Matrix + +| Test ID | Coverage | Layer | Priority | Notes | +| --- | --- | --- | --- | --- | +| `TEST-LPAP-001` packet aggregation and requirement ranking coverage | Assurance/Program | High | Verifies the program consumes sibling packet next-step artifacts and ranks requirement work ahead of escalation work | +| `TEST-LPAP-002` shared-surface escalation merge coverage | Assurance/Program | High | Verifies shared escalations to the same external surface collapse into one bounded handoff packet | +| `TEST-LPAP-003` post-local promotion escalation coverage | Assurance/Program | High | Verifies the program emits a machine-readable promotion escalation instead of `null` when local packets are complete | +| `TEST-LPAP-004` concurrent bundle workspace safety coverage | Assurance/Program | High | Verifies packet-local CI surfaces use run-scoped audit bundle roots instead of deleting a shared `surface-bundle` workspace | + +## Entry Criteria + +- The program knowledgebase, SRS, and RTM stay in sync. +- The sibling packet local-CI entrypoints stay available through `package.json`. + +## Exit Criteria + +- Program selector tests pass. +- The program CI report and next-step artifacts validate against their schemas. +- When packet-local work is complete, the program emits a bounded post-local escalation instead of a silent terminal `null`. diff --git a/docs/testing/pester-service-model-test-plan.md b/docs/testing/pester-service-model-test-plan.md index 0e91b2485..686b9fff1 100644 --- a/docs/testing/pester-service-model-test-plan.md +++ b/docs/testing/pester-service-model-test-plan.md @@ -3,13 +3,14 @@ ## Overview - Release or baseline: - Pester service-model assurance packet `v0.1.0` + Pester service-model assurance packet `v0.1.20` - Owner: `#2069` with retained fork basis on `#2078` - Scope: Trusted routing, context receipts, selection receipts, readiness receipts, - execution-only behavior, and evidence classification for the Pester service - model + execution-only behavior, named execution packs, durable progress telemetry, + local replay surfaces, schema-governed retained artifacts, and evidence + classification for the Pester service model ## Test Items @@ -22,8 +23,30 @@ | `pester-gate.yml` + trusted pilot routing | Workflow | High | Verifies admission and orchestration across layers | | `pester-selection.yml` selection contract | Workflow | High | Verifies pack shaping and dispatcher-profile resolution leave execution clean | | `Invoke-PesterTests.ps1` execution contract | Execution | High | Verifies the dispatcher remains the execution engine only | +| `Invoke-PesterExecutionFinalize.ps1` + `Invoke-PesterExecutionPublication.ps1` finalize or publication contract | Execution | High | Verifies summary, manifest, session-index, leak-report, and operator-facing publication leave the dispatcher and are owned by dedicated helpers | | `Invoke-PesterExecutionPostprocess.ps1` execution-post contract | Execution | High | Verifies XML integrity classification and machine-readable summary repair occur after dispatch | +| `Invoke-PesterExecutionTelemetry.ps1` telemetry contract | Execution | High | Verifies dispatcher events and handshake markers are normalized into a durable telemetry artifact with last-known phase and pack identity | | `Run-PesterExecutionOnly.Local.ps1` local harness | Execution | High | Mirrors lock, LV guard, fixture prep, dispatcher profile, dispatch, execution-post, and local execution receipt without the workflow shell | +| `Write-PesterSummaryToStepSummary.Tests.ps1` + `Write-PesterSummaryToStepSummary.CompactMode.Tests.ps1` | Unit | High | Verifies failure-detail readers accept current and legacy payload shapes and keep summary/badge output truthful | +| `PesterFailurePayloadShape.Tests.ps1` | Unit | High | Verifies top-failure readers accept current and legacy payload shapes and do not silently drop failure detail | +| `PesterFailureProducerConsistency.Tests.ps1` + finalize degradation coverage | Unit/Execution | High | Verifies execution emits populated canonical failure detail for real failures and repairs empty detail to an explicit unavailable-details state when summary counts are nonzero | +| `TEST-PSM-012` named-pack and execution-group coverage | Contract/Execution | High | Verifies selection receipts, local entrypoints, and evidence retain a named execution-pack or test-group identity instead of relying on `IncludePatterns` alone | +| `TEST-PSM-013` local path-hygiene coverage | Unit/Execution | High | Verifies `PesterPathHygiene.Tests.ps1` and `Run-PesterExecutionOnly.Local.PathHygiene.Tests.ps1` for relocate and block behavior before dispatch | +| `TEST-PSM-014` retained-artifact replay coverage | Integration | High | Verifies `Invoke-PesterEvidenceClassification.Tests.ps1` and `Replay-PesterServiceModelArtifacts.Local.Tests.ps1` can rebuild postprocess, summary, totals, session index, and evidence outputs from mounted artifacts | +| `TEST-PSM-015` side-effect ownership coverage | Contract/Execution | High | Verifies dispatch stops at machine capture while finalize, postprocess, and publication helpers own summary, session-index, artifact manifest, leak-report, and operator-facing publication behavior | +| `TEST-PSM-016` durable progress telemetry coverage | Execution/Integration | High | Verifies long-running execution retains `dispatcher-events.ndjson` plus `pester-execution-telemetry.json`, and replay surfaces can inspect telemetry without rerunning dispatch | +| `TEST-PSM-017` schema-governance coverage | Contract | High | Verifies incompatible receipt or artifact schema changes are rejected explicitly by postprocess, evidence, and replay consumers with `unsupported-schema` classification | +| `TEST-PSM-018` promotion-comparison coverage | Assurance | High | Verifies the release-evidence bundle and promotion dossier retain requirement-to-run comparison evidence for representative named packs versus the current baseline | +| `TEST-PSM-019` named entrypoint coverage | Contract/Execution | High | Verifies common named packs are exposed through stable wrappers or commands instead of only through raw dispatcher arguments | +| `TEST-PSM-020` planned provenance coverage | Assurance/Contract | High | Will verify evidence and promotion outputs retain source raw-artifact, receipt, run, and ref provenance | +| `TEST-PSM-021` operator-outcome coverage | Integration/Assurance | High | Verifies failing gates emit `pester-operator-outcome.json` plus machine-readable classification, reasons, and actionable context before failing the job | +| `TEST-PSM-022` local autonomy-loop coverage | Assurance/Contract | High | Verifies local CI emits a machine-readable ranked backlog and a selected next requirement for LLM-guided local-first development | +| `TEST-PSM-023` autonomy-policy and stop-condition coverage | Assurance/Contract | High | Verifies local autonomy uses explicit policy, active worktree signals, and stop conditions to bound local iteration | +| `TEST-PSM-024` representative retained-artifact replay coverage | Integration/Execution | High | Verifies representative legacy retained artifacts replay locally without throwing and preserve the real evidence classification plus operator outcome | +| `TEST-PSM-025` windows-container surface coverage | Integration/Contract | High | Verifies the local Windows Docker Desktop + NI image surrogate emits an explicit bounded receipt before hosted reruns are chosen, including reachable Windows host bridge use from Unix or WSL coordinators | +| `TEST-PSM-026` proof-check aware autonomy coverage | Assurance/Contract | High | Verifies local CI consumes representative replay and Windows-surface proof checks and reopens implemented requirements when representative proof regresses | +| `TEST-PSM-027` next-step escalation coverage | Assurance/Contract | High | Verifies local CI emits a machine-readable escalation step when the next truthful proof surface is unavailable from the current host | +| `TEST-PSM-028` shared local-program selector coverage | Assurance/Contract | High | Verifies the shared program selector can choose a Pester requirement ahead of sibling escalations and merge the shared Windows Docker Desktop + NI image escalation across packets | ## Entry Criteria @@ -38,6 +61,10 @@ - Hosted packet quality and release-evidence workflows complete. - Any remaining action items are explicitly accepted before the slice widens beyond hosted quality and evidence. +- The local autonomy loop shall stay current with the packet so autonomous work + does not outrun the requirements baseline. +- The autonomy policy and stop conditions shall stay versioned and visible so + autonomous work remains bounded instead of improvisational. ## Coverage Targets @@ -47,6 +74,25 @@ | Local harness contract coverage | Local execution slice stays aligned with workflow execution boundaries | `tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs` | | Receipt coverage | Context, selection, readiness, execution, and evidence all emit auditable artifacts | assurance report + integration runs | | Classification coverage | blocked and defect outcomes remain distinguishable | evidence workflow outputs | +| Failure-detail interface coverage | Current and legacy `pester-failures.json` shapes remain readable and truthful under degradation | `tests/Write-PesterSummaryToStepSummary*.ps1`, `tests/PesterFailurePayloadShape.Tests.ps1` | +| Failure-detail producer consistency coverage | Execution never leaves nonzero failure counts without populated detail or an explicit unavailable-details state | `tests/PesterFailureProducerConsistency.Tests.ps1`, `tests/Invoke-PesterExecutionFinalize.Tests.ps1`, `TEST-PSM-011` | +| Named execution-pack coverage | Pack or group identity remains explicit across selection, execution, local entrypoints, and evidence | `TEST-PSM-012` | +| Local path-hygiene coverage | Local results and session-lock roots avoid synced or externally managed directories unless explicitly relocated or blocked | `TEST-PSM-013` | +| Retained-artifact replay coverage | Postprocess, summary, totals, session index, and evidence can be rebuilt locally from retained artifacts | `TEST-PSM-014` | +| Side-effect ownership coverage | Dispatcher does not reacquire finalize, postprocess, or evidence responsibilities | `TEST-PSM-015` | +| Durable progress telemetry coverage | Long-running execution emits inspectable progress outside live log streaming | `TEST-PSM-016` | +| Schema-governance coverage | Readers reject incompatible receipt or artifact schema drift explicitly with `unsupported-schema` classification | `TEST-PSM-017` | +| Promotion-comparison coverage | Promotion packets retain requirement-to-run comparison evidence on representative named packs | `tools/priority/__tests__/pester-service-model-release-evidence-provenance.test.mjs`, `tools/priority/__tests__/pester-service-model-release-evidence-workflow-contract.test.mjs`, `TEST-PSM-018` | +| Named entrypoint coverage | Common operator workflows use stable named wrappers instead of raw dispatcher arguments only | `TEST-PSM-019` | +| Provenance coverage | Derived evidence and promotion views can be traced back to the exact raw inputs and runs they came from | `tests/Invoke-PesterEvidenceProvenance.Tests.ps1`, `tests/Replay-PesterServiceModelArtifacts.Local.Tests.ps1`, `tools/priority/__tests__/pester-service-model-release-evidence-provenance.test.mjs`, `TEST-PSM-020` | +| Operator-outcome coverage | Failing gates remain explainable without manual log archaeology | `TEST-PSM-021` | +| Local autonomy-loop coverage | Local CI yields a ranked requirement backlog and selected next requirement for LLM-guided work | `tools/priority/pester-service-model-local-ci.mjs`, `TEST-PSM-022` | +| Autonomy-policy coverage | Local CI exposes explicit local-vs-hosted boundaries and active worktree signals | `tools/priority/pester-service-model-autonomy-policy.json`, `TEST-PSM-023` | +| Representative retained-artifact replay coverage | A reduced live-run fixture replays through current postprocess, evidence, and operator-outcome contracts without crashing on compatibility debt | `tests/Replay-PesterServiceModelRepresentativeArtifact.Tests.ps1`, `TEST-PSM-024` | +| Windows-container surface coverage | The local Docker Desktop Windows + pinned NI image surface is probed explicitly before hosted reruns are chosen, and reachable Windows host bridges are consumed before host-unavailable escalation is emitted | `tests/Invoke-PesterWindowsContainerSurfaceProbe.Tests.ps1`, `tests:windows-surface:probe`, `TEST-PSM-025` | +| Proof-check aware autonomy coverage | Local CI reopens implemented requirements when representative local proof regresses and records advisory surface status | `tools/priority/pester-service-model-local-ci.mjs`, `TEST-PSM-026` | +| Next-step escalation coverage | Local CI emits `pester-service-model-next-step.json` with a governed escalation packet when the next truthful proof surface is unavailable from the current host | `tools/priority/pester-service-model-local-ci.mjs`, `docs/schemas/pester-service-model-next-step-v1.schema.json`, `TEST-PSM-027` | +| Shared local-program selector coverage | Shared local CI emits `comparevi-local-program-next-step.json`, chooses requirement work ahead of sibling packet escalations, and merges the shared Windows Docker Desktop + NI image handoff across packets | `tools/priority/__tests__/comparevi-local-program-ci.test.mjs`, `tools/priority/comparevi-local-program-ci.mjs`, `docs/schemas/comparevi-local-program-next-step-v1.schema.json`, `TEST-PSM-028` | | Packet coverage gate | Retained `coverage.xml` and named PR coverage gate | `.github/workflows/pester-service-model-quality.yml` | | Promotion bundle retention | Hosted bundle retains the minimal promotion handoff | `.github/workflows/pester-service-model-release-evidence.yml` | diff --git a/docs/testing/vi-history-local-proof-test-plan.md b/docs/testing/vi-history-local-proof-test-plan.md new file mode 100644 index 000000000..6003839a1 --- /dev/null +++ b/docs/testing/vi-history-local-proof-test-plan.md @@ -0,0 +1,47 @@ +# VI History Local Proof Test Plan + +## Document Control + +- System: VI History local proof control plane +- Version: `v0.1.2` +- Status: Active + +## Verification Matrix + +| Test ID | Coverage | Layer | Priority | Notes | +| --- | --- | --- | --- | --- | +| `TEST-VHLP-001` local Windows workflow replay coverage | Contract/Replay | High | Verifies the governed `vi-history-scenarios-windows` replay lane emits a bounded receipt and compare artifact paths, including bridge-backed Windows launch from Unix or WSL coordinators | +| `TEST-VHLP-002` local refinement profile coverage | Execution/Local | High | Verifies `proof`, `dev-fast`, `warm-dev`, and `windows-mirror-proof` local refinement behavior and retained receipts | +| `TEST-VHLP-003` local operator-session coverage | Operator/Local | High | Verifies local operator-session wrappers consume the refinement helper and emit canonical local session contracts | +| `TEST-VHLP-004` workflow-readiness envelope coverage | Evidence/Decision | High | Verifies the VI History workflow-readiness envelope captures lane lifecycle, verdict, and recommendation | +| `TEST-VHLP-005` local autonomy-loop coverage | Assurance/Contract | High | Verifies local VI History CI emits a report, ranked backlog, and next-step artifact | +| `TEST-VHLP-006` next-step escalation coverage | Assurance/Contract | High | Verifies local VI History CI emits a machine-readable escalation step to the shared Windows Docker Desktop + NI image surface only after native or reachable bridge-backed Windows checks cannot satisfy it | +| `TEST-VHLP-007` shared local-program selector coverage | Assurance/Contract | High | Verifies the shared program selector can choose VI History requirement work explicitly and merge the shared Windows Docker Desktop + NI image escalation across sibling packets | +| `TEST-VHLP-008` clone-backed live-history candidate governance coverage | Assurance/Contract | High | Verifies the packet names `ni/labview-icon-editor` plus `Tooling/deployment/VIP_Pre-Uninstall Custom Action.vi` as the governed clone-backed live-history candidate | +| `TEST-VHLP-009` live-history candidate readiness coverage | Assurance/Contract | High | Verifies local VI History CI validates clone presence, target path presence, and git history, then emits a bounded clone-preparation escalation when the candidate is unavailable | + +## Entry Criteria + +- The VI History local-proof packet docs and RTM stay in sync. +- The Windows replay lane, local refinement helper, operator-session helper, and workflow-readiness helper stay on disk at the declared paths. + +## Exit Criteria + +- Contract tests and PowerShell tests covering the packet pass. +- The local VI History CI emits a machine-readable next step. +- The clone-backed live-history candidate is governed explicitly. +- If the current host cannot satisfy the Windows replay lane, the next step is an explicit escalation packet rather than a prose-only advisory. + +## Traceability Notes + +| Coverage | Proof | Reference | +| --- | --- | --- | +| Local Windows workflow replay coverage | `windows-workflow-replay-lane.mjs` exposes `vi-history-scenarios-windows` replay, supports reachable Windows host bridge launch, and validates its receipt | `tools/priority/__tests__/windows-workflow-replay-lane.test.mjs`, `TEST-VHLP-001` | +| Local refinement profile coverage | `Invoke-VIHistoryLocalRefinement.ps1` emits profile-specific receipts, benchmarks, and review-loop artifacts | `tests/VIHistoryLocalAcceleration.Tests.ps1`, `TEST-VHLP-002` | +| Local operator-session coverage | `Invoke-VIHistoryLocalOperatorSession.ps1` wraps refinement into a local operator-facing contract | `tests/VIHistoryLocalOperatorSession.Tests.ps1`, `TEST-VHLP-003` | +| Workflow-readiness coverage | `Write-VIHistoryWorkflowReadiness.ps1` emits `vi-history/workflow-readiness@v1` | `tests/Write-VIHistoryWorkflowReadiness.Tests.ps1`, `TEST-VHLP-004` | +| Local autonomy-loop coverage | Local VI History CI emits report, next step, and proof-check reasoning | `tools/priority/__tests__/vi-history-local-ci.test.mjs`, `TEST-VHLP-005` | +| Next-step escalation coverage | Local VI History CI emits a shared Windows-surface escalation packet only after native or bridge-backed Windows checks fail or remain non-ready | `tools/priority/__tests__/vi-history-local-ci.test.mjs`, `TEST-VHLP-006` | +| Shared local-program selector coverage | Shared local CI emits `comparevi-local-program-next-step.json`, selects VI History explicitly when it owns the next requirement, and merges the shared Windows surface handoff across packets | `tools/priority/__tests__/comparevi-local-program-ci.test.mjs`, `tools/priority/comparevi-local-program-ci.mjs`, `docs/schemas/comparevi-local-program-next-step-v1.schema.json`, `TEST-VHLP-007` | +| Clone-backed live-history candidate governance coverage | The governed candidate manifest names `ni/labview-icon-editor` and `Tooling/deployment/VIP_Pre-Uninstall Custom Action.vi` explicitly | `tools/priority/__tests__/vi-history-local-proof-contract.test.mjs`, `tools/priority/vi-history-live-candidate.json`, `TEST-VHLP-008` | +| Live-history candidate readiness coverage | Local VI History CI emits `vi-history-live-candidate-readiness.json` and escalates clone preparation when the governed target is unavailable | `tools/priority/__tests__/vi-history-local-ci.test.mjs`, `tools/priority/vi-history-local-ci.mjs`, `docs/schemas/vi-history-live-candidate-v1.schema.json`, `docs/schemas/vi-history-live-candidate-readiness-v1.schema.json`, `TEST-VHLP-009` | diff --git a/docs/testing/windows-docker-shared-surface-test-plan.md b/docs/testing/windows-docker-shared-surface-test-plan.md new file mode 100644 index 000000000..8ee00022e --- /dev/null +++ b/docs/testing/windows-docker-shared-surface-test-plan.md @@ -0,0 +1,44 @@ +# Windows Docker Shared Surface Test Plan + +## Document Control + +- System: Windows Docker shared local proof surface +- Version: `v0.2.1` +- Status: Active + +## Verification Matrix + +| Test ID | Coverage | Layer | Priority | Notes | +| --- | --- | --- | --- | --- | +| `TEST-WDSS-001` shared-surface readiness probe coverage | Probe/Receipt | High | Verifies the shared Windows readiness probe emits bounded readiness states and a machine-readable receipt | +| `TEST-WDSS-002` bootstrap and preflight contract coverage | Bootstrap/Host | High | Verifies the deterministic Windows host bootstrap/preflight commands remain explicit and stable | +| `TEST-WDSS-003` path-hygiene coverage | Safety/Local | High | Verifies the shared surface detects OneDrive-like managed roots and emits a relocation escalation | +| `TEST-WDSS-004` local assurance-loop coverage | Assurance/Contract | High | Verifies the shared-surface local CI emits a report, proof checks, and next-step artifact | +| `TEST-WDSS-005` host-unavailable escalation coverage | Assurance/Contract | High | Verifies local CI emits a machine-readable escalation to `windows-docker-desktop-ni-image` when the current host cannot satisfy the surface | +| `TEST-WDSS-006` shared local-program selector coverage | Assurance/Contract | High | Verifies the shared surface participates explicitly in the program selector beside Pester and VI History | +| `TEST-WDSS-007` reachable Windows host bridge coverage | Assurance/Contract | High | Verifies a Unix or WSL coordinator uses a reachable Windows host bridge for probe and preflight work before emitting host-unavailable escalation | +| `TEST-WDSS-008` UNC-backed WSL staging coverage | Runtime/Windows Docker | High | Verifies UNC-backed WSL inputs and report paths are staged into a Windows-local mount root, synchronized back, and cleaned up after compare execution | + +## Entry Criteria + +- The shared Windows surface packet docs and RTM stay in sync. +- The readiness probe, bootstrap/preflight scripts, and local CI entrypoint stay on disk at the declared paths. + +## Exit Criteria + +- Contract and local-CI tests covering the packet pass. +- The shared-surface local CI emits a machine-readable next step. +- On a non-Windows host with no reachable Windows bridge, the next step is an explicit escalation packet rather than prose-only guidance. + +## Traceability Notes + +| Coverage | Proof | Reference | +| --- | --- | --- | +| Shared-surface readiness probe coverage | `Invoke-PesterWindowsContainerSurfaceProbe.ps1` emits a bounded readiness receipt | `tests/Invoke-PesterWindowsContainerSurfaceProbe.Tests.ps1`, `tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs`, `TEST-WDSS-001` | +| Bootstrap and preflight contract coverage | The packet documents and preserves the deterministic bootstrap plus probe command order and underlying tools | `tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs`, `TEST-WDSS-002` | +| Path-hygiene coverage | Shared-surface local CI emits `windows-docker-shared-surface-path-hygiene.json` and a relocation escalation on OneDrive-like roots | `tools/priority/__tests__/windows-docker-shared-surface-local-ci.test.mjs`, `TEST-WDSS-003` | +| Local assurance-loop coverage | Shared-surface local CI emits report, summary, and next-step artifacts | `tools/priority/__tests__/windows-docker-shared-surface-local-ci.test.mjs`, `TEST-WDSS-004` | +| Host-unavailable escalation coverage | Shared-surface local CI emits an explicit `windows-docker-desktop-ni-image` handoff packet | `tools/priority/__tests__/windows-docker-shared-surface-local-ci.test.mjs`, `TEST-WDSS-005` | +| Shared local-program selector coverage | Program CI consumes the shared-surface next-step artifact beside Pester and VI History | `tools/priority/__tests__/comparevi-local-program-ci.test.mjs`, `tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs`, `TEST-WDSS-006` | +| Reachable Windows host bridge coverage | Shared-surface local CI can consume a reachable Windows host bridge from a Unix or WSL coordinator and only escalates when that bridge is absent or still non-ready | `tools/priority/__tests__/windows-host-bridge.test.mjs`, `tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs`, `TEST-WDSS-007` | +| UNC-backed WSL staging coverage | `Run-NIWindowsContainerCompare.ps1` stages UNC-backed or otherwise non-bindable Windows paths into a local Windows mount root, syncs artifacts back, and records staging status in capture output | `tests/Run-NIWindowsContainerCompare.Tests.ps1`, `tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs`, `tools/priority/__tests__/vi-history-local-proof-contract.test.mjs`, `TEST-WDSS-008` | diff --git a/package.json b/package.json index f1a56ea14..9395da7f5 100644 --- a/package.json +++ b/package.json @@ -178,6 +178,15 @@ "priority:slo": "node tools/priority/slo-metrics.mjs", "priority:operator-status": "pwsh -NoLogo -NoProfile -File tools/priority/Write-OperatorStatusSummary.ps1", "priority:parity": "node tools/priority/report-origin-upstream-parity.mjs", + "priority:vi-history:local-ci": "node tools/priority/vi-history-local-ci.mjs", + "priority:vi-history:next-step": "node tools/priority/vi-history-local-ci.mjs --print-next-step", + "priority:windows-surface:local-ci": "node tools/priority/windows-docker-shared-surface-local-ci.mjs", + "priority:windows-surface:next-step": "node tools/priority/windows-docker-shared-surface-local-ci.mjs --print-next-step", + "priority:program:local-ci": "node tools/priority/comparevi-local-program-ci.mjs", + "priority:program:next-step": "node tools/priority/comparevi-local-program-ci.mjs --print-next-step", + "priority:pester:local-ci": "node tools/priority/pester-service-model-local-ci.mjs", + "priority:pester:next-requirement": "node tools/priority/pester-service-model-local-ci.mjs --print-next", + "priority:pester:next-step": "node tools/priority/pester-service-model-local-ci.mjs --print-next-step", "priority:project:portfolio:check": "tsc -p tsconfig.cli.json && node dist/tools/cli/project-portfolio.js check", "priority:project:portfolio:apply": "tsc -p tsconfig.cli.json && node dist/tools/cli/project-portfolio.js apply", "priority:project:portfolio:snapshot": "tsc -p tsconfig.cli.json && node dist/tools/cli/project-portfolio.js snapshot", @@ -254,8 +263,15 @@ "smoke:vi-history": "pwsh -NoLogo -NoProfile -File tools/Test-PRVIHistorySmoke.ps1", "smoke:vi-stage": "pwsh -NoLogo -NoProfile -File tools/Test-PRVIStagingSmoke.ps1", "tests:execution:local": "pwsh -NoLogo -NoProfile -File tools/Run-PesterExecutionOnly.Local.ps1", - "tests:comparevi": "pwsh -NoLogo -NoProfile -File Invoke-PesterTests.ps1 -IncludePatterns CompareVI*", - "tests:comparevi:int": "pwsh -NoLogo -NoProfile -File Invoke-PesterTests.ps1 -IncludePatterns CompareVI* -IntegrationMode include", + "tests:replay:local": "pwsh -NoLogo -NoProfile -File tools/Replay-PesterServiceModelArtifacts.Local.ps1", + "tests:replay:representative": "pwsh -NoLogo -NoProfile -Command \"& 'tools/Replay-PesterServiceModelArtifacts.Local.ps1' -RawArtifactDir 'tests/fixtures/pester-service-model/legacy-results-xml-truncated/raw' -ExecutionReceiptPath 'tests/fixtures/pester-service-model/legacy-results-xml-truncated/pester-run-receipt.json' -WorkspaceResultsDir 'tests/results/pester-replay-representative'\"", + "tests:pack:full": "pwsh -NoLogo -NoProfile -File tools/Run-PesterExecutionOnly.Local.ps1 -ExecutionPack full", + "tests:pack:comparevi": "pwsh -NoLogo -NoProfile -File tools/Run-PesterExecutionOnly.Local.ps1 -ExecutionPack comparevi", + "tests:pack:dispatcher": "pwsh -NoLogo -NoProfile -File tools/Run-PesterExecutionOnly.Local.ps1 -ExecutionPack dispatcher", + "tests:pack:workflow": "pwsh -NoLogo -NoProfile -File tools/Run-PesterExecutionOnly.Local.ps1 -ExecutionPack workflow", + "tests:windows-surface:probe": "pwsh -NoLogo -NoProfile -File tools/Invoke-PesterWindowsContainerSurfaceProbe.ps1", + "tests:comparevi": "pwsh -NoLogo -NoProfile -File Invoke-PesterTests.ps1 -ExecutionPack comparevi", + "tests:comparevi:int": "pwsh -NoLogo -NoProfile -File Invoke-PesterTests.ps1 -ExecutionPack comparevi -IntegrationMode include", "tests:comparevi:single-int": "pwsh -NoLogo -NoProfile -Command \"$env:LV_BASE_VI=(Resolve-Path 'VI1.vi').Path; $env:LV_HEAD_VI=(Resolve-Path 'VI2.vi').Path; pwsh -NoLogo -NoProfile -File Invoke-PesterTests.ps1 -TestsPath tests/CompareVI.RealCli.SingleRun.Integration.Tests.ps1 -IntegrationMode include\"", "tests:discover": "tsc -p tsconfig.json && node dist/tools/test-discovery.js --tests tests --out tests/results/_agent/test-manifest.json", "watch:pester": "node tools/follow-pester-artifacts.mjs --results tests/results", diff --git a/scripts/Write-PesterSummaryToStepSummary.ps1 b/scripts/Write-PesterSummaryToStepSummary.ps1 index 9db553701..8f744f465 100644 --- a/scripts/Write-PesterSummaryToStepSummary.ps1 +++ b/scripts/Write-PesterSummaryToStepSummary.ps1 @@ -25,12 +25,18 @@ param( [switch]$EmitFailureBadge, [switch]$Compact, [string]$CommentPath, - [string]$BadgeJsonPath + [string]$BadgeJsonPath, + [string]$OperatorOutcomePath = 'pester-operator-outcome.json' ) Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' +$failurePayloadTool = Join-Path (Split-Path -Parent $PSScriptRoot) 'tools' 'PesterFailurePayload.ps1' +if (Test-Path -LiteralPath $failurePayloadTool -PathType Leaf) { + . $failurePayloadTool +} + function Get-SummaryValue { param( $InputObject, @@ -54,6 +60,34 @@ function Get-SummaryValue { return $null } +function Get-OperatorOutcome { + param( + [Parameter(Mandatory = $true)][string]$ResultsDir, + [string]$OutcomePath + ) + + if ([string]::IsNullOrWhiteSpace($OutcomePath)) { + return $null + } + + $resolvedPath = if ([System.IO.Path]::IsPathRooted($OutcomePath)) { + $OutcomePath + } else { + Join-Path $ResultsDir $OutcomePath + } + + if (-not (Test-Path -LiteralPath $resolvedPath -PathType Leaf)) { + return $null + } + + try { + return (Get-Content -LiteralPath $resolvedPath -Raw | ConvertFrom-Json -ErrorAction Stop) + } catch { + Write-Warning ("Failed to parse operator outcome: {0}" -f $_.Exception.Message) + return $null + } +} + if (-not $env:GITHUB_STEP_SUMMARY -and -not $CommentPath) { Write-Warning 'GITHUB_STEP_SUMMARY not set and no -CommentPath provided; skipping summary emission.' return @@ -62,6 +96,7 @@ if (-not $env:GITHUB_STEP_SUMMARY -and -not $CommentPath) { $summaryPath = Join-Path $ResultsDir 'pester-summary.json' $txtPath = Join-Path $ResultsDir 'pester-summary.txt' $xmlPath = Join-Path $ResultsDir 'pester-results.xml' +$operatorOutcome = Get-OperatorOutcome -ResultsDir $ResultsDir -OutcomePath $OperatorOutcomePath $_accumulatedLines = [System.Collections.Generic.List[string]]::new() function Add-Line($s) { $_accumulatedLines.Add([string]$s) | Out-Null } @@ -117,10 +152,15 @@ if ($EmitFailureBadge -or $Compact) { $status = if ($failedCount -gt 0) { 'failed' } else { 'passed' } $failJsonFile = Join-Path $ResultsDir 'pester-failures.json' $failedNames = @() + $failureDetailsStatus = [string](Get-SummaryValue -InputObject $totals -PropertyNames @('failureDetailsStatus')) + $failureDetailsReason = [string](Get-SummaryValue -InputObject $totals -PropertyNames @('failureDetailsReason')) if (Test-Path $failJsonFile) { try { $fj = Get-Content $failJsonFile -Raw | ConvertFrom-Json - $failedNames = @($fj.results | Where-Object { $_.result -eq 'Failed' } | ForEach-Object { $_.Name }) + $detailState = Get-PesterFailureDetailState -FailurePayload $fj -Summary $totals + $failureDetailsStatus = $detailState.detailStatus + $failureDetailsReason = $detailState.unavailableReason + $failedNames = @($detailState.entries | Where-Object { $_.result -eq 'Failed' } | ForEach-Object { $_.name }) } catch { } } $badgeObj = [pscustomobject]@{ @@ -134,6 +174,13 @@ if ($EmitFailureBadge -or $Compact) { badgeMarkdown = $badge badgeText = ($badge -replace '\*','') failedTests = $failedNames + failureDetailsStatus = $failureDetailsStatus + failureDetailsReason = $failureDetailsReason + classification = if ($operatorOutcome) { [string]$operatorOutcome.classification } else { '' } + gateStatus = if ($operatorOutcome) { [string]$operatorOutcome.gateStatus } else { '' } + nextActionId = if ($operatorOutcome) { [string]$operatorOutcome.nextActionId } else { '' } + nextAction = if ($operatorOutcome) { [string]$operatorOutcome.nextAction } else { '' } + reasons = if ($operatorOutcome) { @($operatorOutcome.reasons) } else { @() } generatedAt = (Get-Date).ToUniversalTime().ToString('o') } try { @@ -169,19 +216,35 @@ if ($Compact) { if ($duration -ne $null) { $pieces += ("{0}s" -f $duration) } Add-Line '' Add-Line ("**Totals:** {0}" -f ($pieces -join ' • ')) + $executionPack = Get-SummaryValue -InputObject $totals -PropertyNames @('executionPack','ExecutionPack') + if ($executionPack) { + Add-Line ("**Pack:** {0}" -f $executionPack) + } if ($failedCount -gt 0) { # failed test names (short) $failJsonPath = Join-Path $ResultsDir 'pester-failures.json' if (Test-Path $failJsonPath) { try { $failData = Get-Content $failJsonPath -Raw | ConvertFrom-Json - $failedNames = @($failData.results | Where-Object { $_.result -eq 'Failed' } | ForEach-Object { $_.Name }) + $detailState = Get-PesterFailureDetailState -FailurePayload $failData -Summary $totals + $failedNames = @($detailState.entries | Where-Object { $_.result -eq 'Failed' } | ForEach-Object { $_.name }) if ($failedNames.Count) { Add-Line ("**Failures:** {0}" -f ($failedNames -join ', ')) + } elseif ($detailState.detailStatus -eq 'unavailable') { + $reasonSuffix = if ($detailState.unavailableReason) { " ({0})" -f $detailState.unavailableReason } else { '' } + Add-Line ("**Failure Details:** unavailable{0}" -f $reasonSuffix) } } catch { Write-Warning 'Compact mode: failed to parse failure names.' } + } elseif ([string](Get-SummaryValue -InputObject $totals -PropertyNames @('failureDetailsStatus')) -eq 'unavailable') { + $reasonValue = [string](Get-SummaryValue -InputObject $totals -PropertyNames @('failureDetailsReason')) + $reasonSuffix = if ($reasonValue) { " ({0})" -f $reasonValue } else { '' } + Add-Line ("**Failure Details:** unavailable{0}" -f $reasonSuffix) } } + if ($operatorOutcome -and [string]$operatorOutcome.classification -ne 'ok') { + Add-Line ("**Gate Outcome:** {0} ({1})" -f [string]$operatorOutcome.classification, [string]$operatorOutcome.gateStatus) + Add-Line ("**Next Action:** {0}" -f [string]$operatorOutcome.nextAction) + } Flush-Outputs Write-Host 'Pester summary (compact) written.' -ForegroundColor Green return @@ -199,13 +262,28 @@ $skippedValue = Get-SummaryValue -InputObject $totals -PropertyNames @('Skipped' if ($skippedValue -ne $null) { Add-Line ("| Skipped | {0} |" -f $skippedValue) } $durationValue = Get-SummaryValue -InputObject $totals -PropertyNames @('Duration','duration') if ($durationValue -ne $null) { Add-Line ("| Duration (s) | {0} |" -f $durationValue) } +$executionPackValue = Get-SummaryValue -InputObject $totals -PropertyNames @('executionPack','ExecutionPack') +if ($executionPackValue) { Add-Line ("| Execution Pack | {0} |" -f $executionPackValue) } + +if ($operatorOutcome -and [string]$operatorOutcome.classification -ne 'ok') { + Add-Line '' + Add-Line '### Operator Outcome' + Add-Line '' + Add-Line ("- Gate status: {0}" -f [string]$operatorOutcome.gateStatus) + Add-Line ("- Classification: {0}" -f [string]$operatorOutcome.classification) + if (@($operatorOutcome.reasons).Count -gt 0) { + Add-Line ("- Reasons: {0}" -f ((@($operatorOutcome.reasons)) -join ', ')) + } + Add-Line ("- Next action: {0}" -f [string]$operatorOutcome.nextAction) +} # Optional failed test details from failures JSON if present $failJson = Join-Path $ResultsDir 'pester-failures.json' if (Test-Path $failJson) { try { $failData = Get-Content $failJson -Raw | ConvertFrom-Json - $failed = @($failData.results | Where-Object { $_.result -eq 'Failed' }) + $detailState = Get-PesterFailureDetailState -FailurePayload $failData -Summary $totals + $failed = @($detailState.entries | Where-Object { $_.result -eq 'Failed' }) if ($failed.Count) { Add-Line '' switch ($FailedTestsCollapseStyle) { @@ -244,8 +322,17 @@ if (Test-Path $failJson) { } } if ($FailedTestsCollapseStyle -like 'Details*') { Add-Line '' } + } elseif ($detailState.detailStatus -eq 'unavailable') { + Add-Line '' + $reasonSuffix = if ($detailState.unavailableReason) { " ({0})" -f $detailState.unavailableReason } else { '' } + Add-Line ("Failure details unavailable{0}." -f $reasonSuffix) } } catch { Write-Warning ("Failed to parse failure JSON: {0}" -f $_.Exception.Message) } +} elseif ([string](Get-SummaryValue -InputObject $totals -PropertyNames @('failureDetailsStatus')) -eq 'unavailable') { + Add-Line '' + $reasonValue = [string](Get-SummaryValue -InputObject $totals -PropertyNames @('failureDetailsReason')) + $reasonSuffix = if ($reasonValue) { " ({0})" -f $reasonValue } else { '' } + Add-Line ("Failure details unavailable{0}." -f $reasonSuffix) } Flush-Outputs diff --git a/tests/Invoke-PesterEvidenceClassification.Tests.ps1 b/tests/Invoke-PesterEvidenceClassification.Tests.ps1 new file mode 100644 index 000000000..800e435cf --- /dev/null +++ b/tests/Invoke-PesterEvidenceClassification.Tests.ps1 @@ -0,0 +1,124 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Describe 'Invoke-PesterEvidenceClassification' -Tag 'Unit' { + BeforeAll { + $script:repoRoot = Split-Path -Parent $PSScriptRoot + $script:toolPath = Join-Path $script:repoRoot 'tools/Invoke-PesterEvidenceClassification.ps1' + } + + It 'classifies retained passing evidence as ok' { + $tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("pester-evidence-classification-" + [Guid]::NewGuid().ToString('N')) + $resultsDir = Join-Path $tempRoot 'results' + $receiptPath = Join-Path $tempRoot 'pester-run-receipt.json' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + + try { + ([ordered]@{ + schemaVersion = '1.7.1' + total = 3 + passed = 3 + failed = 0 + errors = 0 + skipped = 0 + duration_s = 0.42 + resultsXmlStatus = 'complete' + } | ConvertTo-Json -Depth 5) | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-summary.json') -Encoding UTF8 + + ([ordered]@{ + schema = 'pester-execution-receipt@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + contextStatus = 'ready' + readinessStatus = 'ready' + selectionStatus = 'ready' + selectionExecutionPack = 'dispatcher' + selectionExecutionPackSource = 'declared' + dispatcherExitCode = 0 + executionJobResult = 'success' + status = 'completed' + } | ConvertTo-Json -Depth 5) | Set-Content -LiteralPath $receiptPath -Encoding UTF8 + + & $script:toolPath -ResultsDir $resultsDir -ExecutionReceiptPath $receiptPath -RawArtifactDownload staged | Out-Host + $LASTEXITCODE | Should -Be 0 + + $classification = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-evidence-classification.json') -Raw | ConvertFrom-Json + $classification.classification | Should -Be 'ok' + $classification.selectionExecutionPack | Should -Be 'dispatcher' + $classification.rawArtifactDownload | Should -Be 'staged' + } finally { + if (Test-Path -LiteralPath $tempRoot) { + Remove-Item -LiteralPath $tempRoot -Recurse -Force -ErrorAction SilentlyContinue + } + } + } + + It 'classifies an incompatible execution receipt schema as unsupported-schema instead of throwing' { + $tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("pester-evidence-classification-" + [Guid]::NewGuid().ToString('N')) + $resultsDir = Join-Path $tempRoot 'results' + $receiptPath = Join-Path $tempRoot 'pester-run-receipt.json' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + + try { + ([ordered]@{ + schema = 'pester-execution-receipt@v2' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + status = 'completed' + } | ConvertTo-Json -Depth 5) | Set-Content -LiteralPath $receiptPath -Encoding UTF8 + + & $script:toolPath -ResultsDir $resultsDir -ExecutionReceiptPath $receiptPath -RawArtifactDownload staged | Out-Host + $LASTEXITCODE | Should -Be 0 + + $classification = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-evidence-classification.json') -Raw | ConvertFrom-Json + $classification.classification | Should -Be 'unsupported-schema' + $classification.executionReceiptSchemaStatus | Should -Be 'unsupported-schema' + $classification.reasons | Should -Contain 'execution-receipt-unsupported-schema' + $classification.reasons | Should -Contain 'execution-receipt-schema=pester-execution-receipt@v2' + } finally { + if (Test-Path -LiteralPath $tempRoot) { + Remove-Item -LiteralPath $tempRoot -Recurse -Force -ErrorAction SilentlyContinue + } + } + } + + It 'classifies a legacy execution receipt without selectionExecutionPack fields without throwing' { + $tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("pester-evidence-classification-" + [Guid]::NewGuid().ToString('N')) + $resultsDir = Join-Path $tempRoot 'results' + $receiptPath = Join-Path $tempRoot 'pester-run-receipt.json' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + + try { + ([ordered]@{ + schemaVersion = '1.7.1' + total = 4 + passed = 2 + failed = 1 + errors = 0 + skipped = 1 + resultsXmlStatus = 'truncated-root' + } | ConvertTo-Json -Depth 5) | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-summary.json') -Encoding UTF8 + + ([ordered]@{ + schema = 'pester-execution-receipt@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + contextStatus = 'ready' + readinessStatus = 'ready' + selectionStatus = 'ready' + dispatcherExitCode = -1 + executionJobResult = 'success' + status = 'results-xml-truncated' + } | ConvertTo-Json -Depth 5) | Set-Content -LiteralPath $receiptPath -Encoding UTF8 + + & $script:toolPath -ResultsDir $resultsDir -ExecutionReceiptPath $receiptPath -RawArtifactDownload staged | Out-Host + $LASTEXITCODE | Should -Be 0 + + $classification = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-evidence-classification.json') -Raw | ConvertFrom-Json + $classification.classification | Should -Be 'results-xml-truncated' + $classification.selectionExecutionPack | Should -Be '' + $classification.selectionExecutionPackSource | Should -Be '' + } finally { + if (Test-Path -LiteralPath $tempRoot) { + Remove-Item -LiteralPath $tempRoot -Recurse -Force -ErrorAction SilentlyContinue + } + } + } +} diff --git a/tests/Invoke-PesterEvidenceProvenance.Tests.ps1 b/tests/Invoke-PesterEvidenceProvenance.Tests.ps1 new file mode 100644 index 000000000..d50585850 --- /dev/null +++ b/tests/Invoke-PesterEvidenceProvenance.Tests.ps1 @@ -0,0 +1,113 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Describe 'Invoke-PesterEvidenceProvenance' -Tag 'Evidence' { + BeforeAll { + $script:repoRoot = Split-Path -Parent $PSScriptRoot + $script:toolPath = Join-Path $script:repoRoot 'tools/Invoke-PesterEvidenceProvenance.ps1' + } + + It 'records source raw inputs, receipt identity, derived evidence outputs, and run context for evidence' { + $tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("pester-evidence-provenance-" + [Guid]::NewGuid().ToString('N')) + $resultsDir = Join-Path $tempRoot 'results' + $contractDir = Join-Path $tempRoot 'execution-contract' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + New-Item -ItemType Directory -Path $contractDir -Force | Out-Null + + $originalRepo = $env:GITHUB_REPOSITORY + $originalWorkflow = $env:GITHUB_WORKFLOW + $originalEvent = $env:GITHUB_EVENT_NAME + $originalRunId = $env:GITHUB_RUN_ID + $originalRunAttempt = $env:GITHUB_RUN_ATTEMPT + $originalRef = $env:GITHUB_REF + $originalRefName = $env:GITHUB_REF_NAME + $originalSha = $env:GITHUB_SHA + $originalServerUrl = $env:GITHUB_SERVER_URL + + try { + $env:GITHUB_REPOSITORY = 'LabVIEW-Community-CI-CD/compare-vi-cli-action' + $env:GITHUB_WORKFLOW = 'Pester evidence' + $env:GITHUB_EVENT_NAME = 'workflow_call' + $env:GITHUB_RUN_ID = '777' + $env:GITHUB_RUN_ATTEMPT = '3' + $env:GITHUB_REF = 'refs/heads/integration/pester-service-model' + $env:GITHUB_REF_NAME = 'integration/pester-service-model' + $env:GITHUB_SHA = '0123456789abcdef0123456789abcdef01234567' + $env:GITHUB_SERVER_URL = 'https://github.com' + + '' | + Set-Content -LiteralPath (Join-Path $resultsDir 'pester-results.xml') -Encoding UTF8 + ([ordered]@{ + schema = 'pester-summary/v1' + schemaVersion = '1.7.1' + total = 2 + passed = 2 + failed = 0 + errors = 0 + duration_s = 1 + } | ConvertTo-Json -Depth 6) | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-summary.json') -Encoding UTF8 + ([ordered]@{ + schema = 'pester-execution-postprocess@v1' + schemaVersion = '1.0.0' + status = 'complete' + } | ConvertTo-Json -Depth 6) | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-execution-postprocess.json') -Encoding UTF8 + ([ordered]@{ + schema = 'pester-execution-telemetry@v1' + schemaVersion = '1.0.0' + telemetryStatus = 'telemetry-available' + } | ConvertTo-Json -Depth 6) | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-execution-telemetry.json') -Encoding UTF8 + ([ordered]@{ + schema = 'pester-evidence-classification@v1' + classification = 'ok' + } | ConvertTo-Json -Depth 6) | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-evidence-classification.json') -Encoding UTF8 + ([ordered]@{ + schema = 'pester-operator-outcome@v1' + gateStatus = 'pass' + } | ConvertTo-Json -Depth 6) | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-operator-outcome.json') -Encoding UTF8 + ([ordered]@{ + schema = 'pester-totals/v1' + total = 2 + } | ConvertTo-Json -Depth 6) | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-totals.json') -Encoding UTF8 + ([ordered]@{ + schema = 'session-index/v1' + entries = @() + } | ConvertTo-Json -Depth 6) | Set-Content -LiteralPath (Join-Path $resultsDir 'session-index.json') -Encoding UTF8 + ([ordered]@{ + schema = 'pester-execution-receipt@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + status = 'completed' + dispatcherExitCode = 0 + } | ConvertTo-Json -Depth 6) | Set-Content -LiteralPath (Join-Path $contractDir 'pester-run-receipt.json') -Encoding UTF8 + + & $script:toolPath ` + -ResultsDir $resultsDir ` + -ExecutionReceiptPath (Join-Path $contractDir 'pester-run-receipt.json') ` + -RawArtifactName 'pester-run-raw' ` + -RawArtifactDownload 'success' | Out-Host + $LASTEXITCODE | Should -Be 0 + + $provenance = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-evidence-provenance.json') -Raw | ConvertFrom-Json -ErrorAction Stop + $provenance.schema | Should -Be 'pester-derived-provenance@v1' + $provenance.provenanceKind | Should -Be 'evidence' + $provenance.subject.rawArtifactName | Should -Be 'pester-run-raw' + $provenance.runContext.runId | Should -Be '777' + ($provenance.sourceInputs | Where-Object role -eq 'summary').present | Should -BeTrue + ($provenance.sourceInputs | Where-Object role -eq 'execution-receipt').schema | Should -Be 'pester-execution-receipt@v1' + ($provenance.derivedOutputs | Where-Object role -eq 'classification').schema | Should -Be 'pester-evidence-classification@v1' + ($provenance.derivedOutputs | Where-Object role -eq 'operator-outcome').schema | Should -Be 'pester-operator-outcome@v1' + } finally { + $env:GITHUB_REPOSITORY = $originalRepo + $env:GITHUB_WORKFLOW = $originalWorkflow + $env:GITHUB_EVENT_NAME = $originalEvent + $env:GITHUB_RUN_ID = $originalRunId + $env:GITHUB_RUN_ATTEMPT = $originalRunAttempt + $env:GITHUB_REF = $originalRef + $env:GITHUB_REF_NAME = $originalRefName + $env:GITHUB_SHA = $originalSha + $env:GITHUB_SERVER_URL = $originalServerUrl + if (Test-Path -LiteralPath $tempRoot) { + Remove-Item -LiteralPath $tempRoot -Recurse -Force -ErrorAction SilentlyContinue + } + } + } +} diff --git a/tests/Invoke-PesterExecutionFinalize.Tests.ps1 b/tests/Invoke-PesterExecutionFinalize.Tests.ps1 index 99684c449..c18080bf8 100644 --- a/tests/Invoke-PesterExecutionFinalize.Tests.ps1 +++ b/tests/Invoke-PesterExecutionFinalize.Tests.ps1 @@ -11,15 +11,24 @@ Describe 'Invoke-PesterExecutionFinalize' { $tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("pester-finalize-" + [Guid]::NewGuid().ToString('N')) $resultsDir = Join-Path $tempRoot 'artifacts' $repoResultsDir = Join-Path $tempRoot 'tests/results' + $stepSummaryPath = Join-Path $tempRoot 'step-summary.md' New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null New-Item -ItemType Directory -Path $repoResultsDir -Force | Out-Null try { + $env:GITHUB_STEP_SUMMARY = $stepSummaryPath @( '', '' ) -join [Environment]::NewLine | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-results.xml') -Encoding UTF8 - '[]' | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-failures.json') -Encoding UTF8 + ([pscustomobject]@{ + schema = 'pester-failures@v2' + schemaVersion = '1.1.0' + detailStatus = 'not-applicable' + detailCount = 0 + summary = [pscustomobject]@{ total = 3; failed = 0; errors = 0; skipped = 0 } + results = @() + } | ConvertTo-Json -Depth 6) | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-failures.json') -Encoding UTF8 '{"schema":"pester-leak-report@v1","leakDetected":false}' | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-leak-report.json') -Encoding UTF8 '{"schema":"pester-result-shapes/v1","schemaVersion":"1.1.0","generatedAt":"2026-03-31T00:00:00Z","totalEntries":3,"overall":{"hasPath":3,"hasTags":2},"byType":[]}' | Set-Content -LiteralPath (Join-Path $resultsDir 'result-shapes.json') -Encoding UTF8 'shape text' | Set-Content -LiteralPath (Join-Path $resultsDir 'result-shapes.txt') -Encoding UTF8 @@ -58,12 +67,42 @@ Describe 'Invoke-PesterExecutionFinalize' { procsBefore = @() procsAfter = @() } + leakReportPayload = [ordered]@{ + schema = 'pester-leak-report/v1' + schemaVersion = '1.0.0' + generatedAt = [DateTime]::UtcNow.ToString('o') + targets = @('LVCompare', 'LabVIEW') + graceSeconds = 0 + waitedMs = 0 + procsBefore = @() + procsAfter = @() + runningJobs = @() + allJobs = @() + jobsBefore = @() + leakDetected = $false + actions = @() + killedProcs = @() + stoppedJobs = @() + notes = @('unit-test') + } + publication = [ordered]@{ + disableStepSummary = $false + selectedTests = @('Sample.Tests.ps1', 'Another.Tests.ps1') + discovery = 'manual-scan' + rerunCommand = 'gh workflow run "Validate" -R repo/name' + guard = [ordered]@{ + enabled = $true + heartbeats = 3 + heartbeatPath = 'tests/results/hb.log' + partialLogPath = 'tests/results/pester-partial.log' + } + } includeIntegration = $false integrationMode = 'exclude' integrationSource = 'explicit' summarySchemaVersion = '1.7.1' manifestVersion = '1.0.0' - failuresSchemaVersion = '1.0.0' + failuresSchemaVersion = '1.1.0' leakReportSchemaVersion = '1.0.0' diagnosticsSchemaVersion = '1.1.0' } @@ -79,6 +118,8 @@ Describe 'Invoke-PesterExecutionFinalize' { $manifestPath = Join-Path $resultsDir 'pester-artifacts.json' $sessionIndexPath = Join-Path $resultsDir 'session-index.json' $compareReportPath = Join-Path $resultsDir 'compare-report.html' + $leakReportPath = Join-Path $resultsDir 'pester-leak-report.json' + $publicationReportPath = Join-Path $resultsDir 'pester-execution-publication.json' Test-Path -LiteralPath $summaryPath | Should -BeTrue Test-Path -LiteralPath $summaryJsonPath | Should -BeTrue @@ -87,18 +128,30 @@ Describe 'Invoke-PesterExecutionFinalize' { Test-Path -LiteralPath $indexPath | Should -BeTrue Test-Path -LiteralPath $manifestPath | Should -BeTrue Test-Path -LiteralPath $sessionIndexPath | Should -BeTrue + Test-Path -LiteralPath $leakReportPath | Should -BeTrue + Test-Path -LiteralPath $publicationReportPath | Should -BeTrue + Test-Path -LiteralPath $stepSummaryPath | Should -BeTrue (Get-Content -LiteralPath $summaryPath -Raw) | Should -Match 'Diagnostics Summary' + (Get-Content -LiteralPath $stepSummaryPath -Raw) | Should -Match '## Pester Test Summary' + (Get-Content -LiteralPath $stepSummaryPath -Raw) | Should -Match '### Session' + (Get-Content -LiteralPath $stepSummaryPath -Raw) | Should -Match '### Selected Tests' + (Get-Content -LiteralPath $stepSummaryPath -Raw) | Should -Match 'Sample.Tests.ps1' + (Get-Content -LiteralPath $stepSummaryPath -Raw) | Should -Match '### Diagnostics Summary' $summaryJson = Get-Content -LiteralPath $summaryJsonPath -Raw | ConvertFrom-Json $summaryJson.total | Should -Be 3 $summaryJson.executionPostprocessStatus | Should -Be 'complete' + $summaryJson.failureDetailsStatus | Should -Be 'not-applicable' + $summaryJson.failureDetailsCount | Should -Be 0 $sessionIndex = Get-Content -LiteralPath $sessionIndexPath -Raw | ConvertFrom-Json $sessionIndex.summary.total | Should -Be 3 $sessionIndex.files.pesterSummaryJson | Should -Be 'pester-summary.json' $sessionIndex.files.compareReportHtml | Should -Be 'compare-report.html' $sessionIndex.files.resultsIndexHtml | Should -Be 'results-index.html' + $sessionIndex.stepSummary | Should -Match '### Selected Tests' + $sessionIndex.stepSummary | Should -Match '### Guard' $manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json $artifactFiles = @($manifest.artifacts | ForEach-Object { $_.file }) @@ -107,6 +160,124 @@ Describe 'Invoke-PesterExecutionFinalize' { $artifactFiles | Should -Contain 'session-index.json' $artifactFiles | Should -Contain 'compare-report.html' $artifactFiles | Should -Contain 'results-index.html' + $artifactFiles | Should -Contain 'pester-leak-report.json' + + $publicationReport = Get-Content -LiteralPath $publicationReportPath -Raw | ConvertFrom-Json + $publicationReport.summaryWritten | Should -BeTrue + $publicationReport.sessionSummaryWritten | Should -BeTrue + $publicationReport.metadataWritten | Should -BeTrue + } finally { + Remove-Item Env:GITHUB_STEP_SUMMARY -ErrorAction SilentlyContinue + if (Test-Path -LiteralPath $tempRoot) { + Remove-Item -LiteralPath $tempRoot -Recurse -Force -ErrorAction SilentlyContinue + } + } + } + + It 'accepts a rooted jsonSummaryPath in the finalize context without duplicating resultsDir' { + $tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("pester-finalize-rooted-" + [Guid]::NewGuid().ToString('N')) + $resultsDir = Join-Path $tempRoot 'artifacts' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + + try { + $rootedSummaryPath = Join-Path $resultsDir 'pester-summary.json' + $contextPath = Join-Path $resultsDir 'pester-execution-finalize-context.json' + $context = [ordered]@{ + schema = 'pester-execution-finalize-context@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + repoRoot = $tempRoot + resultsDir = $resultsDir + jsonSummaryPath = $rootedSummaryPath + summaryText = "=== Pester Test Summary ===`nTotal Tests: 1`nPassed: 1`nFailed: 0`nErrors: 0`nSkipped: 0`nDuration: 0.10s" + summaryPayload = [ordered]@{ + total = 1 + passed = 1 + failed = 0 + errors = 0 + skipped = 0 + duration_s = 0.10 + timestamp = '2026-03-31T00:00:00Z' + schemaVersion = '1.7.1' + executionPostprocessStatus = 'complete' + resultsXmlStatus = 'complete' + } + includeIntegration = $false + integrationMode = 'exclude' + integrationSource = 'explicit' + summarySchemaVersion = '1.7.1' + manifestVersion = '1.0.0' + failuresSchemaVersion = '1.1.0' + leakReportSchemaVersion = '1.0.0' + diagnosticsSchemaVersion = '1.1.0' + } + $context | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $contextPath -Encoding UTF8 + + & $toolPath -ContextPath $contextPath | Out-Host + $LASTEXITCODE | Should -Be 0 + Test-Path -LiteralPath $rootedSummaryPath | Should -BeTrue + } finally { + if (Test-Path -LiteralPath $tempRoot) { + Remove-Item -LiteralPath $tempRoot -Recurse -Force -ErrorAction SilentlyContinue + } + } + } + + It 'repairs empty failure detail to explicit unavailable state when summary reports failures' { + $tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("pester-finalize-unavailable-" + [Guid]::NewGuid().ToString('N')) + $resultsDir = Join-Path $tempRoot 'artifacts' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + + try { + @( + '', + '' + ) -join [Environment]::NewLine | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-results.xml') -Encoding UTF8 + '[]' | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-failures.json') -Encoding UTF8 + + $contextPath = Join-Path $resultsDir 'pester-execution-finalize-context.json' + $context = [ordered]@{ + schema = 'pester-execution-finalize-context@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + repoRoot = $tempRoot + resultsDir = $resultsDir + jsonSummaryPath = 'pester-summary.json' + summaryText = "=== Pester Test Summary ===`nTotal Tests: 4`nPassed: 3`nFailed: 1`nErrors: 0`nSkipped: 0`nDuration: 0.50s" + summaryPayload = [ordered]@{ + total = 4 + passed = 3 + failed = 1 + errors = 0 + skipped = 0 + duration_s = 0.50 + timestamp = '2026-03-31T00:00:00Z' + schemaVersion = '1.7.1' + executionPostprocessStatus = 'results-xml-truncated' + resultsXmlStatus = 'truncated-root' + } + includeIntegration = $false + integrationMode = 'exclude' + integrationSource = 'explicit' + summarySchemaVersion = '1.7.1' + manifestVersion = '1.0.0' + failuresSchemaVersion = '1.1.0' + leakReportSchemaVersion = '1.0.0' + diagnosticsSchemaVersion = '1.1.0' + } + $context | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $contextPath -Encoding UTF8 + + & $toolPath -ContextPath $contextPath | Out-Host + $LASTEXITCODE | Should -Be 0 + + $summaryJson = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-summary.json') -Raw | ConvertFrom-Json + $failuresJson = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-failures.json') -Raw | ConvertFrom-Json + + $summaryJson.failureDetailsStatus | Should -Be 'unavailable' + $summaryJson.failureDetailsReason | Should -Be 'results-xml-truncated' + $summaryJson.failureDetailsCount | Should -Be 0 + $failuresJson.detailStatus | Should -Be 'unavailable' + $failuresJson.unavailableReason | Should -Be 'results-xml-truncated' + $failuresJson.detailCount | Should -Be 0 + @($failuresJson.results).Count | Should -Be 0 } finally { if (Test-Path -LiteralPath $tempRoot) { Remove-Item -LiteralPath $tempRoot -Recurse -Force -ErrorAction SilentlyContinue diff --git a/tests/Invoke-PesterExecutionPostprocess.Tests.ps1 b/tests/Invoke-PesterExecutionPostprocess.Tests.ps1 index 906edf106..452f3f807 100644 --- a/tests/Invoke-PesterExecutionPostprocess.Tests.ps1 +++ b/tests/Invoke-PesterExecutionPostprocess.Tests.ps1 @@ -23,6 +23,8 @@ Describe 'Invoke-PesterExecutionPostprocess.ps1' -Tag 'Unit' { duration_s = 1.25 pesterVersion = '5.7.1' includeIntegration = $false + executionPack = 'comparevi' + schemaVersion = '1.7.1' } | ConvertTo-Json | Set-Content -LiteralPath $summaryPath -Encoding UTF8 & $script:postprocessTool -ResultsDir $resultsDir | Out-Null @@ -39,6 +41,8 @@ Describe 'Invoke-PesterExecutionPostprocess.ps1' -Tag 'Unit' { $summary.resultsXmlStatus | Should -Be 'complete' $summary.duration_s | Should -Be 1.25 $summary.pesterVersion | Should -Be '5.7.1' + $summary.executionPack | Should -Be 'comparevi' + $summary.schemaVersion | Should -Be '1.7.1' } It 'writes a repaired machine-readable summary when XML is truncated' { @@ -67,12 +71,53 @@ Describe 'Invoke-PesterExecutionPostprocess.ps1' -Tag 'Unit' { $report.summaryWritten | Should -BeTrue $summary.executionPostprocessStatus | Should -Be 'results-xml-truncated' $summary.resultsXmlStatus | Should -Be 'truncated-root' + $summary.schemaVersion | Should -Be '1.7.1' $summary.total | Should -Be 1033 $summary.failed | Should -Be 156 $summary.errors | Should -Be 0 $summary.passed | Should -Be 877 } + It 'repairs a schema-lite legacy summary when XML is truncated' { + $resultsDir = Join-Path $TestDrive 'legacy-schema-lite' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + $xmlPath = Join-Path $resultsDir 'pester-results.xml' + $summaryPath = Join-Path $resultsDir 'pester-summary.json' + @' + + + + + + + + + Expected 0, but got 1. +'@ | Set-Content -LiteralPath $xmlPath -Encoding UTF8 + @{ + total = 4 + passed = 2 + failed = 1 + errors = 0 + skipped = 1 + timestamp = '2026-03-31T21:26:48.0203952Z' + resultsXmlStatus = 'truncated-root' + } | ConvertTo-Json | Set-Content -LiteralPath $summaryPath -Encoding UTF8 + + & $script:postprocessTool -ResultsDir $resultsDir | Out-Null + + $report = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-execution-postprocess.json') -Raw | ConvertFrom-Json + $summary = Get-Content -LiteralPath $summaryPath -Raw | ConvertFrom-Json + + $report.status | Should -Be 'results-xml-truncated' + $report.summarySchemaStatus | Should -Be 'legacy-schema-lite' + $report.summaryWritten | Should -BeTrue + $report.schemaClassification | Should -Be 'legacy-schema-lite' + $summary.schemaVersion | Should -Be '1.7.1' + $summary.executionPostprocessStatus | Should -Be 'results-xml-truncated' + $summary.resultsXmlStatus | Should -Be 'truncated-root' + } + It 'classifies malformed closed XML with recoverable root attributes as invalid-results-xml' { $resultsDir = Join-Path $TestDrive 'invalid' New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null @@ -96,6 +141,7 @@ Describe 'Invoke-PesterExecutionPostprocess.ps1' -Tag 'Unit' { $report.summaryWritten | Should -BeTrue $summary.executionPostprocessStatus | Should -Be 'invalid-results-xml' $summary.resultsXmlStatus | Should -Be 'invalid-root-attributes' + $summary.schemaVersion | Should -Be '1.7.1' $summary.total | Should -Be 8 $summary.failed | Should -Be 2 $summary.errors | Should -Be 1 @@ -117,4 +163,34 @@ Describe 'Invoke-PesterExecutionPostprocess.ps1' -Tag 'Unit' { $report.summaryWritten | Should -BeFalse (Test-Path -LiteralPath $summaryPath) | Should -BeFalse } + + It 'fails closed with unsupported-schema when an existing summary has an incompatible schemaVersion' { + $resultsDir = Join-Path $TestDrive 'unsupported-summary-schema' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + $xmlPath = Join-Path $resultsDir 'pester-results.xml' + $summaryPath = Join-Path $resultsDir 'pester-summary.json' + @' + + +'@ | Set-Content -LiteralPath $xmlPath -Encoding UTF8 + @{ + schemaVersion = '2.0.0' + total = 99 + failed = 99 + errors = 0 + skipped = 0 + } | ConvertTo-Json | Set-Content -LiteralPath $summaryPath -Encoding UTF8 + + & $script:postprocessTool -ResultsDir $resultsDir | Out-Null + + $report = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-execution-postprocess.json') -Raw | ConvertFrom-Json + $summary = Get-Content -LiteralPath $summaryPath -Raw | ConvertFrom-Json + + $report.status | Should -Be 'unsupported-schema' + $report.summaryWritten | Should -BeFalse + $report.summarySchemaStatus | Should -Be 'unsupported-schema' + $report.summarySchemaReason | Should -Be 'pester-summary-unsupported-schema-version' + $summary.schemaVersion | Should -Be '2.0.0' + ($summary.PSObject.Properties.Name -contains 'resultsXmlStatus') | Should -BeFalse + } } diff --git a/tests/Invoke-PesterExecutionPublication.Tests.ps1 b/tests/Invoke-PesterExecutionPublication.Tests.ps1 new file mode 100644 index 000000000..4f653218d --- /dev/null +++ b/tests/Invoke-PesterExecutionPublication.Tests.ps1 @@ -0,0 +1,99 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Describe 'Invoke-PesterExecutionPublication' { + BeforeAll { + $repoRoot = Split-Path -Parent $PSScriptRoot + $toolPath = Join-Path $repoRoot 'tools/Invoke-PesterExecutionPublication.ps1' + } + + It 'publishes summary, session, metadata, and diagnostics from finalized artifacts' { + $tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("pester-publication-" + [Guid]::NewGuid().ToString('N')) + $resultsDir = Join-Path $tempRoot 'artifacts' + $summaryPath = Join-Path $tempRoot 'step-summary.md' + $contextPath = Join-Path $resultsDir 'pester-execution-finalize-context.json' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + + try { + $env:GITHUB_STEP_SUMMARY = $summaryPath + + ([ordered]@{ + total = 2 + passed = 2 + failed = 0 + errors = 0 + skipped = 0 + duration_s = 1.25 + schemaVersion = '1.7.1' + } | ConvertTo-Json -Depth 6) | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-summary.json') -Encoding UTF8 + + ([ordered]@{ + schema = 'session-index/v1' + schemaVersion = '1.0.0' + status = 'ok' + summary = [ordered]@{ + total = 2 + passed = 2 + failed = 0 + errors = 0 + skipped = 0 + duration_s = 1.25 + } + stepSummary = @( + '### Selected Tests', + '', + '- Sample.Tests.ps1', + '', + '### Configuration', + '', + '- IncludeIntegration: False', + '- Integration Mode: exclude', + '- Integration Source: explicit', + '- Discovery: manual-scan' + ) -join "`n" + } | ConvertTo-Json -Depth 8) | Set-Content -LiteralPath (Join-Path $resultsDir 'session-index.json') -Encoding UTF8 + + ([ordered]@{ + schema = 'pester-result-shapes/v1' + schemaVersion = '1.1.0' + generatedAt = [DateTime]::UtcNow.ToString('o') + totalEntries = 2 + overall = [ordered]@{ hasPath = 2; hasTags = 1 } + byType = @() + } | ConvertTo-Json -Depth 8) | Set-Content -LiteralPath (Join-Path $resultsDir 'result-shapes.json') -Encoding UTF8 + + ([ordered]@{ + schema = 'pester-execution-finalize-context@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + repoRoot = $repoRoot + resultsDir = $resultsDir + publication = [ordered]@{ + disableStepSummary = $false + selectedTests = @('Sample.Tests.ps1') + discovery = 'manual-scan' + rerunCommand = 'gh workflow run "Validate" -R repo/name' + } + } | ConvertTo-Json -Depth 8) | Set-Content -LiteralPath $contextPath -Encoding UTF8 + + & $toolPath -ContextPath $contextPath | Out-Host + $LASTEXITCODE | Should -Be 0 + + Test-Path -LiteralPath $summaryPath | Should -BeTrue + $content = Get-Content -LiteralPath $summaryPath -Raw + $content | Should -Match '## Pester Test Summary' + $content | Should -Match '### Session' + $content | Should -Match '### Selected Tests' + $content | Should -Match '### Diagnostics Summary' + + $report = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-execution-publication.json') -Raw | ConvertFrom-Json + $report.summaryWritten | Should -BeTrue + $report.sessionSummaryWritten | Should -BeTrue + $report.metadataWritten | Should -BeTrue + } finally { + Remove-Item Env:GITHUB_STEP_SUMMARY -ErrorAction SilentlyContinue + if (Test-Path -LiteralPath $tempRoot) { + Remove-Item -LiteralPath $tempRoot -Recurse -Force -ErrorAction SilentlyContinue + } + } + } +} diff --git a/tests/Invoke-PesterExecutionTelemetry.Tests.ps1 b/tests/Invoke-PesterExecutionTelemetry.Tests.ps1 new file mode 100644 index 000000000..bf736df3b --- /dev/null +++ b/tests/Invoke-PesterExecutionTelemetry.Tests.ps1 @@ -0,0 +1,89 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Describe 'Invoke-PesterExecutionTelemetry' { + BeforeAll { + $repoRoot = Split-Path -Parent $PSScriptRoot + $toolPath = Join-Path $repoRoot 'tools/Invoke-PesterExecutionTelemetry.ps1' + } + + It 'materializes a durable telemetry report from dispatcher events and handshake markers' { + $tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("pester-telemetry-" + [Guid]::NewGuid().ToString('N')) + $resultsDir = Join-Path $tempRoot 'artifacts' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + + try { + @( + '{"schema":"comparevi/runtime-event/v1","tsUtc":"2026-03-31T00:00:00Z","source":"pester-dispatcher","phase":"lifecycle","level":"info","message":"Dispatcher session initialized."}', + '{"schema":"comparevi/runtime-event/v1","tsUtc":"2026-03-31T00:05:00Z","source":"pester-dispatcher","phase":"dispatch","level":"info","message":"Running pack."}', + '{"schema":"comparevi/runtime-event/v1","tsUtc":"2026-03-31T00:10:00Z","source":"pester-dispatcher","phase":"postprocess","level":"notice","message":"Summary repair complete."}' + ) | Set-Content -LiteralPath (Join-Path $resultsDir 'dispatcher-events.ndjson') -Encoding UTF8 + + ([ordered]@{ + schema = 'session-index/v1' + executionPack = 'dispatcher' + executionPackSource = 'selection-receipt' + integrationMode = 'exclude' + integrationSource = 'explicit' + } | ConvertTo-Json -Depth 8) | Set-Content -LiteralPath (Join-Path $resultsDir 'session-index.json') -Encoding UTF8 + + ([ordered]@{ + schemaVersion = '1.7.1' + total = 3 + passed = 3 + failed = 0 + } | ConvertTo-Json -Depth 8) | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-summary.json') -Encoding UTF8 + + $handshakeDir = Join-Path $resultsDir 'workflow' + New-Item -ItemType Directory -Path $handshakeDir -Force | Out-Null + ([ordered]@{ + name = 'finalize' + status = 'ok' + atUtc = '2026-03-31T00:11:00Z' + } | ConvertTo-Json -Depth 6) | Set-Content -LiteralPath (Join-Path $handshakeDir 'handshake-finalize.json') -Encoding UTF8 + + & $toolPath -ResultsDir $resultsDir | Out-Host + $LASTEXITCODE | Should -Be 0 + + $reportPath = Join-Path $resultsDir 'pester-execution-telemetry.json' + Test-Path -LiteralPath $reportPath | Should -BeTrue + + $report = Get-Content -LiteralPath $reportPath -Raw | ConvertFrom-Json -Depth 20 + $report.schema | Should -Be 'pester-execution-telemetry@v1' + $report.telemetryStatus | Should -Be 'telemetry-available' + $report.executionPack | Should -Be 'dispatcher' + $report.integrationMode | Should -Be 'exclude' + $report.eventCount | Should -Be 3 + $report.lastKnownPhase | Should -Be 'finalize' + $report.lastKnownPhaseSource | Should -Be 'handshake' + $report.handshake.count | Should -Be 1 + $report.handshake.lastStatus | Should -Be 'ok' + (@($report.phases | ForEach-Object { $_.phase })) | Should -Contain 'dispatch' + $report.lastEvent.phase | Should -Be 'postprocess' + } finally { + if (Test-Path -LiteralPath $tempRoot) { + Remove-Item -LiteralPath $tempRoot -Recurse -Force -ErrorAction SilentlyContinue + } + } + } + + It 'writes a missing telemetry report when dispatcher events are absent' { + $tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("pester-telemetry-empty-" + [Guid]::NewGuid().ToString('N')) + $resultsDir = Join-Path $tempRoot 'artifacts' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + + try { + & $toolPath -ResultsDir $resultsDir | Out-Host + $LASTEXITCODE | Should -Be 0 + + $report = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-execution-telemetry.json') -Raw | ConvertFrom-Json -Depth 20 + $report.telemetryStatus | Should -Be 'telemetry-missing' + $report.eventCount | Should -Be 0 + $report.lastKnownPhase | Should -BeNullOrEmpty + } finally { + if (Test-Path -LiteralPath $tempRoot) { + Remove-Item -LiteralPath $tempRoot -Recurse -Force -ErrorAction SilentlyContinue + } + } + } +} diff --git a/tests/Invoke-PesterOperatorOutcome.Tests.ps1 b/tests/Invoke-PesterOperatorOutcome.Tests.ps1 new file mode 100644 index 000000000..910dce829 --- /dev/null +++ b/tests/Invoke-PesterOperatorOutcome.Tests.ps1 @@ -0,0 +1,70 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Describe 'Invoke-PesterOperatorOutcome' -Tag 'Unit' { + BeforeAll { + $script:repoRoot = Split-Path -Parent $PSScriptRoot + $script:toolPath = Join-Path $script:repoRoot 'tools/Invoke-PesterOperatorOutcome.ps1' + } + + It 'writes a passing operator outcome when classification is ok' { + $resultsDir = Join-Path $TestDrive 'ok' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + ([ordered]@{ + schema = 'pester-evidence-classification@v1' + classification = 'ok' + reasons = @() + contextStatus = 'ready' + readinessStatus = 'ready' + selectionStatus = 'ready' + rawArtifactDownload = 'staged' + dispatcherExitCode = 0 + selectionExecutionPack = 'dispatcher' + } | ConvertTo-Json -Depth 6) | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-evidence-classification.json') -Encoding UTF8 + ([ordered]@{ + schemaVersion = '1.7.1' + total = 3 + passed = 3 + failed = 0 + errors = 0 + } | ConvertTo-Json -Depth 6) | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-summary.json') -Encoding UTF8 + + & $script:toolPath -ResultsDir $resultsDir | Out-Null + + $outcome = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-operator-outcome.json') -Raw | ConvertFrom-Json + $outcome.gateStatus | Should -Be 'pass' + $outcome.classification | Should -Be 'ok' + $outcome.nextActionId | Should -Be 'no-action' + } + + It 'writes fail-closed operator guidance for unsupported schema' { + $resultsDir = Join-Path $TestDrive 'unsupported-schema' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + ([ordered]@{ + schema = 'pester-evidence-classification@v1' + classification = 'unsupported-schema' + reasons = @('execution-receipt-unsupported-schema') + contextStatus = 'ready' + readinessStatus = 'ready' + selectionStatus = 'ready' + rawArtifactDownload = 'staged' + dispatcherExitCode = -1 + selectionExecutionPack = 'dispatcher' + } | ConvertTo-Json -Depth 6) | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-evidence-classification.json') -Encoding UTF8 + ([ordered]@{ + schemaVersion = '1.7.1' + total = 0 + passed = 0 + failed = 0 + errors = 0 + } | ConvertTo-Json -Depth 6) | Set-Content -LiteralPath (Join-Path $resultsDir 'pester-summary.json') -Encoding UTF8 + + & $script:toolPath -ResultsDir $resultsDir -ContinueOnError false | Out-Null + + $outcome = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-operator-outcome.json') -Raw | ConvertFrom-Json + $outcome.gateStatus | Should -Be 'fail' + $outcome.classification | Should -Be 'unsupported-schema' + $outcome.nextActionId | Should -Be 'reconcile-schema-contract' + $outcome.reasons | Should -Contain 'execution-receipt-unsupported-schema' + } +} diff --git a/tests/Invoke-PesterWindowsContainerSurfaceProbe.Tests.ps1 b/tests/Invoke-PesterWindowsContainerSurfaceProbe.Tests.ps1 new file mode 100644 index 000000000..1224fdc2e --- /dev/null +++ b/tests/Invoke-PesterWindowsContainerSurfaceProbe.Tests.ps1 @@ -0,0 +1,48 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Describe 'Invoke-PesterWindowsContainerSurfaceProbe' -Tag 'Unit' { + BeforeAll { + $script:repoRoot = Split-Path -Parent $PSScriptRoot + $script:toolPath = Join-Path $script:repoRoot 'tools/Invoke-PesterWindowsContainerSurfaceProbe.ps1' + } + + It 'records not-windows-host explicitly when the current host is not Windows' { + $resultsDir = Join-Path $TestDrive 'linux' + + & $script:toolPath -ResultsDir $resultsDir -HostPlatformOverride 'Unix' | Out-Host + $LASTEXITCODE | Should -Be 0 + + $receipt = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-windows-container-surface.json') -Raw | ConvertFrom-Json + $receipt.status | Should -Be 'not-windows-host' + $receipt.reason | Should -Be 'surface-requires-windows-host' + $receipt.recommendedCommands | Should -Contain 'npm run compare:docker:ni:windows:probe' + } + + It 'records ready when Docker reports a Windows engine and the pinned NI image is available' { + $resultsDir = Join-Path $TestDrive 'ready' + $server = '{"Os":"windows","Version":"27.5.1","Platform":{"Name":"Docker Desktop 4.43.0"}}' + $image = '{"Id":"sha256:1234","RepoTags":["nationalinstruments/labview:2026q1-windows"]}' + + & $script:toolPath -ResultsDir $resultsDir -HostPlatformOverride 'Win32NT' -DockerServerJson $server -ImageInspectJson $image | Out-Host + $LASTEXITCODE | Should -Be 0 + + $receipt = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-windows-container-surface.json') -Raw | ConvertFrom-Json + $receipt.status | Should -Be 'ready' + $receipt.reason | Should -Be 'windows-container-surface-ready' + $receipt.pinnedImagePresent | Should -BeTrue + } + + It 'records docker-engine-not-windows when the Docker server resolves to Linux' { + $resultsDir = Join-Path $TestDrive 'linux-engine' + $server = '{"Os":"linux","Version":"27.5.1","Platform":{"Name":"Docker Desktop 4.43.0"}}' + + & $script:toolPath -ResultsDir $resultsDir -HostPlatformOverride 'Win32NT' -DockerServerJson $server | Out-Host + $LASTEXITCODE | Should -Be 0 + + $receipt = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-windows-container-surface.json') -Raw | ConvertFrom-Json + $receipt.status | Should -Be 'docker-engine-not-windows' + $receipt.reason | Should -Be 'docker-server-not-windows' + $receipt.pinnedImagePresent | Should -BeFalse + } +} diff --git a/tests/PesterExecutionPacks.Tests.ps1 b/tests/PesterExecutionPacks.Tests.ps1 new file mode 100644 index 000000000..72412a334 --- /dev/null +++ b/tests/PesterExecutionPacks.Tests.ps1 @@ -0,0 +1,32 @@ +Describe 'Pester execution pack resolution' -Tag 'Unit' { + BeforeAll { + . (Join-Path (Join-Path $PSScriptRoot '..') 'tools/PesterExecutionPacks.ps1') + } + + It 'resolves a named execution pack with canonical identity and base patterns' { + $resolved = Resolve-PesterExecutionPack -ExecutionPack comparevi + + $resolved.executionPack | Should -Be 'comparevi' + $resolved.executionPackSource | Should -Be 'declared' + @($resolved.baseIncludePatterns) | Should -Contain 'CompareVI*.ps1' + @($resolved.effectiveIncludePatterns) | Should -Contain 'CompareVI*.ps1' + @($resolved.refineIncludePatterns) | Should -Be @() + } + + It 'treats omitted pack selection as the full pack and only keeps refinements as refinements' { + $resolved = Resolve-PesterExecutionPack -ExecutionPack '' -RefineIncludePatterns @('tests/Alpha.Unit.Tests.ps1', 'Alpha.Unit.Tests.ps1') + + $resolved.executionPack | Should -Be 'full' + $resolved.executionPackSource | Should -Be 'default' + @($resolved.baseIncludePatterns) | Should -Be @() + @($resolved.refineIncludePatterns) | Should -Be @('tests/Alpha.Unit.Tests.ps1', 'Alpha.Unit.Tests.ps1') + @($resolved.effectiveIncludePatterns) | Should -Be @('tests/Alpha.Unit.Tests.ps1', 'Alpha.Unit.Tests.ps1') + } + + It 'accepts legacy aliases but returns canonical pack names' { + $resolved = Resolve-PesterExecutionPack -ExecutionPack summary + + $resolved.executionPack | Should -Be 'psummary' + @($resolved.baseIncludePatterns) | Should -Contain 'PesterSummary*.ps1' + } +} diff --git a/tests/PesterFailurePayloadShape.Tests.ps1 b/tests/PesterFailurePayloadShape.Tests.ps1 new file mode 100644 index 000000000..28baa962d --- /dev/null +++ b/tests/PesterFailurePayloadShape.Tests.ps1 @@ -0,0 +1,169 @@ +Describe 'Pester failure payload shape compatibility' -Tag 'Unit' { + BeforeAll { + $writeScript = Join-Path (Join-Path $PSScriptRoot '..') 'tools/Write-PesterTopFailures.ps1' + $printScript = Join-Path (Join-Path $PSScriptRoot '..') 'tools/Print-PesterTopFailures.ps1' + } + + It 'Write-PesterTopFailures accepts object results payloads' { + $resultsDir = Join-Path $TestDrive 'write-object' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + $payload = [pscustomobject]@{ + results = @( + [pscustomobject]@{ + name = 'Object.Shape.Failure' + result = 'Failed' + message = 'object payload failure' + } + ) + } | ConvertTo-Json -Depth 6 + Set-Content -LiteralPath (Join-Path $resultsDir 'pester-failures.json') -Value $payload -Encoding UTF8 + $env:GITHUB_STEP_SUMMARY = Join-Path $TestDrive 'write-object-summary.md' + + & $writeScript -ResultsDir $resultsDir -Top 5 + + $content = Get-Content -LiteralPath $env:GITHUB_STEP_SUMMARY -Raw + $content | Should -Match 'Object.Shape.Failure' + } + + It 'Write-PesterTopFailures accepts array payloads' { + $resultsDir = Join-Path $TestDrive 'write-array' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + $payload = @( + [pscustomobject]@{ + name = 'Array.Shape.Failure' + result = 'Failed' + message = 'array payload failure' + } + ) | ConvertTo-Json -Depth 6 + Set-Content -LiteralPath (Join-Path $resultsDir 'pester-failures.json') -Value $payload -Encoding UTF8 + $env:GITHUB_STEP_SUMMARY = Join-Path $TestDrive 'write-array-summary.md' + + & $writeScript -ResultsDir $resultsDir -Top 5 + + $content = Get-Content -LiteralPath $env:GITHUB_STEP_SUMMARY -Raw + $content | Should -Match 'Array.Shape.Failure' + } + + It 'Write-PesterTopFailures reports unavailable details when summary shows failures but payload is empty' { + $resultsDir = Join-Path $TestDrive 'write-unavailable' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + Set-Content -LiteralPath (Join-Path $resultsDir 'pester-failures.json') -Value '[]' -Encoding UTF8 + $summary = [pscustomobject]@{ + total = 4 + passed = 3 + failed = 1 + errors = 0 + resultsXmlStatus = 'truncated-root' + } | ConvertTo-Json -Depth 6 + Set-Content -LiteralPath (Join-Path $resultsDir 'pester-summary.json') -Value $summary -Encoding UTF8 + $env:GITHUB_STEP_SUMMARY = Join-Path $TestDrive 'write-unavailable-summary.md' + + & $writeScript -ResultsDir $resultsDir -Top 5 + + $content = Get-Content -LiteralPath $env:GITHUB_STEP_SUMMARY -Raw + $content | Should -Match 'failure details unavailable' + $content | Should -Match 'resultsXmlStatus=truncated-root' + } + + It 'Write-PesterTopFailures prefers explicit unavailable-detail reason from canonical payload' { + $resultsDir = Join-Path $TestDrive 'write-explicit-unavailable' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + $payload = [pscustomobject]@{ + schema = 'pester-failures@v2' + schemaVersion = '1.1.0' + detailStatus = 'unavailable' + unavailableReason = 'failure-payload-unparseable' + detailCount = 0 + summary = [pscustomobject]@{ + total = 4 + failed = 1 + errors = 0 + skipped = 0 + } + results = @() + } | ConvertTo-Json -Depth 6 + Set-Content -LiteralPath (Join-Path $resultsDir 'pester-failures.json') -Value $payload -Encoding UTF8 + $summary = [pscustomobject]@{ + total = 4 + passed = 3 + failed = 1 + errors = 0 + failureDetailsStatus = 'unavailable' + failureDetailsReason = 'failure-payload-unparseable' + } | ConvertTo-Json -Depth 6 + Set-Content -LiteralPath (Join-Path $resultsDir 'pester-summary.json') -Value $summary -Encoding UTF8 + $env:GITHUB_STEP_SUMMARY = Join-Path $TestDrive 'write-explicit-unavailable-summary.md' + + & $writeScript -ResultsDir $resultsDir -Top 5 + + $content = Get-Content -LiteralPath $env:GITHUB_STEP_SUMMARY -Raw + $content | Should -Match 'reason=failure-payload-unparseable' + } + + It 'Write-PesterTopFailures appends operator-outcome next action when present' { + $resultsDir = Join-Path $TestDrive 'write-operator-outcome' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + $payload = [pscustomobject]@{ + results = @( + [pscustomobject]@{ + name = 'Outcome.Linked.Failure' + result = 'Failed' + message = 'outcome-linked failure' + } + ) + } | ConvertTo-Json -Depth 6 + Set-Content -LiteralPath (Join-Path $resultsDir 'pester-failures.json') -Value $payload -Encoding UTF8 + $outcome = [pscustomobject]@{ + schema = 'pester-operator-outcome@v1' + gateStatus = 'fail' + classification = 'test-failures' + nextAction = 'Review pester-failures.json, the top-failures summary, and the failing test names before deciding whether to rerun or fix source.' + } | ConvertTo-Json -Depth 6 + Set-Content -LiteralPath (Join-Path $resultsDir 'pester-operator-outcome.json') -Value $outcome -Encoding UTF8 + $env:GITHUB_STEP_SUMMARY = Join-Path $TestDrive 'write-operator-outcome-summary.md' + + & $writeScript -ResultsDir $resultsDir -Top 5 + + $content = Get-Content -LiteralPath $env:GITHUB_STEP_SUMMARY -Raw + $content | Should -Match 'gate outcome: test-failures \(fail\)' + $content | Should -Match 'next action: Review pester-failures.json' + } + + It 'Print-PesterTopFailures returns items from object results payloads' { + $resultsDir = Join-Path $TestDrive 'print-object' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + $payload = [pscustomobject]@{ + results = @( + [pscustomobject]@{ + name = 'Object.Print.Failure' + result = 'Failed' + message = 'object print failure' + } + ) + } | ConvertTo-Json -Depth 6 + Set-Content -LiteralPath (Join-Path $resultsDir 'pester-failures.json') -Value $payload -Encoding UTF8 + + $items = & $printScript -ResultsDir $resultsDir -Top 5 -ConsoleLevel quiet -PassThru + + @($items).Count | Should -Be 1 + $items[0].name | Should -Be 'Object.Print.Failure' + } + + It 'Print-PesterTopFailures returns items from array payloads' { + $resultsDir = Join-Path $TestDrive 'print-array' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + $payload = @( + [pscustomobject]@{ + name = 'Array.Print.Failure' + result = 'Failed' + message = 'array print failure' + } + ) | ConvertTo-Json -Depth 6 + Set-Content -LiteralPath (Join-Path $resultsDir 'pester-failures.json') -Value $payload -Encoding UTF8 + + $items = & $printScript -ResultsDir $resultsDir -Top 5 -ConsoleLevel quiet -PassThru + + @($items).Count | Should -Be 1 + $items[0].name | Should -Be 'Array.Print.Failure' + } +} diff --git a/tests/PesterFailureProducerConsistency.Tests.ps1 b/tests/PesterFailureProducerConsistency.Tests.ps1 new file mode 100644 index 000000000..6d0fee797 --- /dev/null +++ b/tests/PesterFailureProducerConsistency.Tests.ps1 @@ -0,0 +1,62 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Describe 'Pester failure producer consistency' -Tag 'Unit' { + BeforeAll { + $script:repoRoot = Split-Path -Parent $PSScriptRoot + $script:dispatcherPath = Join-Path $script:repoRoot 'Invoke-PesterTests.ps1' + } + + It 'dispatcher emits canonical failure detail when a test fails' { + $tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("pester-failure-producer-" + [Guid]::NewGuid().ToString('N')) + $testsDir = Join-Path $tempRoot 'tests' + $resultsDir = Join-Path $tempRoot 'results' + New-Item -ItemType Directory -Path $testsDir -Force | Out-Null + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + + try { + @' +Describe "Producer consistency sample" { + It "fails intentionally" { + 1 | Should -Be 2 + } +} +'@ | Set-Content -LiteralPath (Join-Path $testsDir 'ProducerConsistency.Sample.Tests.ps1') -Encoding UTF8 + + $dispatcherError = $null + try { + & $script:dispatcherPath ` + -TestsPath $testsDir ` + -ResultsPath $resultsDir ` + -JsonSummaryPath 'pester-summary.json' ` + -ExecutionPack full ` + -IntegrationMode exclude ` + -IncludePatterns 'ProducerConsistency.Sample.Tests.ps1' ` + -EmitFailuresJsonAlways | Out-Null + } catch { + $dispatcherError = $_ + } + + $dispatcherError | Should -Not -BeNullOrEmpty + $dispatcherError.Exception.Message | Should -Match 'Test execution completed with failures' + + $summary = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-summary.json') -Raw | ConvertFrom-Json + $failures = Get-Content -LiteralPath (Join-Path $resultsDir 'pester-failures.json') -Raw | ConvertFrom-Json + + ($summary.failed + $summary.errors) | Should -BeGreaterThan 0 + $summary.failureDetailsStatus | Should -Be 'available' + $summary.failureDetailsCount | Should -BeGreaterThan 0 + $failures.schema | Should -Be 'pester-failures@v2' + $failures.schemaVersion | Should -Be '1.1.0' + $failures.detailStatus | Should -Be 'available' + $failures.detailCount | Should -BeGreaterThan 0 + @($failures.results).Count | Should -BeGreaterThan 0 + $failures.results[0].result | Should -Be 'Failed' + $failures.results[0].name | Should -Match 'fails intentionally' + } finally { + if (Test-Path -LiteralPath $tempRoot) { + Remove-Item -LiteralPath $tempRoot -Recurse -Force -ErrorAction SilentlyContinue + } + } + } +} diff --git a/tests/PesterPathHygiene.Tests.ps1 b/tests/PesterPathHygiene.Tests.ps1 new file mode 100644 index 000000000..81a2fb6f5 --- /dev/null +++ b/tests/PesterPathHygiene.Tests.ps1 @@ -0,0 +1,44 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Describe 'Pester path hygiene helper' -Tag 'Unit' { + BeforeAll { + $script:repoRoot = Split-Path -Parent $PSScriptRoot + . (Join-Path $script:repoRoot 'tools/PesterPathHygiene.ps1') + } + + It 'detects OneDrive-like managed roots as path-hygiene risks' { + $riskyPath = Join-Path $TestDrive 'OneDrive - Contoso/results' + + $risks = @(Get-PesterPathHygieneRisks -PathValue $riskyPath) + + $risks.Count | Should -BeGreaterThan 0 + $risks[0].id | Should -Be 'onedrive-managed-root' + } + + It 'relocates risky results and session-lock roots into a safe root' { + $riskyResults = Join-Path $TestDrive 'OneDrive - Contoso/results' + $riskyLocks = Join-Path $TestDrive 'OneDrive - Contoso/locks' + $safeRoot = Join-Path $TestDrive 'safe-root' + + $plan = Resolve-PesterPathHygienePlan -ResultsPath $riskyResults -SessionLockRoot $riskyLocks -Mode relocate -SafeRoot $safeRoot + + $plan.status | Should -Be 'relocated' + $plan.effectiveResultsPath | Should -Be ([System.IO.Path]::GetFullPath((Join-Path $safeRoot 'results'))) + $plan.effectiveSessionLockRoot | Should -Be ([System.IO.Path]::GetFullPath((Join-Path $safeRoot 'session-lock'))) + @($plan.risks).Count | Should -BeGreaterThan 0 + } + + It 'blocks risky results and session-lock roots when block mode is requested' { + $riskyResults = Join-Path $TestDrive 'OneDrive - Contoso/results' + $riskyLocks = Join-Path $TestDrive 'OneDrive - Contoso/locks' + $safeRoot = Join-Path $TestDrive 'safe-root' + + $plan = Resolve-PesterPathHygienePlan -ResultsPath $riskyResults -SessionLockRoot $riskyLocks -Mode block -SafeRoot $safeRoot + + $plan.status | Should -Be 'path-hygiene-blocked' + $plan.receiptRoot | Should -Be ([System.IO.Path]::GetFullPath((Join-Path $safeRoot 'blocked-results'))) + $plan.effectiveSessionLockRoot | Should -Be ([System.IO.Path]::GetFullPath((Join-Path $safeRoot 'blocked-session-lock'))) + @($plan.risks | ForEach-Object { $_.id }) | Should -Contain 'onedrive-managed-root' + } +} diff --git a/tests/PesterServiceModelSchema.Tests.ps1 b/tests/PesterServiceModelSchema.Tests.ps1 new file mode 100644 index 000000000..f222f0a4c --- /dev/null +++ b/tests/PesterServiceModelSchema.Tests.ps1 @@ -0,0 +1,47 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Describe 'PesterServiceModelSchema' -Tag 'Unit' { + BeforeAll { + $script:repoRoot = Split-Path -Parent $PSScriptRoot + . (Join-Path $script:repoRoot 'tools/PesterServiceModelSchema.ps1') + } + + It 'accepts the supported execution receipt schema contract' { + $receiptPath = Join-Path $TestDrive 'pester-run-receipt.json' + ([ordered]@{ + schema = 'pester-execution-receipt@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + status = 'completed' + } | ConvertTo-Json -Depth 4) | Set-Content -LiteralPath $receiptPath -Encoding UTF8 + + $state = Test-PesterServiceModelSchemaContract ` + -DocumentState (Read-PesterServiceModelJsonDocument -PathValue $receiptPath -ContractName 'execution-receipt') ` + -ExpectedSchema 'pester-execution-receipt@v1' + + $state.valid | Should -BeTrue + $state.classification | Should -Be 'ok' + $state.reason | Should -Be 'execution-receipt-ok' + } + + It 'rejects an incompatible summary schemaVersion major explicitly' { + $summaryPath = Join-Path $TestDrive 'pester-summary.json' + ([ordered]@{ + schemaVersion = '2.0.0' + total = 1 + passed = 1 + failed = 0 + errors = 0 + skipped = 0 + } | ConvertTo-Json -Depth 4) | Set-Content -LiteralPath $summaryPath -Encoding UTF8 + + $state = Test-PesterServiceModelSchemaContract ` + -DocumentState (Read-PesterServiceModelJsonDocument -PathValue $summaryPath -ContractName 'pester-summary') ` + -ExpectedSchemaVersionMajor 1 ` + -RequireSchemaVersion + + $state.valid | Should -BeFalse + $state.classification | Should -Be 'unsupported-schema' + $state.reason | Should -Be 'pester-summary-unsupported-schema-version' + } +} diff --git a/tests/PesterSummary.Context.Tests.ps1 b/tests/PesterSummary.Context.Tests.ps1 index 9db61a3c1..7395d8c65 100644 --- a/tests/PesterSummary.Context.Tests.ps1 +++ b/tests/PesterSummary.Context.Tests.ps1 @@ -11,12 +11,12 @@ Describe 'Pester Summary Context Emission' { New-Item -ItemType Directory -Path $tempDir | Out-Null $testsDir = Join-Path $tempDir 'tests' New-Item -ItemType Directory -Path $testsDir | Out-Null - Set-Content -LiteralPath (Join-Path $testsDir 'Mini.Tests.ps1') -Value "Describe 'Mini' { It 'passes' { 1 | Should -Be 1 } }" -Encoding UTF8 + Set-Content -LiteralPath (Join-Path $testsDir 'Invoke-PesterTests.Sample.Tests.ps1') -Value "Describe 'Mini' { It 'passes' { 1 | Should -Be 1 } }" -Encoding UTF8 Push-Location $root try { $resDir = Join-Path $tempDir 'results' - & pwsh -NoLogo -NoProfile -File $dispatcher -TestsPath $testsDir -ResultsPath $resDir -EmitContext | Out-Null + & pwsh -NoLogo -NoProfile -File $dispatcher -TestsPath $testsDir -ResultsPath $resDir -EmitContext -ExecutionPack dispatcher | Out-Null $summaryPath = Join-Path $resDir 'pester-summary.json' Test-Path $summaryPath | Should -BeTrue $json = Get-Content -LiteralPath $summaryPath -Raw | ConvertFrom-Json @@ -32,6 +32,9 @@ Describe 'Pester Summary Context Emission' { $json.selection.totalDiscoveredFileCount | Should -BeGreaterThan 0 $json.selection.selectedTestFileCount | Should -BeGreaterThan 0 $json.selection.maxTestFilesApplied | Should -BeFalse + $json.selection.executionPack | Should -Be 'dispatcher' + $json.selection.executionPackSource | Should -Be 'declared' + @($json.selection.effectiveIncludePatterns) | Should -Contain 'Invoke-PesterTests*.ps1' } finally { Pop-Location Remove-Item -Recurse -Force $tempDir -ErrorAction SilentlyContinue diff --git a/tests/Replay-PesterServiceModelArtifacts.Local.Tests.ps1 b/tests/Replay-PesterServiceModelArtifacts.Local.Tests.ps1 new file mode 100644 index 000000000..971ba939c --- /dev/null +++ b/tests/Replay-PesterServiceModelArtifacts.Local.Tests.ps1 @@ -0,0 +1,132 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Describe 'Replay-PesterServiceModelArtifacts.Local' -Tag 'Execution' { + BeforeAll { + $script:repoRoot = Split-Path -Parent $PSScriptRoot + $script:toolPath = Join-Path $script:repoRoot 'tools/Replay-PesterServiceModelArtifacts.Local.ps1' + } + + It 'rebuilds postprocess, summary, totals, session index, and evidence from retained artifacts' { + $tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("pester-replay-local-" + [Guid]::NewGuid().ToString('N')) + $rawArtifactDir = Join-Path $tempRoot 'raw-artifact' + $workspaceDir = Join-Path $tempRoot 'workspace-results' + $receiptPath = Join-Path $tempRoot 'pester-run-receipt.json' + New-Item -ItemType Directory -Path $rawArtifactDir -Force | Out-Null + + try { + @( + '', + '' + ) -join [Environment]::NewLine | Set-Content -LiteralPath (Join-Path $rawArtifactDir 'pester-results.xml') -Encoding UTF8 + @( + '{"schema":"comparevi/runtime-event/v1","tsUtc":"2026-03-31T00:00:00Z","source":"pester-dispatcher","phase":"lifecycle","level":"info","message":"Dispatcher session initialized."}', + '{"schema":"comparevi/runtime-event/v1","tsUtc":"2026-03-31T00:03:00Z","source":"pester-dispatcher","phase":"dispatch","level":"info","message":"Pack execution complete."}' + ) | Set-Content -LiteralPath (Join-Path $rawArtifactDir 'dispatcher-events.ndjson') -Encoding UTF8 + + ([ordered]@{ + schema = 'pester-failures@v2' + schemaVersion = '1.1.0' + detailStatus = 'not-applicable' + detailCount = 0 + summary = [ordered]@{ total = 2; failed = 0; errors = 0; skipped = 0 } + results = @() + } | ConvertTo-Json -Depth 6) | Set-Content -LiteralPath (Join-Path $rawArtifactDir 'pester-failures.json') -Encoding UTF8 + + ([ordered]@{ + schema = 'pester-execution-receipt@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + repository = 'LabVIEW-Community-CI-CD/compare-vi-cli-action' + contextStatus = 'ready' + readinessStatus = 'ready' + selectionStatus = 'ready' + selectionExecutionPack = 'dispatcher' + selectionExecutionPackSource = 'declared' + dispatcherExitCode = 0 + executionJobResult = 'success' + status = 'completed' + } | ConvertTo-Json -Depth 6) | Set-Content -LiteralPath $receiptPath -Encoding UTF8 + + & $script:toolPath -RawArtifactDir $rawArtifactDir -ExecutionReceiptPath $receiptPath -WorkspaceResultsDir $workspaceDir | Out-Host + $LASTEXITCODE | Should -Be 0 + + Test-Path -LiteralPath (Join-Path $workspaceDir 'pester-execution-postprocess.json') -PathType Leaf | Should -BeTrue + Test-Path -LiteralPath (Join-Path $workspaceDir 'pester-summary.json') -PathType Leaf | Should -BeTrue + Test-Path -LiteralPath (Join-Path $workspaceDir 'pester-totals.json') -PathType Leaf | Should -BeTrue + Test-Path -LiteralPath (Join-Path $workspaceDir 'pester-execution-telemetry.json') -PathType Leaf | Should -BeTrue + Test-Path -LiteralPath (Join-Path $workspaceDir 'session-index.json') -PathType Leaf | Should -BeTrue + Test-Path -LiteralPath (Join-Path $workspaceDir 'pester-evidence-classification.json') -PathType Leaf | Should -BeTrue + Test-Path -LiteralPath (Join-Path $workspaceDir 'pester-operator-outcome.json') -PathType Leaf | Should -BeTrue + Test-Path -LiteralPath (Join-Path $workspaceDir 'pester-evidence-provenance.json') -PathType Leaf | Should -BeTrue + Test-Path -LiteralPath (Join-Path $workspaceDir 'pester-local-replay-receipt.json') -PathType Leaf | Should -BeTrue + + $classification = Get-Content -LiteralPath (Join-Path $workspaceDir 'pester-evidence-classification.json') -Raw | ConvertFrom-Json + $classification.classification | Should -Be 'ok' + $operatorOutcome = Get-Content -LiteralPath (Join-Path $workspaceDir 'pester-operator-outcome.json') -Raw | ConvertFrom-Json + $operatorOutcome.gateStatus | Should -Be 'pass' + $operatorOutcome.classification | Should -Be 'ok' + $provenance = Get-Content -LiteralPath (Join-Path $workspaceDir 'pester-evidence-provenance.json') -Raw | ConvertFrom-Json + $provenance.provenanceKind | Should -Be 'local-replay' + ($provenance.sourceInputs | Where-Object role -eq 'source-raw-artifacts').fileCount | Should -Be 3 + + $replayReceipt = Get-Content -LiteralPath (Join-Path $workspaceDir 'pester-local-replay-receipt.json') -Raw | ConvertFrom-Json + $replayReceipt.classification | Should -Be 'ok' + $replayReceipt.operatorOutcomePresent | Should -BeTrue + $replayReceipt.operatorOutcomeGateStatus | Should -Be 'pass' + $replayReceipt.provenancePresent | Should -BeTrue + $replayReceipt.provenanceKind | Should -Be 'local-replay' + $replayReceipt.telemetryPresent | Should -BeTrue + $replayReceipt.telemetryStatus | Should -Be 'telemetry-available' + $replayReceipt.telemetryEventCount | Should -Be 2 + $replayReceipt.telemetryLastKnownPhase | Should -Be 'dispatch' + $replayReceipt.workspaceResultsDir | Should -Be ([System.IO.Path]::GetFullPath($workspaceDir)) + } finally { + if (Test-Path -LiteralPath $tempRoot) { + Remove-Item -LiteralPath $tempRoot -Recurse -Force -ErrorAction SilentlyContinue + } + } + } + + It 'fails closed with unsupported-schema when the retained execution receipt is incompatible' { + $tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("pester-replay-local-" + [Guid]::NewGuid().ToString('N')) + $rawArtifactDir = Join-Path $tempRoot 'raw-artifact' + $workspaceDir = Join-Path $tempRoot 'workspace-results' + $receiptPath = Join-Path $tempRoot 'pester-run-receipt.json' + New-Item -ItemType Directory -Path $rawArtifactDir -Force | Out-Null + + try { + @( + '', + '' + ) -join [Environment]::NewLine | Set-Content -LiteralPath (Join-Path $rawArtifactDir 'pester-results.xml') -Encoding UTF8 + + ([ordered]@{ + schema = 'pester-execution-receipt@v2' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + status = 'completed' + } | ConvertTo-Json -Depth 6) | Set-Content -LiteralPath $receiptPath -Encoding UTF8 + + & $script:toolPath -RawArtifactDir $rawArtifactDir -ExecutionReceiptPath $receiptPath -WorkspaceResultsDir $workspaceDir -SkipSessionIndex | Out-Host + $LASTEXITCODE | Should -Be 0 + + $classification = Get-Content -LiteralPath (Join-Path $workspaceDir 'pester-evidence-classification.json') -Raw | ConvertFrom-Json + $classification.classification | Should -Be 'unsupported-schema' + $operatorOutcome = Get-Content -LiteralPath (Join-Path $workspaceDir 'pester-operator-outcome.json') -Raw | ConvertFrom-Json + $operatorOutcome.gateStatus | Should -Be 'fail' + $operatorOutcome.nextActionId | Should -Be 'reconcile-schema-contract' + + $replayReceipt = Get-Content -LiteralPath (Join-Path $workspaceDir 'pester-local-replay-receipt.json') -Raw | ConvertFrom-Json + $replayReceipt.classification | Should -Be 'unsupported-schema' + $replayReceipt.operatorOutcomePresent | Should -BeTrue + $replayReceipt.operatorOutcomeGateStatus | Should -Be 'fail' + $replayReceipt.provenancePresent | Should -BeTrue + $replayReceipt.provenanceKind | Should -Be 'local-replay' + $replayReceipt.stagedExecutionReceiptSchemaStatus | Should -Be 'unsupported-schema' + $replayReceipt.stagedExecutionReceiptSchemaReason | Should -Be 'execution-receipt-unsupported-schema' + } finally { + if (Test-Path -LiteralPath $tempRoot) { + Remove-Item -LiteralPath $tempRoot -Recurse -Force -ErrorAction SilentlyContinue + } + } + } +} diff --git a/tests/Replay-PesterServiceModelRepresentativeArtifact.Tests.ps1 b/tests/Replay-PesterServiceModelRepresentativeArtifact.Tests.ps1 new file mode 100644 index 000000000..dfd02e04b --- /dev/null +++ b/tests/Replay-PesterServiceModelRepresentativeArtifact.Tests.ps1 @@ -0,0 +1,39 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Describe 'Representative retained-artifact replay' -Tag 'Execution' { + BeforeAll { + $script:repoRoot = Split-Path -Parent $PSScriptRoot + $script:toolPath = Join-Path $script:repoRoot 'tools/Replay-PesterServiceModelArtifacts.Local.ps1' + $script:fixtureRoot = Join-Path $script:repoRoot 'tests/fixtures/pester-service-model/legacy-results-xml-truncated' + } + + It 'replays a schema-lite truncated-XML run without throwing and preserves the real evidence classification' { + $workspaceDir = Join-Path $TestDrive 'workspace-results' + $rawArtifactDir = Join-Path $script:fixtureRoot 'raw' + $receiptPath = Join-Path $script:fixtureRoot 'pester-run-receipt.json' + + & $script:toolPath -RawArtifactDir $rawArtifactDir -ExecutionReceiptPath $receiptPath -WorkspaceResultsDir $workspaceDir | Out-Host + $LASTEXITCODE | Should -Be 0 + + $classification = Get-Content -LiteralPath (Join-Path $workspaceDir 'pester-evidence-classification.json') -Raw | ConvertFrom-Json + $classification.classification | Should -Be 'results-xml-truncated' + $classification.selectionExecutionPack | Should -Be '' + $classification.summarySchemaStatus | Should -Be 'ok' + + $operatorOutcome = Get-Content -LiteralPath (Join-Path $workspaceDir 'pester-operator-outcome.json') -Raw | ConvertFrom-Json + $operatorOutcome.gateStatus | Should -Be 'fail' + $operatorOutcome.nextActionId | Should -Be 'inspect-results-xml-truncation' + + $summary = Get-Content -LiteralPath (Join-Path $workspaceDir 'pester-summary.json') -Raw | ConvertFrom-Json + $summary.schemaVersion | Should -Be '1.7.1' + $summary.executionPostprocessStatus | Should -Be 'results-xml-truncated' + $summary.resultsXmlStatus | Should -Be 'truncated-root' + + $replayReceipt = Get-Content -LiteralPath (Join-Path $workspaceDir 'pester-local-replay-receipt.json') -Raw | ConvertFrom-Json + $replayReceipt.classification | Should -Be 'results-xml-truncated' + $replayReceipt.operatorOutcomePresent | Should -BeTrue + $replayReceipt.operatorOutcomeGateStatus | Should -Be 'fail' + $replayReceipt.stagedExecutionReceiptSchemaStatus | Should -Be 'ok' + } +} diff --git a/tests/Run-NIWindowsContainerCompare.Tests.ps1 b/tests/Run-NIWindowsContainerCompare.Tests.ps1 index 036027cc7..a1f3c824c 100644 --- a/tests/Run-NIWindowsContainerCompare.Tests.ps1 +++ b/tests/Run-NIWindowsContainerCompare.Tests.ps1 @@ -646,6 +646,45 @@ exit 0 ($output -join "`n") | Should -Match 'observedDockerHost=npipe:////./pipe/docker_engine' } + It 'normalizes provider-qualified base and head VI paths before building Docker mounts' { + $work = Join-Path $TestDrive 'provider-qualified-inputs' + New-Item -ItemType Directory -Path $work | Out-Null + & $script:NewDockerStub -WorkRoot $work | Out-Null + + $logPath = Join-Path $work 'docker-log.ndjson' + Set-Item Env:DOCKER_STUB_LOG $logPath + Set-Item Env:DOCKER_STUB_OSTYPE 'windows' + Set-Item Env:DOCKER_STUB_IMAGE_EXISTS '1' + Set-Item Env:DOCKER_STUB_CONTEXT 'desktop-windows' + Set-Item Env:DOCKER_STUB_RUN_EXIT_CODE '1' + Set-Item Env:DOCKER_STUB_RUN_STDOUT 'CreateComparisonReport completed with diff.' + + $baseVi = Join-Path $work 'Base.vi' + $headVi = Join-Path $work 'Head.vi' + Set-Content -LiteralPath $baseVi -Value 'base' -Encoding utf8 + Set-Content -LiteralPath $headVi -Value 'head' -Encoding utf8 + $providerBaseVi = "Microsoft.PowerShell.Core\FileSystem::$baseVi" + $providerHeadVi = "Microsoft.PowerShell.Core\FileSystem::$headVi" + $reportPath = Join-Path $work 'out\compare-report.html' + + $output = & pwsh -NoLogo -NoProfile -File $script:RunnerScript ` + -BaseVi $providerBaseVi ` + -HeadVi $providerHeadVi ` + -ReportPath $reportPath ` + -RuntimeEngineReadyTimeoutSeconds 5 ` + -RuntimeEngineReadyPollSeconds 1 2>&1 + $LASTEXITCODE | Should -Be 1 -Because ($output -join "`n") + + $capturePath = Join-Path (Split-Path -Parent $reportPath) 'ni-windows-container-capture.json' + Test-Path -LiteralPath $capturePath | Should -BeTrue + + $capture = Get-Content -LiteralPath $capturePath -Raw | ConvertFrom-Json + [string]$capture.baseVi | Should -Be ([System.IO.Path]::GetFullPath($baseVi)) + [string]$capture.headVi | Should -Be ([System.IO.Path]::GetFullPath($headVi)) + [string]$capture.baseVi | Should -Not -Match 'Microsoft\.PowerShell\.Core\\FileSystem::' + [string]$capture.headVi | Should -Not -Match 'Microsoft\.PowerShell\.Core\\FileSystem::' + } + It 'validates report flag labels against docker compare flags' { $work = Join-Path $TestDrive 'compare-report-flag-labels' New-Item -ItemType Directory -Path $work | Out-Null diff --git a/tests/Run-PesterExecutionOnly.Local.PathHygiene.Tests.ps1 b/tests/Run-PesterExecutionOnly.Local.PathHygiene.Tests.ps1 new file mode 100644 index 000000000..c5c59abfd --- /dev/null +++ b/tests/Run-PesterExecutionOnly.Local.PathHygiene.Tests.ps1 @@ -0,0 +1,91 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Describe 'Run-PesterExecutionOnly.Local path hygiene' -Tag 'Execution' { + BeforeAll { + $script:repoRoot = Split-Path -Parent $PSScriptRoot + $script:harnessPath = Join-Path $script:repoRoot 'tools/Run-PesterExecutionOnly.Local.ps1' + } + + It 'blocks unsafe managed roots before dispatch and writes a blocked receipt to the safe root' { + $tempRoot = Join-Path $TestDrive 'path-hygiene-block' + $testsDir = Join-Path $tempRoot 'tests' + $safeRoot = Join-Path $tempRoot 'safe-root' + $riskyResults = Join-Path $tempRoot 'OneDrive - Contoso/results' + $riskyLocks = Join-Path $tempRoot 'OneDrive - Contoso/locks' + New-Item -ItemType Directory -Path $testsDir, $safeRoot -Force | Out-Null + + @' +Describe "Path hygiene block sample" { + It "would pass if dispatch ran" { + 1 | Should -Be 1 + } +} +'@ | Set-Content -LiteralPath (Join-Path $testsDir 'PathHygiene.Block.Sample.Tests.ps1') -Encoding UTF8 + + $output = & pwsh -NoLogo -NoProfile -File $script:harnessPath ` + -TestsPath $testsDir ` + -ResultsPath $riskyResults ` + -SessionLockRoot $riskyLocks ` + -PathHygieneMode block ` + -PathHygieneSafeRoot $safeRoot 2>&1 + + $LASTEXITCODE | Should -Be 1 -Because (($output | ForEach-Object { [string]$_ }) -join [Environment]::NewLine) + + $receiptPath = Join-Path $safeRoot 'blocked-results/pester-run-receipt.json' + Test-Path -LiteralPath $receiptPath -PathType Leaf | Should -BeTrue + Test-Path -LiteralPath $riskyResults | Should -BeFalse + Test-Path -LiteralPath $riskyLocks | Should -BeFalse + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json + $receipt.status | Should -Be 'path-hygiene-blocked' + $receipt.executionJobResult | Should -Be 'skipped' + $receipt.pathHygieneStatus | Should -Be 'path-hygiene-blocked' + $receipt.localHarness.pathHygiene.status | Should -Be 'path-hygiene-blocked' + $receipt.localHarness.pathHygiene.effectiveResultsPath | Should -Be ((Join-Path $safeRoot 'blocked-results') -replace '\\', '/') + @($receipt.localHarness.pathHygiene.risks | ForEach-Object { $_.id }) | Should -Contain 'onedrive-managed-root' + } + + It 'relocates unsafe managed roots into the safe root and completes dispatch there' { + $tempRoot = Join-Path $TestDrive 'path-hygiene-relocate' + $testsDir = Join-Path $tempRoot 'tests' + $safeRoot = Join-Path $tempRoot 'safe-root' + $riskyResults = Join-Path $tempRoot 'OneDrive - Contoso/results' + $riskyLocks = Join-Path $tempRoot 'OneDrive - Contoso/locks' + New-Item -ItemType Directory -Path $testsDir, $safeRoot -Force | Out-Null + + @' +Describe "Path hygiene relocate sample" { + It "passes" { + 1 | Should -Be 1 + } +} +'@ | Set-Content -LiteralPath (Join-Path $testsDir 'PathHygiene.Relocate.Sample.Tests.ps1') -Encoding UTF8 + + $output = & pwsh -NoLogo -NoProfile -File $script:harnessPath ` + -TestsPath $testsDir ` + -ResultsPath $riskyResults ` + -SessionLockRoot $riskyLocks ` + -PathHygieneMode relocate ` + -PathHygieneSafeRoot $safeRoot 2>&1 + + $LASTEXITCODE | Should -Be 0 -Because (($output | ForEach-Object { [string]$_ }) -join [Environment]::NewLine) + + $effectiveResults = Join-Path $safeRoot 'results' + $effectiveLocks = Join-Path $safeRoot 'session-lock' + $receiptPath = Join-Path $effectiveResults 'pester-run-receipt.json' + Test-Path -LiteralPath $receiptPath -PathType Leaf | Should -BeTrue + Test-Path -LiteralPath (Join-Path $effectiveResults 'pester-summary.json') -PathType Leaf | Should -BeTrue + Test-Path -LiteralPath $riskyResults | Should -BeFalse + Test-Path -LiteralPath $riskyLocks | Should -BeFalse + + $receipt = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json + $receipt.status | Should -Be 'completed' + $receipt.pathHygieneStatus | Should -Be 'relocated' + $receipt.localHarness.pathHygiene.status | Should -Be 'relocated' + $receipt.localHarness.pathHygiene.requestedResultsPath | Should -Be ($riskyResults -replace '\\', '/') + $receipt.localHarness.pathHygiene.effectiveResultsPath | Should -Be ($effectiveResults -replace '\\', '/') + $receipt.localHarness.sessionLockRoot | Should -Be ($effectiveLocks -replace '\\', '/') + @($receipt.localHarness.pathHygiene.risks | ForEach-Object { $_.id }) | Should -Contain 'onedrive-managed-root' + } +} diff --git a/tests/Write-PesterSummaryToStepSummary.CompactMode.Tests.ps1 b/tests/Write-PesterSummaryToStepSummary.CompactMode.Tests.ps1 index 20a242f94..2ce51c7b4 100644 --- a/tests/Write-PesterSummaryToStepSummary.CompactMode.Tests.ps1 +++ b/tests/Write-PesterSummaryToStepSummary.CompactMode.Tests.ps1 @@ -35,14 +35,79 @@ Describe 'Write-PesterSummaryToStepSummary compact & metadata' -Tag 'Unit' { It 'emits badge JSON metadata when -BadgeJsonPath provided' { $badgeJson = Join-Path $TestDrive 'badge/meta.json' + $outcome = [pscustomobject]@{ + schema = 'pester-operator-outcome@v1' + gateStatus = 'fail' + classification = 'test-failures' + nextActionId = 'review-top-failures' + nextAction = 'Review pester-failures.json, the top-failures summary, and the failing test names before deciding whether to rerun or fix source.' + reasons = @('summary-failed=1') + } | ConvertTo-Json -Depth 6 + Set-Content (Join-Path $resultsDir 'pester-operator-outcome.json') $outcome -Encoding UTF8 & $ScriptPath -ResultsDir $resultsDir -Compact -EmitFailureBadge -BadgeJsonPath $badgeJson Test-Path $badgeJson | Should -BeTrue $obj = Get-Content $badgeJson -Raw | ConvertFrom-Json $obj.status | Should -Be 'failed' $obj.total | Should -Be 5 $obj.failedTests | Should -Contain 'Failing.Test' + $obj.failureDetailsStatus | Should -Be 'available' + $obj.classification | Should -Be 'test-failures' + $obj.gateStatus | Should -Be 'fail' + $obj.nextActionId | Should -Be 'review-top-failures' $obj.badgeMarkdown | Should -Match '❌ Tests Failed:' } + + It 'reads array-shaped failures payloads in compact mode' { + $fails = @([pscustomobject]@{ Name='Failing.ArrayTest'; result='Failed'; Duration=0.12 }) | ConvertTo-Json -Depth 5 + Set-Content (Join-Path $resultsDir 'pester-failures.json') $fails -Encoding UTF8 + & $ScriptPath -ResultsDir $resultsDir -Compact -EmitFailureBadge + $content = Get-Content $env:GITHUB_STEP_SUMMARY -Raw + $content | Should -Match '\*\*Failures:\*\* Failing.ArrayTest' + } + + It 'surfaces explicit unavailable-details state in compact mode' { + $summary = [pscustomobject]@{ + total = 5 + passed = 4 + failed = 1 + skipped = 0 + errors = 0 + duration = 2.5 + failureDetailsStatus = 'unavailable' + failureDetailsReason = 'results-xml-truncated' + } | ConvertTo-Json -Depth 6 + Set-Content (Join-Path $resultsDir 'pester-summary.json') $summary -Encoding UTF8 + $fails = [pscustomobject]@{ + schema = 'pester-failures@v2' + schemaVersion = '1.1.0' + detailStatus = 'unavailable' + unavailableReason = 'results-xml-truncated' + detailCount = 0 + summary = [pscustomobject]@{ total = 5; failed = 1; errors = 0; skipped = 0 } + results = @() + } | ConvertTo-Json -Depth 6 + Set-Content (Join-Path $resultsDir 'pester-failures.json') $fails -Encoding UTF8 + + & $ScriptPath -ResultsDir $resultsDir -Compact -EmitFailureBadge + $content = Get-Content $env:GITHUB_STEP_SUMMARY -Raw + $content | Should -Match '\*\*Failure Details:\*\* unavailable' + $content | Should -Match 'results-xml-truncated' + } + + It 'surfaces operator outcome in compact mode when present' { + $outcome = [pscustomobject]@{ + schema = 'pester-operator-outcome@v1' + gateStatus = 'fail' + classification = 'unsupported-schema' + nextAction = 'Regenerate retained artifacts with the supported schema contract or update readers before rerunning the gate.' + } | ConvertTo-Json -Depth 6 + Set-Content (Join-Path $resultsDir 'pester-operator-outcome.json') $outcome -Encoding UTF8 + + & $ScriptPath -ResultsDir $resultsDir -Compact -EmitFailureBadge + $content = Get-Content $env:GITHUB_STEP_SUMMARY -Raw + $content | Should -Match '\*\*Gate Outcome:\*\* unsupported-schema \(fail\)' + $content | Should -Match '\*\*Next Action:\*\* Regenerate retained artifacts' + } } Context 'All passing compact' { diff --git a/tests/Write-PesterSummaryToStepSummary.Tests.ps1 b/tests/Write-PesterSummaryToStepSummary.Tests.ps1 index d662ecf25..8e49c5574 100644 --- a/tests/Write-PesterSummaryToStepSummary.Tests.ps1 +++ b/tests/Write-PesterSummaryToStepSummary.Tests.ps1 @@ -5,7 +5,7 @@ Describe 'Write-PesterSummaryToStepSummary script' -Tag 'Unit' { New-Item -ItemType Directory -Path $resultsDir | Out-Null # Minimal summary JSON $summary = [pscustomobject]@{ - total = 3; passed = 2; failed = 1; errors = 0; skipped = 0; duration = 1.23 + total = 3; passed = 2; failed = 1; errors = 0; skipped = 0; duration = 1.23; executionPack = 'comparevi' } | ConvertTo-Json -Depth 5 Set-Content -Path (Join-Path $resultsDir 'pester-summary.json') -Value $summary -Encoding UTF8 # Failures JSON (single failure) @@ -25,6 +25,7 @@ Describe 'Write-PesterSummaryToStepSummary script' -Tag 'Unit' { $content | Should -Match '\| Total \| 3 \|' $content | Should -Match '\| Passed \| 2 \|' $content | Should -Match '\| Failed \| 1 \|' + $content | Should -Match '\| Execution Pack \| comparevi \|' $content | Should -Match '
    Failed Tests' $content | Should -Match 'Sample.Test' $content | Should -Match '
    ' @@ -35,6 +36,7 @@ Describe 'Write-PesterSummaryToStepSummary script' -Tag 'Unit' { & $scriptPath -ResultsDir $resultsDir -FailedTestsCollapseStyle None $c2 = Get-Content $env:GITHUB_STEP_SUMMARY -Raw $c2 | Should -Match '### Failed Tests' + $c2 | Should -Match '\| Execution Pack \| comparevi \|' $c2 | Should -Not -Match '
    ' } @@ -53,8 +55,78 @@ Describe 'Write-PesterSummaryToStepSummary script' -Tag 'Unit' { (Get-Content $env:GITHUB_STEP_SUMMARY -Raw) | Should -Match '\*\*❌ Tests Failed:\*\* 1 of 3' } + It 'emits explicit unavailable-details note when canonical payload marks failure detail unavailable' { + $env:GITHUB_STEP_SUMMARY = Join-Path $TestDrive 'STEP_SUMMARY_unavailable.md' + $summary = [pscustomobject]@{ + total = 3 + passed = 2 + failed = 1 + errors = 0 + skipped = 0 + duration = 1.23 + executionPack = 'comparevi' + failureDetailsStatus = 'unavailable' + failureDetailsReason = 'results-xml-truncated' + } | ConvertTo-Json -Depth 6 + Set-Content -Path (Join-Path $resultsDir 'pester-summary.json') -Value $summary -Encoding UTF8 + $fail = [pscustomobject]@{ + schema = 'pester-failures@v2' + schemaVersion = '1.1.0' + detailStatus = 'unavailable' + unavailableReason = 'results-xml-truncated' + detailCount = 0 + summary = [pscustomobject]@{ total = 3; failed = 1; errors = 0; skipped = 0 } + results = @() + } | ConvertTo-Json -Depth 6 + Set-Content -Path (Join-Path $resultsDir 'pester-failures.json') -Value $fail -Encoding UTF8 + + & $scriptPath -ResultsDir $resultsDir + + $content = Get-Content $env:GITHUB_STEP_SUMMARY -Raw + $content | Should -Match 'Failure details unavailable' + $content | Should -Match 'results-xml-truncated' + } + + It 'surfaces operator outcome classification and next action when present' { + $env:GITHUB_STEP_SUMMARY = Join-Path $TestDrive 'STEP_SUMMARY_outcome.md' + $summary = [pscustomobject]@{ + schemaVersion = '1.7.1' + total = 3 + passed = 2 + failed = 1 + errors = 0 + skipped = 0 + duration = 1.23 + executionPack = 'comparevi' + } | ConvertTo-Json -Depth 6 + Set-Content -Path (Join-Path $resultsDir 'pester-summary.json') -Value $summary -Encoding UTF8 + $outcome = [pscustomobject]@{ + schema = 'pester-operator-outcome@v1' + gateStatus = 'fail' + classification = 'unsupported-schema' + reasons = @('execution-receipt-unsupported-schema') + nextAction = 'Regenerate retained artifacts with the supported schema contract or update readers before rerunning the gate.' + } | ConvertTo-Json -Depth 6 + Set-Content -Path (Join-Path $resultsDir 'pester-operator-outcome.json') -Value $outcome -Encoding UTF8 + + & $scriptPath -ResultsDir $resultsDir + + $content = Get-Content $env:GITHUB_STEP_SUMMARY -Raw + $content | Should -Match '### Operator Outcome' + $content | Should -Match 'Classification: unsupported-schema' + $content | Should -Match 'Next action: Regenerate retained artifacts' + } + It 'links failed test name when Relative link style selected' { $env:GITHUB_STEP_SUMMARY = Join-Path $TestDrive 'STEP_SUMMARY_links.md' + $summary = [pscustomobject]@{ + total = 3; passed = 2; failed = 1; errors = 0; skipped = 0; duration = 1.23; executionPack = 'comparevi' + } | ConvertTo-Json -Depth 5 + Set-Content -Path (Join-Path $resultsDir 'pester-summary.json') -Value $summary -Encoding UTF8 + $fail = [pscustomobject]@{ + results = @([pscustomobject]@{ Name = 'Sample.Test'; result = 'Failed'; Duration = 0.45 }) + } | ConvertTo-Json -Depth 5 + Set-Content -Path (Join-Path $resultsDir 'pester-failures.json') -Value $fail -Encoding UTF8 & $scriptPath -ResultsDir $resultsDir -FailedTestsLinkStyle Relative $c4 = Get-Content $env:GITHUB_STEP_SUMMARY -Raw $c4 | Should -Match '\[Sample.Test\]\(tests/Sample.Test.Tests.ps1\)' @@ -68,4 +140,11 @@ Describe 'Write-PesterSummaryToStepSummary script' -Tag 'Unit' { Set-Content -Path (Join-Path $alt 'pester-summary.json') -Value '{"total":0,"passed":0,"failed":0}' -Encoding UTF8 { & $scriptPath -ResultsDir $alt } | Should -Not -Throw } + + It 'tolerates array-shaped failures payloads without throwing' { + $env:GITHUB_STEP_SUMMARY = Join-Path $TestDrive 'STEP_SUMMARY_array.md' + Set-Content -Path (Join-Path $resultsDir 'pester-failures.json') -Value '[]' -Encoding UTF8 + { & $scriptPath -ResultsDir $resultsDir } | Should -Not -Throw + (Get-Content $env:GITHUB_STEP_SUMMARY -Raw) | Should -Match '\| Failed \| 1 \|' + } } diff --git a/tests/fixtures/pester-service-model/legacy-results-xml-truncated/pester-run-receipt.json b/tests/fixtures/pester-service-model/legacy-results-xml-truncated/pester-run-receipt.json new file mode 100644 index 000000000..bf4d38761 --- /dev/null +++ b/tests/fixtures/pester-service-model/legacy-results-xml-truncated/pester-run-receipt.json @@ -0,0 +1,13 @@ +{ + "schema": "pester-execution-receipt@v1", + "generatedAtUtc": "2026-03-31T21:27:07.6826294Z", + "contextStatus": "ready", + "readinessStatus": "ready", + "selectionStatus": "ready", + "executionJobResult": "success", + "dispatcherExitCode": -1, + "status": "results-xml-truncated", + "rawArtifactName": "pester-run-raw", + "rawArtifactExpected": true, + "source": "pester-run/finalize" +} diff --git a/tests/fixtures/pester-service-model/legacy-results-xml-truncated/raw/dispatcher-events.ndjson b/tests/fixtures/pester-service-model/legacy-results-xml-truncated/raw/dispatcher-events.ndjson new file mode 100644 index 000000000..04299833d --- /dev/null +++ b/tests/fixtures/pester-service-model/legacy-results-xml-truncated/raw/dispatcher-events.ndjson @@ -0,0 +1,2 @@ +{"schema":"comparevi/runtime-event/v1","tsUtc":"2026-03-31T21:00:00Z","source":"pester-dispatcher","phase":"start","level":"info","message":"Dispatcher session initialized."} +{"schema":"comparevi/runtime-event/v1","tsUtc":"2026-03-31T21:00:03Z","source":"pester-dispatcher","phase":"dispatch","level":"error","message":"Representative replay fixture recorded failing tests before XML truncation."} diff --git a/tests/fixtures/pester-service-model/legacy-results-xml-truncated/raw/pester-failures.json b/tests/fixtures/pester-service-model/legacy-results-xml-truncated/raw/pester-failures.json new file mode 100644 index 000000000..56491ffaa --- /dev/null +++ b/tests/fixtures/pester-service-model/legacy-results-xml-truncated/raw/pester-failures.json @@ -0,0 +1,20 @@ +{ + "schema": "pester-failures@v2", + "schemaVersion": "1.1.0", + "detailStatus": "available", + "detailCount": 1, + "summary": { + "total": 4, + "failed": 1, + "errors": 0, + "skipped": 1 + }, + "results": [ + { + "name": "RepresentativeReplay.fails", + "file": "tests/RepresentativeReplayFixture.Tests.ps1", + "status": "failed", + "message": "Expected 0, but got 1." + } + ] +} diff --git a/tests/fixtures/pester-service-model/legacy-results-xml-truncated/raw/pester-summary.json b/tests/fixtures/pester-service-model/legacy-results-xml-truncated/raw/pester-summary.json new file mode 100644 index 000000000..8f2433027 --- /dev/null +++ b/tests/fixtures/pester-service-model/legacy-results-xml-truncated/raw/pester-summary.json @@ -0,0 +1,15 @@ +{ + "total": 4, + "passed": 2, + "failed": 1, + "errors": 0, + "skipped": 1, + "timestamp": "2026-03-31T21:26:48.0203952Z", + "resultsXmlStatus": "truncated-root", + "resultsXmlSummarySource": "root-attributes", + "resultsXmlCloseTagPresent": false, + "resultsXmlSizeBytes": 512, + "resultsXmlParseError": "Unexpected end of file.", + "executionPostprocessStatus": "results-xml-truncated", + "executionPostprocessSchema": "pester-execution-postprocess@v1" +} diff --git a/tools/Invoke-CompareCli.ps1 b/tools/Invoke-CompareCli.ps1 index 3224270a4..6a3b5e205 100644 --- a/tools/Invoke-CompareCli.ps1 +++ b/tools/Invoke-CompareCli.ps1 @@ -8,20 +8,7 @@ param( Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' - -function Get-IncludePatterns { - param([string]$Name) - switch ($Name.ToLowerInvariant()) { - 'dispatcher' { return @('Invoke-PesterTests*.ps1','PesterAvailability.Tests.ps1','NestedDispatcher*.Tests.ps1') } - 'fixtures' { return @('Fixtures.*.ps1','FixtureValidation*.ps1','FixtureSummary*.ps1','ViBinaryHandling.Tests.ps1','FixtureValidationDiff.Tests.ps1') } - 'schema' { return @('Schema.*.ps1','SchemaLite*.ps1') } - 'comparevi' { return @('CompareVI*.ps1','CanonicalCli.Tests.ps1','Args.Tokenization.Tests.ps1') } - 'loop' { return @('CompareLoop*.ps1','Run-AutonomousIntegrationLoop*.ps1','LoopMetrics.Tests.ps1','Integration-ControlLoop*.ps1','IntegrationControlLoop*.ps1') } - 'psummary' { return @('PesterSummary*.ps1','Write-PesterSummaryToStepSummary*.ps1','AggregationHints*.ps1') } - 'workflow' { return @('Workflow*.ps1','On-FixtureValidationFail.Tests.ps1','Watch.FlakyRecovery.Tests.ps1','FunctionShadowing*.ps1','FunctionProxy.Tests.ps1','RunSummary.Tool*.ps1','Action.CompositeOutputs.Tests.ps1','Binding.MinRepro.Tests.ps1','ArtifactTracking*.ps1','Guard.*.Tests.ps1') } - default { return @('*.ps1') } - } -} +. (Join-Path $PSScriptRoot 'PesterExecutionPacks.ps1') function Resolve-LegacyIncludeIntegration { param( @@ -102,10 +89,9 @@ if (-not (Test-Path -LiteralPath $resultsDir -PathType Container)) { New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null } -$includePatterns = Get-IncludePatterns -Name $Category -$includePatterns = @($includePatterns | Where-Object { $_ }) +$executionPack = Resolve-PesterExecutionPack -ExecutionPack $Category -Write-Host "[cli] category=$Category results=$resultsDir include=$($includePatterns -join ',')" -ForegroundColor Cyan +Write-Host "[cli] category=$Category results=$resultsDir executionPack=$($executionPack.executionPack) include=$($executionPack.effectiveIncludePatterns -join ',')" -ForegroundColor Cyan $legacyIncludeSpecified = $PSBoundParameters.ContainsKey('IncludeIntegration') $modeSpecified = $PSBoundParameters.ContainsKey('IntegrationMode') @@ -145,10 +131,10 @@ Write-Host "[cli] integrationMode=$integrationModeResolved includeIntegration=$i & "$PSScriptRoot/Invoke-PesterTests.ps1" ` -TestsPath 'tests' ` + -ExecutionPack $executionPack.executionPack ` -IntegrationMode $integrationModeResolved ` -ResultsPath $resultsDir ` - -EmitFailuresJsonAlways ` - -IncludePatterns $includePatterns + -EmitFailuresJsonAlways $pesterExit = $LASTEXITCODE $summaryPath = Join-Path $resultsDir 'pester-summary.json' @@ -156,6 +142,7 @@ $cliRun = [ordered]@{ schema = 'compare-cli-run/v1' generatedAtUtc = (Get-Date).ToUniversalTime().ToString('o') category = $Category + executionPack = $executionPack.executionPack includeIntegration = [bool]$includeIntegrationBool integrationMode = $integrationModeResolved integrationSource = $integrationReason diff --git a/tools/Invoke-PesterEvidenceClassification.ps1 b/tools/Invoke-PesterEvidenceClassification.ps1 new file mode 100644 index 000000000..ca093ddfc --- /dev/null +++ b/tools/Invoke-PesterEvidenceClassification.ps1 @@ -0,0 +1,249 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $false)] + [string]$ResultsDir = 'tests/results', + + [Parameter(Mandatory = $false)] + [string]$ExecutionReceiptPath, + + [Parameter(Mandatory = $false)] + [string]$ContextStatus = 'unknown', + + [Parameter(Mandatory = $false)] + [string]$ReadinessStatus = 'unknown', + + [Parameter(Mandatory = $false)] + [string]$SelectionStatus = 'unknown', + + [Parameter(Mandatory = $false)] + [string]$ExecutionJobResult = '', + + [Parameter(Mandatory = $false)] + [string]$DispatcherExitCode = '', + + [Parameter(Mandatory = $false)] + [string]$RawArtifactDownload = 'local', + + [Parameter(Mandatory = $false)] + [string]$OutputPath = 'pester-evidence-classification.json' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$schemaToolPath = Join-Path $PSScriptRoot 'PesterServiceModelSchema.ps1' +if (-not (Test-Path -LiteralPath $schemaToolPath -PathType Leaf)) { + throw "Schema tool not found: $schemaToolPath" +} +. $schemaToolPath + +function Resolve-OptionalPath { + param( + [Parameter(Mandatory = $true)][string]$BasePath, + [string]$PathValue + ) + + if ([string]::IsNullOrWhiteSpace($PathValue)) { + return $null + } + if ([System.IO.Path]::IsPathRooted($PathValue)) { + return [System.IO.Path]::GetFullPath($PathValue) + } + return [System.IO.Path]::GetFullPath((Join-Path $BasePath $PathValue)) +} + +function Get-OptionalStringProperty { + param( + $InputObject, + [Parameter(Mandatory = $true)][string]$Name, + [string]$DefaultValue = '' + ) + + if ($null -eq $InputObject) { + return $DefaultValue + } + + if (-not ($InputObject.PSObject.Properties.Name -contains $Name)) { + return $DefaultValue + } + + return [string]$InputObject.$Name +} + +$resolvedResultsDir = [System.IO.Path]::GetFullPath($ResultsDir) +if (-not (Test-Path -LiteralPath $resolvedResultsDir -PathType Container)) { + New-Item -ItemType Directory -Path $resolvedResultsDir -Force | Out-Null +} + +$summaryPath = Join-Path $resolvedResultsDir 'pester-summary.json' +$resolvedReceiptPath = Resolve-OptionalPath -BasePath $resolvedResultsDir -PathValue $ExecutionReceiptPath +$resolvedOutputPath = Resolve-OptionalPath -BasePath $resolvedResultsDir -PathValue $OutputPath +if (-not $resolvedOutputPath) { + $resolvedOutputPath = Join-Path $resolvedResultsDir 'pester-evidence-classification.json' +} + +$executionReceipt = $null +$executionReceiptPresent = $false +$executionReceiptStatus = 'missing' +$executionReceiptSchemaState = $null +if ($resolvedReceiptPath -and (Test-Path -LiteralPath $resolvedReceiptPath -PathType Leaf)) { + $executionReceiptSchemaState = Test-PesterServiceModelSchemaContract ` + -DocumentState (Read-PesterServiceModelJsonDocument -PathValue $resolvedReceiptPath -ContractName 'execution-receipt') ` + -ExpectedSchema 'pester-execution-receipt@v1' + $executionReceiptPresent = $true + if ($executionReceiptSchemaState.valid) { + $executionReceipt = $executionReceiptSchemaState.document + $executionReceiptStatus = [string]$executionReceipt.status + } else { + $executionReceiptStatus = 'unsupported-schema' + } +} + +$summarySchemaState = Test-PesterServiceModelSchemaContract ` + -DocumentState (Read-PesterServiceModelJsonDocument -PathValue $summaryPath -ContractName 'pester-summary') ` + -ExpectedSchemaVersionMajor 1 ` + -RequireSchemaVersion + +$effectiveContextStatus = if ($ContextStatus -and $ContextStatus -ne 'unknown') { $ContextStatus } elseif ($executionReceipt) { Get-OptionalStringProperty -InputObject $executionReceipt -Name 'contextStatus' -DefaultValue 'unknown' } else { 'unknown' } +$effectiveReadinessStatus = if ($ReadinessStatus -and $ReadinessStatus -ne 'unknown') { $ReadinessStatus } elseif ($executionReceipt) { Get-OptionalStringProperty -InputObject $executionReceipt -Name 'readinessStatus' -DefaultValue 'unknown' } else { 'unknown' } +$effectiveSelectionStatus = if ($SelectionStatus -and $SelectionStatus -ne 'unknown') { $SelectionStatus } elseif ($executionReceipt) { Get-OptionalStringProperty -InputObject $executionReceipt -Name 'selectionStatus' -DefaultValue 'unknown' } else { 'unknown' } +$effectiveExecutionJobResult = if (-not [string]::IsNullOrWhiteSpace($ExecutionJobResult)) { $ExecutionJobResult } elseif ($executionReceipt -and $executionReceipt.PSObject.Properties.Name -contains 'executionJobResult') { [string]$executionReceipt.executionJobResult } else { '' } +$effectiveDispatcherExitCode = if (-not [string]::IsNullOrWhiteSpace($DispatcherExitCode)) { $DispatcherExitCode } elseif ($executionReceipt -and $executionReceipt.PSObject.Properties.Name -contains 'dispatcherExitCode') { [string]$executionReceipt.dispatcherExitCode } else { '-1' } +if ([string]::IsNullOrWhiteSpace($effectiveDispatcherExitCode)) { + $effectiveDispatcherExitCode = '-1' +} + +$classification = 'seam-defect' +$reasons = New-Object System.Collections.Generic.List[string] +if ($effectiveContextStatus -ne 'ready') { + $reasons.Add(("context-status={0}" -f $effectiveContextStatus)) | Out-Null +} +if ($effectiveReadinessStatus -ne 'ready') { + $reasons.Add(("readiness-status={0}" -f $effectiveReadinessStatus)) | Out-Null +} +if ($effectiveSelectionStatus -ne 'ready') { + $reasons.Add(("selection-status={0}" -f $effectiveSelectionStatus)) | Out-Null +} +if ($effectiveExecutionJobResult -eq 'skipped') { + $reasons.Add('execution-job-skipped') | Out-Null +} elseif ($effectiveExecutionJobResult -eq 'cancelled') { + $reasons.Add('execution-job-cancelled') | Out-Null +} elseif ($effectiveExecutionJobResult -eq 'results-xml-truncated') { + $reasons.Add('execution-job-results-xml-truncated') | Out-Null +} elseif ($effectiveExecutionJobResult -eq 'invalid-results-xml') { + $reasons.Add('execution-job-invalid-results-xml') | Out-Null +} elseif ($effectiveExecutionJobResult -eq 'missing-results-xml') { + $reasons.Add('execution-job-missing-results-xml') | Out-Null +} elseif ($effectiveExecutionJobResult -eq 'unsupported-schema') { + $reasons.Add('execution-job-unsupported-schema') | Out-Null +} elseif ($effectiveExecutionJobResult -eq 'seam-defect') { + $reasons.Add('execution-job-seam-defect') | Out-Null +} elseif ($effectiveExecutionJobResult -eq 'unknown') { + $reasons.Add('execution-job-unknown') | Out-Null +} +if ($RawArtifactDownload -notin @('success', 'skipped', 'local', 'not-requested', 'staged')) { + $reasons.Add(("raw-artifact-download={0}" -f $RawArtifactDownload)) | Out-Null +} + +if (-not $executionReceiptPresent) { + $reasons.Add('execution-receipt-missing') | Out-Null +} elseif ($executionReceiptSchemaState -and -not $executionReceiptSchemaState.valid) { + $classification = 'unsupported-schema' + $reasons.Add([string]$executionReceiptSchemaState.reason) | Out-Null + if (-not [string]::IsNullOrWhiteSpace([string]$executionReceiptSchemaState.actualSchema)) { + $reasons.Add(("execution-receipt-schema={0}" -f [string]$executionReceiptSchemaState.actualSchema)) | Out-Null + } + if (-not [string]::IsNullOrWhiteSpace([string]$executionReceiptSchemaState.actualSchemaVersion)) { + $reasons.Add(("execution-receipt-schema-version={0}" -f [string]$executionReceiptSchemaState.actualSchemaVersion)) | Out-Null + } + if (-not [string]::IsNullOrWhiteSpace([string]$executionReceiptSchemaState.parseError)) { + $reasons.Add(("execution-receipt-parse-error={0}" -f [string]$executionReceiptSchemaState.parseError)) | Out-Null + } +} elseif (($effectiveContextStatus -ne 'ready' -and $effectiveExecutionJobResult -in @('skipped', 'cancelled')) -or $executionReceiptStatus -eq 'context-blocked') { + $classification = 'context-blocked' +} elseif ($effectiveReadinessStatus -ne 'ready' -and $effectiveExecutionJobResult -in @('skipped', 'cancelled')) { + $classification = 'readiness-blocked' +} elseif (($effectiveSelectionStatus -ne 'ready' -and $effectiveExecutionJobResult -in @('skipped', 'cancelled')) -or $executionReceiptStatus -eq 'selection-blocked') { + $classification = 'selection-blocked' +} elseif ($executionReceiptStatus -eq 'results-xml-truncated') { + $classification = 'results-xml-truncated' + $reasons.Add('execution-receipt-results-xml-truncated') | Out-Null +} elseif ($executionReceiptStatus -eq 'invalid-results-xml') { + $classification = 'invalid-results-xml' + $reasons.Add('execution-receipt-invalid-results-xml') | Out-Null +} elseif ($executionReceiptStatus -eq 'missing-results-xml') { + $classification = 'missing-results-xml' + $reasons.Add('execution-receipt-missing-results-xml') | Out-Null +} elseif ($executionReceiptStatus -eq 'unsupported-schema') { + $classification = 'unsupported-schema' + $reasons.Add('execution-receipt-unsupported-schema') | Out-Null +} elseif ($executionReceiptStatus -eq 'seam-defect') { + $reasons.Add('execution-receipt-seam-defect') | Out-Null +} elseif ($executionReceiptStatus -eq 'test-failures') { + $classification = 'test-failures' +} elseif ($summarySchemaState.present -and -not $summarySchemaState.valid) { + $classification = 'unsupported-schema' + $reasons.Add([string]$summarySchemaState.reason) | Out-Null + if (-not [string]::IsNullOrWhiteSpace([string]$summarySchemaState.actualSchemaVersion)) { + $reasons.Add(("summary-schema-version={0}" -f [string]$summarySchemaState.actualSchemaVersion)) | Out-Null + } + if (-not [string]::IsNullOrWhiteSpace([string]$summarySchemaState.parseError)) { + $reasons.Add(("summary-parse-error={0}" -f [string]$summarySchemaState.parseError)) | Out-Null + } +} elseif (Test-Path -LiteralPath $summaryPath -PathType Leaf) { + try { + $summary = Get-Content -LiteralPath $summaryPath -Raw | ConvertFrom-Json -ErrorAction Stop + if ($executionReceipt -and $executionReceipt.PSObject.Properties.Name -contains 'dispatcherExitCode') { + if ([string]$executionReceipt.dispatcherExitCode -and [string]$executionReceipt.dispatcherExitCode -ne $effectiveDispatcherExitCode) { + $reasons.Add('dispatcher-exit-mismatch') | Out-Null + } + } + if (($summary.PSObject.Properties.Name -contains 'resultsXmlStatus') -and [string]$summary.resultsXmlStatus -like 'truncated*') { + $classification = 'results-xml-truncated' + $reasons.Add(("results-xml-status={0}" -f [string]$summary.resultsXmlStatus)) | Out-Null + } elseif (($summary.PSObject.Properties.Name -contains 'resultsXmlStatus') -and [string]$summary.resultsXmlStatus -like 'invalid*') { + $classification = 'invalid-results-xml' + $reasons.Add(("results-xml-status={0}" -f [string]$summary.resultsXmlStatus)) | Out-Null + } elseif (([int]$summary.failed + [int]$summary.errors) -gt 0 -or $effectiveDispatcherExitCode -ne '0') { + $classification = 'test-failures' + } else { + $classification = 'ok' + } + } catch { + $classification = 'seam-defect' + $reasons.Add('summary-unparseable') | Out-Null + } +} else { + $reasons.Add('summary-missing') | Out-Null +} + +$payload = [ordered]@{ + schema = 'pester-evidence-classification@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + contextStatus = $effectiveContextStatus + readinessStatus = $effectiveReadinessStatus + selectionStatus = $effectiveSelectionStatus + selectionExecutionPack = Get-OptionalStringProperty -InputObject $executionReceipt -Name 'selectionExecutionPack' + selectionExecutionPackSource = Get-OptionalStringProperty -InputObject $executionReceipt -Name 'selectionExecutionPackSource' + executionJobResult = $effectiveExecutionJobResult + rawArtifactDownload = $RawArtifactDownload + dispatcherExitCode = [int]$effectiveDispatcherExitCode + summaryPresent = (Test-Path -LiteralPath $summaryPath -PathType Leaf) + executionReceiptSchemaStatus = if ($executionReceiptSchemaState) { [string]$executionReceiptSchemaState.classification } else { 'missing' } + summarySchemaStatus = [string]$summarySchemaState.classification + classification = $classification + reasons = @($reasons) +} +$payload | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $resolvedOutputPath -Encoding UTF8 + +if ($env:GITHUB_OUTPUT) { + "classification=$classification" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "path=$resolvedOutputPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 +} + +Write-Host '### Pester evidence classification' -ForegroundColor Cyan +Write-Host ("classification : {0}" -f $classification) +Write-Host ("receiptPresent : {0}" -f $executionReceiptPresent) +Write-Host ("path : {0}" -f $resolvedOutputPath) + +exit 0 diff --git a/tools/Invoke-PesterEvidenceProvenance.ps1 b/tools/Invoke-PesterEvidenceProvenance.ps1 new file mode 100644 index 000000000..cb857ad84 --- /dev/null +++ b/tools/Invoke-PesterEvidenceProvenance.ps1 @@ -0,0 +1,370 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $false)] + [string]$ResultsDir = 'tests/results', + + [Parameter(Mandatory = $false)] + [string]$ExecutionReceiptPath = 'tests/execution-contract/pester-run-receipt.json', + + [Parameter(Mandatory = $false)] + [string]$ClassificationPath = 'pester-evidence-classification.json', + + [Parameter(Mandatory = $false)] + [string]$OperatorOutcomePath = 'pester-operator-outcome.json', + + [Parameter(Mandatory = $false)] + [string]$TelemetryPath = 'pester-execution-telemetry.json', + + [Parameter(Mandatory = $false)] + [string]$PostprocessReportPath = 'pester-execution-postprocess.json', + + [Parameter(Mandatory = $false)] + [string]$SummaryPath = 'pester-summary.json', + + [Parameter(Mandatory = $false)] + [string]$FailuresPath = 'pester-failures.json', + + [Parameter(Mandatory = $false)] + [string]$ResultsXmlPath = 'pester-results.xml', + + [Parameter(Mandatory = $false)] + [string]$DispatcherEventsPath = 'dispatcher-events.ndjson', + + [Parameter(Mandatory = $false)] + [string]$TotalsPath = 'pester-totals.json', + + [Parameter(Mandatory = $false)] + [string]$SessionIndexPath = 'session-index.json', + + [Parameter(Mandatory = $false)] + [string]$OutputPath = 'pester-evidence-provenance.json', + + [Parameter(Mandatory = $false)] + [string]$RawArtifactName = 'pester-run-raw', + + [Parameter(Mandatory = $false)] + [string]$RawArtifactDownload = 'local', + + [Parameter(Mandatory = $false)] + [string]$ExecutionReceiptArtifactName = 'pester-execution-contract', + + [Parameter(Mandatory = $false)] + [string]$SourceRawArtifactDir, + + [Parameter(Mandatory = $false)] + [ValidateSet('evidence', 'local-replay')] + [string]$ProvenanceKind = 'evidence' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Resolve-OptionalPath { + param( + [Parameter(Mandatory = $true)][string]$BasePath, + [string]$PathValue + ) + + if ([string]::IsNullOrWhiteSpace($PathValue)) { + return $null + } + if ([System.IO.Path]::IsPathRooted($PathValue)) { + return [System.IO.Path]::GetFullPath($PathValue) + } + return [System.IO.Path]::GetFullPath((Join-Path $BasePath $PathValue)) +} + +function ConvertTo-PortablePath { + param([AllowNull()][string]$PathValue) + + if ([string]::IsNullOrWhiteSpace($PathValue)) { + return $null + } + + return ([string]$PathValue).Replace('\', '/') +} + +function Get-RepoRelativePath { + param( + [Parameter(Mandatory = $true)][string]$RepoRoot, + [AllowNull()][string]$PathValue + ) + + if ([string]::IsNullOrWhiteSpace($PathValue)) { + return $null + } + + $resolvedPath = [System.IO.Path]::GetFullPath($PathValue) + $resolvedRoot = [System.IO.Path]::GetFullPath($RepoRoot) + $relative = [System.IO.Path]::GetRelativePath($resolvedRoot, $resolvedPath) + if ([string]::IsNullOrWhiteSpace($relative)) { + return '.' + } + if ($relative.StartsWith('..')) { + return $null + } + return ConvertTo-PortablePath $relative +} + +function Get-OptionalGitValue { + param([string[]]$Args) + + try { + $output = & git @Args 2>$null + if ($LASTEXITCODE -eq 0) { + return (($output | Out-String).Trim()) + } + } catch { + return $null + } + + return $null +} + +function Read-OptionalJsonMetadata { + param([Parameter(Mandatory = $true)][string]$PathValue) + + if (-not (Test-Path -LiteralPath $PathValue -PathType Leaf)) { + return $null + } + + try { + $document = Get-Content -LiteralPath $PathValue -Raw | ConvertFrom-Json -ErrorAction Stop + return [ordered]@{ + schema = if ($document.PSObject.Properties.Name -contains 'schema') { [string]$document.schema } else { $null } + schemaVersion = if ($document.PSObject.Properties.Name -contains 'schemaVersion') { [string]$document.schemaVersion } else { $null } + status = if ($document.PSObject.Properties.Name -contains 'status') { [string]$document.status } elseif ($document.PSObject.Properties.Name -contains 'classification') { [string]$document.classification } else { $null } + } + } catch { + return [ordered]@{ + schema = $null + schemaVersion = $null + status = 'invalid-json' + } + } +} + +function New-FileDescriptor { + param( + [Parameter(Mandatory = $true)][string]$RepoRoot, + [Parameter(Mandatory = $true)][string]$PathValue, + [Parameter(Mandatory = $true)][string]$Kind, + [Parameter(Mandatory = $true)][string]$Role, + [string]$ArtifactName + ) + + $resolvedPath = [System.IO.Path]::GetFullPath($PathValue) + $present = Test-Path -LiteralPath $resolvedPath -PathType Leaf + $descriptor = [ordered]@{ + kind = $Kind + role = $Role + artifactName = if ([string]::IsNullOrWhiteSpace($ArtifactName)) { $null } else { $ArtifactName } + path = ConvertTo-PortablePath $resolvedPath + repoRelativePath = Get-RepoRelativePath -RepoRoot $RepoRoot -PathValue $resolvedPath + present = $present + } + + if (-not $present) { + return $descriptor + } + + $item = Get-Item -LiteralPath $resolvedPath + $hash = Get-FileHash -LiteralPath $resolvedPath -Algorithm SHA256 + $descriptor.sizeBytes = [int64]$item.Length + $descriptor.sha256 = $hash.Hash.ToLowerInvariant() + $descriptor.lastWriteTimeUtc = $item.LastWriteTimeUtc.ToString('o') + + $metadata = Read-OptionalJsonMetadata -PathValue $resolvedPath + if ($metadata) { + $descriptor.schema = $metadata.schema + $descriptor.schemaVersion = $metadata.schemaVersion + $descriptor.status = $metadata.status + } + + return $descriptor +} + +function New-DirectoryDescriptor { + param( + [Parameter(Mandatory = $true)][string]$RepoRoot, + [Parameter(Mandatory = $true)][string]$PathValue, + [Parameter(Mandatory = $true)][string]$Kind, + [Parameter(Mandatory = $true)][string]$Role, + [string]$ArtifactName + ) + + $resolvedPath = [System.IO.Path]::GetFullPath($PathValue) + $present = Test-Path -LiteralPath $resolvedPath -PathType Container + $descriptor = [ordered]@{ + kind = $Kind + role = $Role + artifactName = if ([string]::IsNullOrWhiteSpace($ArtifactName)) { $null } else { $ArtifactName } + path = ConvertTo-PortablePath $resolvedPath + repoRelativePath = Get-RepoRelativePath -RepoRoot $RepoRoot -PathValue $resolvedPath + present = $present + fileCount = 0 + files = @() + } + + if (-not $present) { + return $descriptor + } + + $files = @(Get-ChildItem -LiteralPath $resolvedPath -File -Recurse | Sort-Object FullName) + $descriptor.fileCount = $files.Count + $descriptor.files = @( + $files | ForEach-Object { + $hash = Get-FileHash -LiteralPath $_.FullName -Algorithm SHA256 + [ordered]@{ + path = ConvertTo-PortablePath $_.FullName + repoRelativePath = Get-RepoRelativePath -RepoRoot $RepoRoot -PathValue $_.FullName + relativePath = ConvertTo-PortablePath ([System.IO.Path]::GetRelativePath($resolvedPath, $_.FullName)) + sizeBytes = [int64]$_.Length + sha256 = $hash.Hash.ToLowerInvariant() + } + } + ) + + return $descriptor +} + +function Get-RunContext { + param( + [Parameter(Mandatory = $true)][string]$RepoRoot, + [Parameter(Mandatory = $true)][string]$ProvenanceKind + ) + + $branch = if ($env:GITHUB_REF_NAME) { $env:GITHUB_REF_NAME } else { Get-OptionalGitValue -Args @('rev-parse', '--abbrev-ref', 'HEAD') } + $headSha = if ($env:GITHUB_SHA) { $env:GITHUB_SHA } else { Get-OptionalGitValue -Args @('rev-parse', 'HEAD') } + $repository = if ($env:GITHUB_REPOSITORY) { $env:GITHUB_REPOSITORY } else { Split-Path -Leaf $RepoRoot } + $runId = if ($env:GITHUB_RUN_ID) { $env:GITHUB_RUN_ID } else { $null } + $serverUrl = if ($env:GITHUB_SERVER_URL) { $env:GITHUB_SERVER_URL.TrimEnd('/') } else { 'https://github.com' } + + return [ordered]@{ + source = if ($runId) { 'github-actions' } else { 'local' } + repository = $repository + workflow = if ($env:GITHUB_WORKFLOW) { $env:GITHUB_WORKFLOW } else { if ($ProvenanceKind -eq 'local-replay') { 'Pester local replay' } else { 'Pester evidence' } } + eventName = if ($env:GITHUB_EVENT_NAME) { $env:GITHUB_EVENT_NAME } else { 'local' } + runId = $runId + runAttempt = if ($env:GITHUB_RUN_ATTEMPT) { $env:GITHUB_RUN_ATTEMPT } else { $null } + runUrl = if ($runId -and $repository) { "$serverUrl/$repository/actions/runs/$runId" } else { $null } + ref = if ($env:GITHUB_REF) { $env:GITHUB_REF } else { if ($branch) { "refs/heads/$branch" } else { $null } } + refName = $branch + branch = $branch + headRef = if ($env:GITHUB_HEAD_REF) { $env:GITHUB_HEAD_REF } else { $null } + baseRef = if ($env:GITHUB_BASE_REF) { $env:GITHUB_BASE_REF } else { $null } + headSha = $headSha + } +} + +$repoRoot = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..')) +$resolvedResultsDir = [System.IO.Path]::GetFullPath($ResultsDir) +if (-not (Test-Path -LiteralPath $resolvedResultsDir -PathType Container)) { + New-Item -ItemType Directory -Path $resolvedResultsDir -Force | Out-Null +} +$resolvedExecutionReceiptPath = Resolve-OptionalPath -BasePath $resolvedResultsDir -PathValue $ExecutionReceiptPath +$resolvedClassificationPath = Resolve-OptionalPath -BasePath $resolvedResultsDir -PathValue $ClassificationPath +$resolvedOperatorOutcomePath = Resolve-OptionalPath -BasePath $resolvedResultsDir -PathValue $OperatorOutcomePath +$resolvedTelemetryPath = Resolve-OptionalPath -BasePath $resolvedResultsDir -PathValue $TelemetryPath +$resolvedPostprocessReportPath = Resolve-OptionalPath -BasePath $resolvedResultsDir -PathValue $PostprocessReportPath +$resolvedSummaryPath = Resolve-OptionalPath -BasePath $resolvedResultsDir -PathValue $SummaryPath +$resolvedFailuresPath = Resolve-OptionalPath -BasePath $resolvedResultsDir -PathValue $FailuresPath +$resolvedResultsXmlPath = Resolve-OptionalPath -BasePath $resolvedResultsDir -PathValue $ResultsXmlPath +$resolvedDispatcherEventsPath = Resolve-OptionalPath -BasePath $resolvedResultsDir -PathValue $DispatcherEventsPath +$resolvedTotalsPath = Resolve-OptionalPath -BasePath $resolvedResultsDir -PathValue $TotalsPath +$resolvedSessionIndexPath = Resolve-OptionalPath -BasePath $resolvedResultsDir -PathValue $SessionIndexPath +$resolvedOutputPath = Resolve-OptionalPath -BasePath $resolvedResultsDir -PathValue $OutputPath +if (-not $resolvedOutputPath) { + $resolvedOutputPath = Join-Path $resolvedResultsDir 'pester-evidence-provenance.json' +} +$resolvedSourceRawArtifactDir = if ([string]::IsNullOrWhiteSpace($SourceRawArtifactDir)) { $null } else { Resolve-OptionalPath -BasePath $resolvedResultsDir -PathValue $SourceRawArtifactDir } + +$sourceInputs = New-Object System.Collections.Generic.List[object] +if ($resolvedSourceRawArtifactDir -and ([System.IO.Path]::GetFullPath($resolvedSourceRawArtifactDir) -ne $resolvedResultsDir)) { + $sourceInputs.Add((New-DirectoryDescriptor -RepoRoot $repoRoot -PathValue $resolvedSourceRawArtifactDir -Kind 'raw-artifact-set' -Role 'source-raw-artifacts' -ArtifactName $RawArtifactName)) | Out-Null +} else { + foreach ($entry in @( + @{ Path = $resolvedResultsXmlPath; Kind = 'raw-artifact'; Role = 'results-xml' }, + @{ Path = $resolvedSummaryPath; Kind = 'raw-artifact'; Role = 'summary' }, + @{ Path = $resolvedFailuresPath; Kind = 'raw-artifact'; Role = 'failures' }, + @{ Path = $resolvedPostprocessReportPath; Kind = 'raw-artifact'; Role = 'postprocess' }, + @{ Path = $resolvedTelemetryPath; Kind = 'raw-artifact'; Role = 'telemetry' }, + @{ Path = $resolvedDispatcherEventsPath; Kind = 'raw-artifact'; Role = 'dispatcher-events' } + )) { + $sourceInputs.Add((New-FileDescriptor -RepoRoot $repoRoot -PathValue $entry.Path -Kind $entry.Kind -Role $entry.Role -ArtifactName $RawArtifactName)) | Out-Null + } +} +if ($resolvedExecutionReceiptPath) { + $sourceInputs.Add((New-FileDescriptor -RepoRoot $repoRoot -PathValue $resolvedExecutionReceiptPath -Kind 'execution-receipt' -Role 'execution-receipt' -ArtifactName $ExecutionReceiptArtifactName)) | Out-Null +} + +$derivedOutputs = New-Object System.Collections.Generic.List[object] +switch ($ProvenanceKind) { + 'local-replay' { + foreach ($entry in @( + @{ Path = $resolvedPostprocessReportPath; Kind = 'derived-evidence'; Role = 'postprocess-report' }, + @{ Path = $resolvedTelemetryPath; Kind = 'derived-evidence'; Role = 'telemetry' }, + @{ Path = $resolvedSummaryPath; Kind = 'derived-evidence'; Role = 'summary' }, + @{ Path = $resolvedTotalsPath; Kind = 'derived-evidence'; Role = 'totals' }, + @{ Path = $resolvedSessionIndexPath; Kind = 'derived-evidence'; Role = 'session-index' }, + @{ Path = $resolvedClassificationPath; Kind = 'derived-evidence'; Role = 'classification' }, + @{ Path = $resolvedOperatorOutcomePath; Kind = 'derived-evidence'; Role = 'operator-outcome' } + )) { + $derivedOutputs.Add((New-FileDescriptor -RepoRoot $repoRoot -PathValue $entry.Path -Kind $entry.Kind -Role $entry.Role)) | Out-Null + } + } + default { + foreach ($entry in @( + @{ Path = $resolvedTotalsPath; Kind = 'derived-evidence'; Role = 'totals' }, + @{ Path = $resolvedSessionIndexPath; Kind = 'derived-evidence'; Role = 'session-index' }, + @{ Path = $resolvedClassificationPath; Kind = 'derived-evidence'; Role = 'classification' }, + @{ Path = $resolvedOperatorOutcomePath; Kind = 'derived-evidence'; Role = 'operator-outcome' } + )) { + $derivedOutputs.Add((New-FileDescriptor -RepoRoot $repoRoot -PathValue $entry.Path -Kind $entry.Kind -Role $entry.Role)) | Out-Null + } + } +} + +$subjectId = if ($ProvenanceKind -eq 'local-replay') { 'pester-local-replay' } else { 'pester-evidence' } +$portableSourceRawArtifactDir = ConvertTo-PortablePath $resolvedSourceRawArtifactDir +$portableWorkspaceResultsDir = ConvertTo-PortablePath $resolvedResultsDir +$runContext = Get-RunContext -RepoRoot $repoRoot -ProvenanceKind $ProvenanceKind +$sourceInputsArray = @($sourceInputs.ToArray()) +$derivedOutputsArray = @($derivedOutputs.ToArray()) + +$payload = [ordered]@{ + schema = 'pester-derived-provenance@v1' + schemaVersion = '1.0.0' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + provenanceKind = $ProvenanceKind + producer = [ordered]@{ + id = 'Invoke-PesterEvidenceProvenance.ps1' + version = '1.0.0' + } + subject = [ordered]@{ + id = $subjectId + rawArtifactName = $RawArtifactName + rawArtifactDownload = $RawArtifactDownload + executionReceiptArtifactName = $ExecutionReceiptArtifactName + sourceRawArtifactDir = $portableSourceRawArtifactDir + workspaceResultsDir = $portableWorkspaceResultsDir + } + runContext = $runContext + sourceInputs = $sourceInputsArray + derivedOutputs = $derivedOutputsArray +} + +$payload | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $resolvedOutputPath -Encoding UTF8 + +if ($env:GITHUB_OUTPUT) { + "path=$resolvedOutputPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "provenance_kind=$ProvenanceKind" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 +} + +Write-Host '### Pester evidence provenance' -ForegroundColor Cyan +Write-Host ("kind : {0}" -f $ProvenanceKind) +Write-Host ("sourceCount : {0}" -f @($payload.sourceInputs).Count) +Write-Host ("derivedCount: {0}" -f @($payload.derivedOutputs).Count) +Write-Host ("path : {0}" -f $resolvedOutputPath) + +exit 0 diff --git a/tools/Invoke-PesterExecutionFinalize.ps1 b/tools/Invoke-PesterExecutionFinalize.ps1 index ce373147f..08e8bcf71 100644 --- a/tools/Invoke-PesterExecutionFinalize.ps1 +++ b/tools/Invoke-PesterExecutionFinalize.ps1 @@ -7,6 +7,11 @@ param( Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' +$failurePayloadTool = Join-Path $PSScriptRoot 'PesterFailurePayload.ps1' +if (Test-Path -LiteralPath $failurePayloadTool -PathType Leaf) { + . $failurePayloadTool +} + function Read-JsonObject { param([Parameter(Mandatory = $true)][string]$PathValue) @@ -17,6 +22,20 @@ function Read-JsonObject { return (Get-Content -LiteralPath $PathValue -Raw | ConvertFrom-Json -ErrorAction Stop) } +function Write-JsonFile { + param( + [Parameter(Mandatory = $true)][string]$PathValue, + [Parameter(Mandatory = $true)]$Payload, + [int]$Depth = 10 + ) + + $dir = Split-Path -Parent $PathValue + if ($dir -and -not (Test-Path -LiteralPath $dir -PathType Container)) { + New-Item -ItemType Directory -Path $dir -Force | Out-Null + } + $Payload | ConvertTo-Json -Depth $Depth | Set-Content -LiteralPath $PathValue -Encoding UTF8 +} + function Set-ObjectProperty { param( [Parameter(Mandatory = $true)]$InputObject, @@ -32,6 +51,22 @@ function Set-ObjectProperty { } } +function Write-LeakReportFromPayload { + param( + [Parameter(Mandatory = $true)][string]$ResultsDirectory, + $Payload + ) + + if ($null -eq $Payload) { + return $null + } + + $outputPath = Join-Path $ResultsDirectory 'pester-leak-report.json' + Write-JsonFile -PathValue $outputPath -Payload $Payload -Depth 10 + Write-Host ("Leak report written to: {0}" -f $outputPath) -ForegroundColor Gray + return $outputPath +} + function Get-RunnerProfileSnapshot { $runnerProfile = $null try { @@ -425,7 +460,8 @@ function Write-SessionIndex { [Parameter(Mandatory = $true)][string]$SummaryJsonPath, [Parameter(Mandatory = $true)][bool]$IncludeIntegration, [Parameter()][string]$IntegrationMode, - [Parameter()][string]$IntegrationSource + [Parameter()][string]$IntegrationSource, + $PublicationContext ) if (-not (Test-Path -LiteralPath $ResultsDirectory -PathType Container)) { @@ -670,6 +706,60 @@ function Write-SessionIndex { } } catch {} + if ($PublicationContext) { + try { + $publicationLines = @() + $selectedTests = @() + if ($PublicationContext.PSObject.Properties.Name -contains 'selectedTests') { + $selectedTests = @($PublicationContext.selectedTests | Where-Object { $_ -and "$_" -ne '' }) + } + $publicationLines += '### Selected Tests' + $publicationLines += '' + if ($selectedTests.Count -eq 0) { + $publicationLines += '- (none)' + } else { + foreach ($testName in ($selectedTests | Select-Object -Unique)) { + $publicationLines += ("- {0}" -f $testName) + } + } + $publicationLines += '' + $publicationLines += '### Configuration' + $publicationLines += '' + $publicationLines += ("- IncludeIntegration: {0}" -f $IncludeIntegration) + $publicationLines += ("- Integration Mode: {0}" -f $IntegrationMode) + if ($IntegrationSource) { $publicationLines += ("- Integration Source: {0}" -f $IntegrationSource) } + if ($PublicationContext.PSObject.Properties.Name -contains 'discovery' -and $PublicationContext.discovery) { + $publicationLines += ("- Discovery: {0}" -f [string]$PublicationContext.discovery) + } + if ($PublicationContext.PSObject.Properties.Name -contains 'rerunCommand' -and $PublicationContext.rerunCommand) { + $publicationLines += '' + $publicationLines += '### Re-run (gh)' + $publicationLines += '' + $publicationLines += ("- {0}" -f [string]$PublicationContext.rerunCommand) + } + if ($PublicationContext.PSObject.Properties.Name -contains 'guard' -and $PublicationContext.guard) { + $publicationLines += '' + $publicationLines += '### Guard' + $publicationLines += '' + $publicationLines += ("- Enabled: {0}" -f [bool]$PublicationContext.guard.enabled) + $publicationLines += ("- Heartbeats: {0}" -f [int]$PublicationContext.guard.heartbeats) + if ($PublicationContext.guard.PSObject.Properties.Name -contains 'heartbeatPath' -and $PublicationContext.guard.heartbeatPath) { + $publicationLines += ("- Heartbeat file: {0}" -f [string]$PublicationContext.guard.heartbeatPath) + } + if ($PublicationContext.guard.PSObject.Properties.Name -contains 'partialLogPath' -and $PublicationContext.guard.partialLogPath) { + $publicationLines += ("- Partial log: {0}" -f [string]$PublicationContext.guard.partialLogPath) + } + } + if ($idx['stepSummary']) { + $idx['stepSummary'] = $idx['stepSummary'] + "`n`n" + ($publicationLines -join "`n") + } else { + $idx['stepSummary'] = ($publicationLines -join "`n") + } + } catch { + Write-Warning ("Failed to enrich session index publication block: {0}" -f $_.Exception.Message) + } + } + $dest = Join-Path $ResultsDirectory 'session-index.json' $idx | ConvertTo-Json -Depth 6 | Out-File -FilePath $dest -Encoding utf8 -ErrorAction Stop Write-Host ("Session index written to: {0}" -f $dest) -ForegroundColor Gray @@ -680,13 +770,20 @@ $resolvedContextPath = [System.IO.Path]::GetFullPath($ContextPath) $context = Read-JsonObject -PathValue $resolvedContextPath $resultsDir = [System.IO.Path]::GetFullPath([string]$context.resultsDir) $repoRoot = [System.IO.Path]::GetFullPath([string]$context.repoRoot) -$jsonSummaryLeaf = if ([string]::IsNullOrWhiteSpace([string]$context.jsonSummaryPath)) { 'pester-summary.json' } else { [string]$context.jsonSummaryPath } +$jsonSummaryLeaf = if ([string]::IsNullOrWhiteSpace([string]$context.jsonSummaryPath)) { + 'pester-summary.json' +} else { + Split-Path -Leaf ([string]$context.jsonSummaryPath) +} $summaryPath = Join-Path $resultsDir 'pester-summary.txt' $summaryJsonPath = Join-Path $resultsDir $jsonSummaryLeaf $artifactTrailPath = Join-Path $resultsDir 'pester-artifacts-trail.json' +$publicationContext = if ($context.PSObject.Properties['publication']) { $context.publication } else { $null } +$publicationToolPath = Join-Path $PSScriptRoot 'Invoke-PesterExecutionPublication.ps1' $summaryTextValue = if ($context.PSObject.Properties['summaryText']) { [string]$context.summaryText } else { $null } $hasSummaryPayload = [bool]$context.PSObject.Properties['summaryPayload'] $hasArtifactTrail = [bool]$context.PSObject.Properties['artifactTrail'] +$hasLeakReportPayload = [bool]$context.PSObject.Properties['leakReportPayload'] if (-not (Test-Path -LiteralPath $resultsDir -PathType Container)) { New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null @@ -698,8 +795,14 @@ if (-not [string]::IsNullOrWhiteSpace($summaryTextValue)) { Write-Host ("Summary written to: {0}" -f $summaryPath) -ForegroundColor Gray } -if ($hasSummaryPayload -and $null -ne $context.summaryPayload) { - $context.summaryPayload | ConvertTo-Json -Depth 12 | Out-File -FilePath $summaryJsonPath -Encoding utf8 -ErrorAction Stop +$summaryPayloadToWrite = if ($hasSummaryPayload -and $null -ne $context.summaryPayload) { $context.summaryPayload } else { $null } +if ($null -ne $summaryPayloadToWrite) { + try { + Sync-PesterFailurePayload -Directory $resultsDir -SummaryObject $summaryPayloadToWrite -SchemaVersion ([string]$context.failuresSchemaVersion) | Out-Null + } catch { + Write-Warning ("Failed to synchronize failure-detail payload during finalize: {0}" -f $_.Exception.Message) + } + $summaryPayloadToWrite | ConvertTo-Json -Depth 12 | Out-File -FilePath $summaryJsonPath -Encoding utf8 -ErrorAction Stop Write-Host ("JSON summary written to: {0}" -f $summaryJsonPath) -ForegroundColor Gray } @@ -708,10 +811,21 @@ if ($hasArtifactTrail -and $null -ne $context.artifactTrail) { Write-Host ("Artifact trail written to: {0}" -f $artifactTrailPath) -ForegroundColor Gray } +if ($hasLeakReportPayload -and $null -ne $context.leakReportPayload) { + Write-LeakReportFromPayload -ResultsDirectory $resultsDir -Payload $context.leakReportPayload | Out-Null +} + Copy-CompareReportsAndWriteIndex -RepoRoot $repoRoot -ResultsDirectory $resultsDir -$sessionIndexPath = Write-SessionIndex -ResultsDirectory $resultsDir -SummaryJsonPath $jsonSummaryLeaf -IncludeIntegration ([bool]$context.includeIntegration) -IntegrationMode ([string]$context.integrationMode) -IntegrationSource ([string]$context.integrationSource) +$sessionIndexPath = Write-SessionIndex -ResultsDirectory $resultsDir -SummaryJsonPath $jsonSummaryLeaf -IncludeIntegration ([bool]$context.includeIntegration) -IntegrationMode ([string]$context.integrationMode) -IntegrationSource ([string]$context.integrationSource) -PublicationContext $publicationContext $manifestPath = Write-ArtifactManifest -Directory $resultsDir -SummaryJsonPath $jsonSummaryLeaf -ManifestVersion ([string]$context.manifestVersion) -SummarySchemaVersion ([string]$context.summarySchemaVersion) -FailuresSchemaVersion ([string]$context.failuresSchemaVersion) -LeakReportSchemaVersion ([string]$context.leakReportSchemaVersion) -DiagnosticsSchemaVersion ([string]$context.diagnosticsSchemaVersion) +if (Test-Path -LiteralPath $publicationToolPath -PathType Leaf) { + & $publicationToolPath -ContextPath $resolvedContextPath | Out-Host + if ($LASTEXITCODE -ne 0) { + throw "Invoke-PesterExecutionPublication.ps1 failed with exit code $LASTEXITCODE." + } +} + if ($env:GITHUB_OUTPUT) { "summary_path=$summaryPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 "summary_json_path=$summaryJsonPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 diff --git a/tools/Invoke-PesterExecutionPostprocess.ps1 b/tools/Invoke-PesterExecutionPostprocess.ps1 index a647197b1..acaa57e99 100644 --- a/tools/Invoke-PesterExecutionPostprocess.ps1 +++ b/tools/Invoke-PesterExecutionPostprocess.ps1 @@ -23,6 +23,14 @@ param( Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' +$defaultPesterSummarySchemaVersion = '1.7.1' + +$schemaToolPath = Join-Path $PSScriptRoot 'PesterServiceModelSchema.ps1' +if (-not (Test-Path -LiteralPath $schemaToolPath -PathType Leaf)) { + throw "Schema tool not found: $schemaToolPath" +} +. $schemaToolPath + function Set-ObjectProperty { param( [Parameter(Mandatory = $true)]$InputObject, @@ -38,18 +46,29 @@ function Set-ObjectProperty { } } -function Read-JsonObject { - param([Parameter(Mandatory = $true)][string]$PathValue) +function Test-RepairableLegacySummaryState { + param( + [Parameter(Mandatory = $true)]$State + ) - if (-not (Test-Path -LiteralPath $PathValue -PathType Leaf)) { - return $null + if (-not $State.present -or $State.valid) { + return $false } - try { - return (Get-Content -LiteralPath $PathValue -Raw | ConvertFrom-Json -ErrorAction Stop) - } catch { - return $null + if ([string]$State.reason -ne 'pester-summary-schema-version-missing') { + return $false } + + $document = $State.document + if (-not $document) { + return $false + } + + return ( + ($document.PSObject.Properties.Name -contains 'total') -and + ($document.PSObject.Properties.Name -contains 'failed') -and + ($document.PSObject.Properties.Name -contains 'errors') + ) } $resolvedResultsDir = [System.IO.Path]::GetFullPath($ResultsDir) @@ -65,21 +84,30 @@ if (-not (Test-Path -LiteralPath $xmlSummaryToolPath -PathType Leaf)) { throw "XML summary tool not found: $xmlSummaryToolPath" } -$existingSummary = Read-JsonObject -PathValue $summaryPath -$summaryPresentBefore = [bool]$existingSummary +$existingSummaryState = Test-PesterServiceModelSchemaContract ` + -DocumentState (Read-PesterServiceModelJsonDocument -PathValue $summaryPath -ContractName 'pester-summary') ` + -ExpectedSchemaVersionMajor 1 ` + -RequireSchemaVersion +$repairableLegacySummary = Test-RepairableLegacySummaryState -State $existingSummaryState +$existingSummary = if ($existingSummaryState.valid -or $repairableLegacySummary) { $existingSummaryState.document } else { $null } +$summaryPresentBefore = [bool]$existingSummaryState.present $xmlSummary = & $xmlSummaryToolPath -XmlPath $xmlPath -StabilizationTimeoutSeconds $XmlStabilizationTimeoutSeconds -PollIntervalMilliseconds $XmlPollIntervalMilliseconds if (-not $xmlSummary) { throw 'Get-PesterResultXmlSummary.ps1 returned no result.' } -$postprocessStatus = switch ([string]$xmlSummary.status) { - 'complete' { 'complete'; break } - 'truncated-root' { 'results-xml-truncated'; break } - 'truncated' { 'results-xml-truncated'; break } - 'invalid-root-attributes' { 'invalid-results-xml'; break } - 'invalid' { 'invalid-results-xml'; break } - 'missing' { 'missing-results-xml'; break } - default { 'seam-defect'; break } +$postprocessStatus = if ($existingSummaryState.present -and -not $existingSummaryState.valid -and -not $repairableLegacySummary) { + 'unsupported-schema' +} else { + switch ([string]$xmlSummary.status) { + 'complete' { 'complete'; break } + 'truncated-root' { 'results-xml-truncated'; break } + 'truncated' { 'results-xml-truncated'; break } + 'invalid-root-attributes' { 'invalid-results-xml'; break } + 'invalid' { 'invalid-results-xml'; break } + 'missing' { 'missing-results-xml'; break } + default { 'seam-defect'; break } + } } $summaryWritten = $false @@ -99,6 +127,9 @@ if ($canWriteSummary) { if (-not $summaryPayload.PSObject.Properties['timestamp']) { Set-ObjectProperty -InputObject $summaryPayload -Name 'timestamp' -Value ([DateTime]::UtcNow.ToString('o')) } + if (-not $summaryPayload.PSObject.Properties['schemaVersion']) { + Set-ObjectProperty -InputObject $summaryPayload -Name 'schemaVersion' -Value $defaultPesterSummarySchemaVersion + } Set-ObjectProperty -InputObject $summaryPayload -Name 'resultsXmlStatus' -Value ([string]$xmlSummary.status) Set-ObjectProperty -InputObject $summaryPayload -Name 'resultsXmlSummarySource' -Value $xmlSummary.summarySource Set-ObjectProperty -InputObject $summaryPayload -Name 'resultsXmlCloseTagPresent' -Value ([bool]$xmlSummary.closeTagPresent) @@ -119,6 +150,9 @@ $report = [ordered]@{ xmlPath = $xmlPath summaryPath = $summaryPath summaryPresentBefore = $summaryPresentBefore + summarySchemaStatus = if ($summaryPresentBefore -and $repairableLegacySummary) { 'legacy-schema-lite' } elseif ($summaryPresentBefore) { [string]$existingSummaryState.classification } else { 'missing' } + summarySchemaReason = if ($summaryPresentBefore) { [string]$existingSummaryState.reason } else { 'pester-summary-missing' } + summarySchemaVersion = if ($summaryPresentBefore) { [string]$existingSummaryState.actualSchemaVersion } else { $null } summaryWritten = $summaryWritten status = $postprocessStatus resultsXmlStatus = [string]$xmlSummary.status @@ -131,6 +165,8 @@ $report = [ordered]@{ errors = $xmlSummary.errors skipped = $xmlSummary.skipped parseError = [string]$xmlSummary.parseError + schemaClassification = if ($summaryPresentBefore -and -not $existingSummaryState.valid -and -not $repairableLegacySummary) { 'unsupported-schema' } elseif ($repairableLegacySummary) { 'legacy-schema-lite' } else { 'ok' } + schemaClassificationReason = if ($summaryPresentBefore -and -not $existingSummaryState.valid) { [string]$existingSummaryState.reason } else { 'schema-ok' } } $report | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $reportPath -Encoding UTF8 diff --git a/tools/Invoke-PesterExecutionPublication.ps1 b/tools/Invoke-PesterExecutionPublication.ps1 new file mode 100644 index 000000000..dc0b22c4e --- /dev/null +++ b/tools/Invoke-PesterExecutionPublication.ps1 @@ -0,0 +1,186 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string]$ContextPath +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Read-JsonObject { + param([Parameter(Mandatory = $true)][string]$PathValue) + if (-not (Test-Path -LiteralPath $PathValue -PathType Leaf)) { + throw "JSON file not found: $PathValue" + } + return (Get-Content -LiteralPath $PathValue -Raw | ConvertFrom-Json -ErrorAction Stop) +} + +function Append-MarkdownBlock { + param( + [string[]]$Lines, + [string]$CommentPath + ) + + if (-not $Lines -or $Lines.Count -eq 0) { + return + } + + $content = ($Lines -join "`n") + "`n" + if ($env:GITHUB_STEP_SUMMARY) { + $summaryDir = Split-Path -Parent $env:GITHUB_STEP_SUMMARY + if ($summaryDir -and -not (Test-Path -LiteralPath $summaryDir -PathType Container)) { + New-Item -ItemType Directory -Path $summaryDir -Force | Out-Null + } + Add-Content -LiteralPath $env:GITHUB_STEP_SUMMARY -Value $content -Encoding UTF8 + } + if (-not [string]::IsNullOrWhiteSpace($CommentPath)) { + $commentDir = Split-Path -Parent $CommentPath + if ($commentDir -and -not (Test-Path -LiteralPath $commentDir -PathType Container)) { + New-Item -ItemType Directory -Path $commentDir -Force | Out-Null + } + Add-Content -LiteralPath $CommentPath -Value $content -Encoding UTF8 + } +} + +function Build-DiagnosticsLines { + param([Parameter(Mandatory = $true)][string]$ResultsDirectory) + + $diagPath = Join-Path $ResultsDirectory 'result-shapes.json' + if (-not (Test-Path -LiteralPath $diagPath -PathType Leaf)) { + return @() + } + + try { + $diag = Read-JsonObject -PathValue $diagPath + $total = [int]$diag.totalEntries + $hasPath = [int]$diag.overall.hasPath + $hasTags = [int]$diag.overall.hasTags + $pct = { + param([int]$Numerator, [int]$Denominator) + if ($Denominator -le 0) { return '0%' } + return ('{0:P1}' -f ([double]$Numerator / [double]$Denominator)) + } + return @( + '### Diagnostics Summary', + '', + '| Metric | Count | Percent |', + '|---|---:|---:|', + ("| Total entries | {0} | - |" -f $total), + ("| Has Path | {0} | {1} |" -f $hasPath, (& $pct $hasPath $total)), + ("| Has Tags | {0} | {1} |" -f $hasTags, (& $pct $hasTags $total)) + ) + } catch { + Write-Warning ("Failed to build diagnostics publication block: {0}" -f $_.Exception.Message) + return @() + } +} + +$resolvedContextPath = [System.IO.Path]::GetFullPath($ContextPath) +$context = Read-JsonObject -PathValue $resolvedContextPath +$repoRoot = [System.IO.Path]::GetFullPath([string]$context.repoRoot) +$resultsDir = [System.IO.Path]::GetFullPath([string]$context.resultsDir) +$publication = if ($context.PSObject.Properties['publication']) { $context.publication } else { $null } +$commentPath = if ($publication -and $publication.PSObject.Properties['commentPath'] -and $publication.commentPath) { [string]$publication.commentPath } else { $null } +$toolRepoRoot = Split-Path -Parent $PSScriptRoot + +$summaryWriter = Join-Path $repoRoot 'scripts/Write-PesterSummaryToStepSummary.ps1' +if (-not (Test-Path -LiteralPath $summaryWriter -PathType Leaf)) { + $summaryWriter = Join-Path $toolRepoRoot 'scripts/Write-PesterSummaryToStepSummary.ps1' +} + +$sessionWriter = Join-Path $repoRoot 'tools/Write-SessionIndexSummary.ps1' +if (-not (Test-Path -LiteralPath $sessionWriter -PathType Leaf)) { + $sessionWriter = Join-Path $toolRepoRoot 'tools/Write-SessionIndexSummary.ps1' +} +$sessionIndexPath = Join-Path $resultsDir 'session-index.json' +$reportPath = Join-Path $resultsDir 'pester-execution-publication.json' + +$stepSummaryPresent = -not [string]::IsNullOrWhiteSpace($env:GITHUB_STEP_SUMMARY) +$publicationEnabled = $true +if ($publication -and $publication.PSObject.Properties['disableStepSummary']) { + $publicationEnabled = -not [bool]$publication.disableStepSummary +} + +$summaryWritten = $false +$sessionSummaryWritten = $false +$metadataWritten = $false +$diagnosticsWritten = $false +$sessionIndexMetadata = $false + +if ($publicationEnabled -and ($stepSummaryPresent -or -not [string]::IsNullOrWhiteSpace($commentPath))) { + if (Test-Path -LiteralPath $summaryWriter -PathType Leaf) { + $summaryArgs = @{ + ResultsDir = $resultsDir + } + if (-not [string]::IsNullOrWhiteSpace($commentPath)) { + $summaryArgs.CommentPath = $commentPath + } + try { + & $summaryWriter @summaryArgs | Out-Host + } catch { + throw "Write-PesterSummaryToStepSummary.ps1 failed: $($_.Exception.Message)" + } + $summaryWritten = $true + } + + if ($stepSummaryPresent -and (Test-Path -LiteralPath $sessionWriter -PathType Leaf)) { + try { + & $sessionWriter -ResultsDir $resultsDir | Out-Host + } catch { + throw "Write-SessionIndexSummary.ps1 failed: $($_.Exception.Message)" + } + $sessionSummaryWritten = $true + } + + if (Test-Path -LiteralPath $sessionIndexPath -PathType Leaf) { + try { + $sessionIndex = Read-JsonObject -PathValue $sessionIndexPath + if ($sessionIndex.PSObject.Properties['stepSummary'] -and $sessionIndex.stepSummary) { + Append-MarkdownBlock -Lines @([string]$sessionIndex.stepSummary) -CommentPath $commentPath + $sessionIndexMetadata = $true + } + } catch { + Write-Warning ("Failed to append session index publication metadata: {0}" -f $_.Exception.Message) + } + } + + $diagnosticsLines = Build-DiagnosticsLines -ResultsDirectory $resultsDir + if ($diagnosticsLines.Count -gt 0) { + Append-MarkdownBlock -Lines $diagnosticsLines -CommentPath $commentPath + $diagnosticsWritten = $true + } + + $metadataWritten = $sessionIndexMetadata -or $diagnosticsWritten +} + +$report = [ordered]@{ + schema = 'pester-execution-publication@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + contextPath = $resolvedContextPath + resultsDir = $resultsDir + publicationEnabled = $publicationEnabled + stepSummaryPresent = $stepSummaryPresent + commentPath = $commentPath + summaryWritten = $summaryWritten + sessionSummaryWritten = $sessionSummaryWritten + sessionIndexMetadataWritten = $sessionIndexMetadata + diagnosticsWritten = $diagnosticsWritten + metadataWritten = $metadataWritten +} +$report | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $reportPath -Encoding UTF8 + +if ($env:GITHUB_OUTPUT) { + "path=$reportPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "summary_written=$summaryWritten" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "metadata_written=$metadataWritten" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 +} + +Write-Host '### Pester execution publication' -ForegroundColor Cyan +Write-Host ("enabled : {0}" -f $publicationEnabled) +Write-Host ("summary : {0}" -f $summaryWritten) +Write-Host ("session : {0}" -f $sessionSummaryWritten) +Write-Host ("metadata : {0}" -f $metadataWritten) +Write-Host ("report : {0}" -f $reportPath) + +exit 0 diff --git a/tools/Invoke-PesterExecutionTelemetry.ps1 b/tools/Invoke-PesterExecutionTelemetry.ps1 new file mode 100644 index 000000000..d1d01781e --- /dev/null +++ b/tools/Invoke-PesterExecutionTelemetry.ps1 @@ -0,0 +1,254 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string]$ResultsDir +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Read-JsonObject { + param([Parameter(Mandatory = $true)][string]$PathValue) + + if (-not (Test-Path -LiteralPath $PathValue -PathType Leaf)) { + throw "JSON file not found: $PathValue" + } + + return (Get-Content -LiteralPath $PathValue -Raw | ConvertFrom-Json -ErrorAction Stop) +} + +function Write-JsonFile { + param( + [Parameter(Mandatory = $true)][string]$PathValue, + [Parameter(Mandatory = $true)]$Payload + ) + + $dir = Split-Path -Parent $PathValue + if ($dir -and -not (Test-Path -LiteralPath $dir -PathType Container)) { + New-Item -ItemType Directory -Path $dir -Force | Out-Null + } + + $Payload | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $PathValue -Encoding UTF8 +} + +function Get-SafeDateTime { + param( + [string]$Value, + [datetime]$Fallback + ) + + if (-not [string]::IsNullOrWhiteSpace($Value)) { + try { + return [datetime]::Parse($Value, [System.Globalization.CultureInfo]::InvariantCulture, [System.Globalization.DateTimeStyles]::RoundtripKind) + } catch {} + } + + return $Fallback +} + +function Get-ExecutionIdentity { + param( + $SessionIndex, + $Summary + ) + + $executionPack = $null + $executionPackSource = $null + $integrationMode = $null + $integrationSource = $null + + if ($null -ne $SessionIndex) { + if ($SessionIndex.PSObject.Properties['executionPack']) { $executionPack = [string]$SessionIndex.executionPack } + if ($SessionIndex.PSObject.Properties['executionPackSource']) { $executionPackSource = [string]$SessionIndex.executionPackSource } + if ($SessionIndex.PSObject.Properties['integrationMode']) { $integrationMode = [string]$SessionIndex.integrationMode } + if ($SessionIndex.PSObject.Properties['integrationSource']) { $integrationSource = [string]$SessionIndex.integrationSource } + } + + if ([string]::IsNullOrWhiteSpace($executionPack) -and $null -ne $Summary -and $Summary.PSObject.Properties['executionPack']) { + $executionPack = [string]$Summary.executionPack + } + + return [pscustomobject]@{ + executionPack = $executionPack + executionPackSource = $executionPackSource + integrationMode = $integrationMode + integrationSource = $integrationSource + } +} + +$resolvedResultsDir = [System.IO.Path]::GetFullPath($ResultsDir) +if (-not (Test-Path -LiteralPath $resolvedResultsDir -PathType Container)) { + throw "Results directory not found: $resolvedResultsDir" +} + +$eventsPath = Join-Path $resolvedResultsDir 'dispatcher-events.ndjson' +$sessionIndexPath = Join-Path $resolvedResultsDir 'session-index.json' +$summaryPath = Join-Path $resolvedResultsDir 'pester-summary.json' +$telemetryPath = Join-Path $resolvedResultsDir 'pester-execution-telemetry.json' + +$sessionIndex = if (Test-Path -LiteralPath $sessionIndexPath -PathType Leaf) { + Read-JsonObject -PathValue $sessionIndexPath +} else { + $null +} +$summary = if (Test-Path -LiteralPath $summaryPath -PathType Leaf) { + Read-JsonObject -PathValue $summaryPath +} else { + $null +} + +$events = @() +$phaseTable = [ordered]@{} +$parseErrors = @() +$firstEventAt = $null +$lastEventAt = $null +$lastEvent = $null + +if (Test-Path -LiteralPath $eventsPath -PathType Leaf) { + foreach ($line in (Get-Content -LiteralPath $eventsPath -ErrorAction Stop)) { + if ([string]::IsNullOrWhiteSpace($line)) { + continue + } + + try { + $event = $line | ConvertFrom-Json -ErrorAction Stop + $eventAt = Get-SafeDateTime -Value ([string]$event.tsUtc) -Fallback ([datetime]::UtcNow) + $phase = if ($event.PSObject.Properties['phase']) { [string]$event.phase } else { 'unknown' } + $level = if ($event.PSObject.Properties['level']) { [string]$event.level } else { 'info' } + $message = if ($event.PSObject.Properties['message']) { [string]$event.message } else { '' } + $record = [pscustomobject]@{ + tsUtc = $eventAt.ToUniversalTime().ToString('o') + phase = $phase + level = $level + message = $message + } + $events += $record + + if ($null -eq $firstEventAt -or $eventAt -lt $firstEventAt) { $firstEventAt = $eventAt } + if ($null -eq $lastEventAt -or $eventAt -ge $lastEventAt) { + $lastEventAt = $eventAt + $lastEvent = $record + } + + if (-not $phaseTable.Contains($phase)) { + $phaseTable[$phase] = [ordered]@{ + phase = $phase + count = 0 + firstAtUtc = $record.tsUtc + lastAtUtc = $record.tsUtc + lastLevel = $level + lastMessage = $message + } + } + + $phaseRecord = $phaseTable[$phase] + $phaseRecord.count = [int]$phaseRecord.count + 1 + $phaseRecord.lastAtUtc = $record.tsUtc + $phaseRecord.lastLevel = $level + $phaseRecord.lastMessage = $message + } catch { + $parseErrors += $_.Exception.Message + } + } +} + +$handshakeFiles = @(Get-ChildItem -Path $resolvedResultsDir -Recurse -Filter 'handshake-*.json' -File -ErrorAction SilentlyContinue) +$handshakeMarkers = @() +$lastHandshake = $null +$lastHandshakeAt = $null +foreach ($file in $handshakeFiles) { + $fallbackAt = $file.LastWriteTimeUtc + try { + $payload = Read-JsonObject -PathValue $file.FullName + $phase = if ($payload.PSObject.Properties['name']) { [string]$payload.name } else { [System.IO.Path]::GetFileNameWithoutExtension($file.Name) -replace '^handshake-', '' } + $status = if ($payload.PSObject.Properties['status']) { [string]$payload.status } else { $null } + $markerAt = Get-SafeDateTime -Value ([string]$payload.atUtc) -Fallback $fallbackAt + } catch { + $phase = [System.IO.Path]::GetFileNameWithoutExtension($file.Name) -replace '^handshake-', '' + $status = $null + $markerAt = $fallbackAt + } + + $marker = [pscustomobject]@{ + path = $file.FullName + phase = $phase + status = $status + atUtc = $markerAt.ToUniversalTime().ToString('o') + } + $handshakeMarkers += $marker + + if ($null -eq $lastHandshakeAt -or $markerAt -ge $lastHandshakeAt) { + $lastHandshakeAt = $markerAt + $lastHandshake = $marker + } +} + +$identity = Get-ExecutionIdentity -SessionIndex $sessionIndex -Summary $summary +$telemetryStatus = if ($events.Count -gt 0) { + if ($parseErrors.Count -gt 0) { 'telemetry-partial' } else { 'telemetry-available' } +} elseif ($handshakeMarkers.Count -gt 0) { + 'telemetry-handshake-only' +} else { + 'telemetry-missing' +} + +$lastKnownPhase = $null +$lastKnownPhaseSource = $null +$lastKnownStatus = $null +if ($null -ne $lastEvent -and ($null -eq $lastHandshakeAt -or $lastEventAt -ge $lastHandshakeAt)) { + $lastKnownPhase = $lastEvent.phase + $lastKnownPhaseSource = 'dispatcher-events' +} elseif ($null -ne $lastHandshake) { + $lastKnownPhase = $lastHandshake.phase + $lastKnownPhaseSource = 'handshake' + $lastKnownStatus = $lastHandshake.status +} + +$report = [ordered]@{ + schema = 'pester-execution-telemetry@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + resultsDir = $resolvedResultsDir + telemetryStatus = $telemetryStatus + dispatcherEventsPath = if (Test-Path -LiteralPath $eventsPath -PathType Leaf) { $eventsPath } else { $null } + sessionIndexPath = if (Test-Path -LiteralPath $sessionIndexPath -PathType Leaf) { $sessionIndexPath } else { $null } + summaryPath = if (Test-Path -LiteralPath $summaryPath -PathType Leaf) { $summaryPath } else { $null } + executionPack = $identity.executionPack + executionPackSource = $identity.executionPackSource + integrationMode = $identity.integrationMode + integrationSource = $identity.integrationSource + eventCount = $events.Count + parseErrorCount = $parseErrors.Count + firstEventAtUtc = if ($firstEventAt) { $firstEventAt.ToUniversalTime().ToString('o') } else { $null } + lastEventAtUtc = if ($lastEventAt) { $lastEventAt.ToUniversalTime().ToString('o') } else { $null } + lastKnownPhase = $lastKnownPhase + lastKnownPhaseSource = $lastKnownPhaseSource + lastKnownStatus = $lastKnownStatus + lastEvent = $lastEvent + phases = @($phaseTable.Values) + handshake = [ordered]@{ + count = $handshakeMarkers.Count + lastPhase = if ($lastHandshake) { $lastHandshake.phase } else { $null } + lastStatus = if ($lastHandshake) { $lastHandshake.status } else { $null } + lastAtUtc = if ($lastHandshake) { $lastHandshake.atUtc } else { $null } + markerPaths = @($handshakeMarkers | ForEach-Object { $_.path }) + } + parseErrors = @($parseErrors) +} + +Write-JsonFile -PathValue $telemetryPath -Payload $report + +if ($env:GITHUB_OUTPUT) { + "path=$telemetryPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "status=$telemetryStatus" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "last_known_phase=$lastKnownPhase" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "event_count=$($events.Count)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 +} + +Write-Host '### Pester execution telemetry' -ForegroundColor Cyan +Write-Host ("status : {0}" -f $telemetryStatus) +Write-Host ("events : {0}" -f $events.Count) +Write-Host ("last phase : {0}" -f $lastKnownPhase) +Write-Host ("report : {0}" -f $telemetryPath) + +exit 0 diff --git a/tools/Invoke-PesterOperatorOutcome.ps1 b/tools/Invoke-PesterOperatorOutcome.ps1 new file mode 100644 index 000000000..fddaf9b1f --- /dev/null +++ b/tools/Invoke-PesterOperatorOutcome.ps1 @@ -0,0 +1,220 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $false)] + [string]$ResultsDir = 'tests/results', + + [Parameter(Mandatory = $false)] + [string]$ClassificationPath = 'pester-evidence-classification.json', + + [Parameter(Mandatory = $false)] + [string]$SummaryPath = 'pester-summary.json', + + [Parameter(Mandatory = $false)] + [string]$OutputPath = 'pester-operator-outcome.json', + + [Parameter(Mandatory = $false)] + [string]$ContinueOnError = 'false' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Resolve-OptionalPath { + param( + [Parameter(Mandatory = $true)][string]$BasePath, + [string]$PathValue + ) + + if ([string]::IsNullOrWhiteSpace($PathValue)) { + return $null + } + if ([System.IO.Path]::IsPathRooted($PathValue)) { + return [System.IO.Path]::GetFullPath($PathValue) + } + return [System.IO.Path]::GetFullPath((Join-Path $BasePath $PathValue)) +} + +function Read-JsonObject { + param([Parameter(Mandatory = $true)][string]$PathValue) + if (-not (Test-Path -LiteralPath $PathValue -PathType Leaf)) { + throw "JSON file not found: $PathValue" + } + return (Get-Content -LiteralPath $PathValue -Raw | ConvertFrom-Json -ErrorAction Stop) +} + +function ConvertTo-Bool { + param($Value) + if ($Value -is [bool]) { + return $Value + } + if ($null -eq $Value) { + return $false + } + return ([string]$Value).Trim().ToLowerInvariant() -in @('1', 'true', 'yes', 'on') +} + +function Get-OperatorOutcomeDescriptor { + param( + [Parameter(Mandatory = $true)][string]$Classification, + [Parameter(Mandatory = $true)][bool]$ContinueOnError + ) + + switch ($Classification) { + 'ok' { + return [pscustomobject]@{ + gateStatus = 'pass' + headline = 'Pester gate passed.' + nextActionId = 'no-action' + nextAction = 'No action required.' + actionContext = @('tests/results/pester-summary.json') + } + } + 'context-blocked' { + return [pscustomobject]@{ + gateStatus = if ($ContinueOnError) { 'notice' } else { 'fail' } + headline = 'Pester gate blocked before execution by context debt.' + nextActionId = 'inspect-context-receipt' + nextAction = 'Inspect the context receipt and trusted-routing inputs before rerunning the gate.' + actionContext = @('tests/execution-contract/pester-run-receipt.json', 'tests/results/pester-evidence-classification.json') + } + } + 'readiness-blocked' { + return [pscustomobject]@{ + gateStatus = if ($ContinueOnError) { 'notice' } else { 'fail' } + headline = 'Pester gate blocked by self-hosted readiness debt.' + nextActionId = 'inspect-readiness-receipt' + nextAction = 'Inspect the readiness receipt and ingress-host probe results before rerunning the gate.' + actionContext = @('tests/execution-contract/pester-run-receipt.json', 'tests/results/pester-evidence-classification.json') + } + } + 'selection-blocked' { + return [pscustomobject]@{ + gateStatus = if ($ContinueOnError) { 'notice' } else { 'fail' } + headline = 'Pester gate blocked by selection-contract debt.' + nextActionId = 'inspect-selection-receipt' + nextAction = 'Inspect the selection receipt, named execution pack, and include-pattern refinement before rerunning the gate.' + actionContext = @('tests/execution-contract/pester-run-receipt.json', 'tests/results/pester-evidence-classification.json') + } + } + 'results-xml-truncated' { + return [pscustomobject]@{ + gateStatus = if ($ContinueOnError) { 'notice' } else { 'fail' } + headline = 'Pester gate produced truncated XML results.' + nextActionId = 'inspect-results-xml-truncation' + nextAction = 'Inspect pester-execution-postprocess.json and raw pester-results.xml to resolve XML truncation before trusting summary output.' + actionContext = @('tests/results/pester-execution-postprocess.json', 'tests/results/pester-results.xml', 'tests/results/pester-evidence-classification.json') + } + } + 'invalid-results-xml' { + return [pscustomobject]@{ + gateStatus = if ($ContinueOnError) { 'notice' } else { 'fail' } + headline = 'Pester gate produced invalid XML results.' + nextActionId = 'inspect-invalid-results-xml' + nextAction = 'Inspect pester-execution-postprocess.json and raw pester-results.xml to resolve malformed result XML before trusting summary output.' + actionContext = @('tests/results/pester-execution-postprocess.json', 'tests/results/pester-results.xml', 'tests/results/pester-evidence-classification.json') + } + } + 'missing-results-xml' { + return [pscustomobject]@{ + gateStatus = if ($ContinueOnError) { 'notice' } else { 'fail' } + headline = 'Pester gate completed without result XML.' + nextActionId = 'inspect-missing-results-xml' + nextAction = 'Inspect dispatcher outputs, raw-artifact staging, and execution-post reports to determine why pester-results.xml was not produced.' + actionContext = @('tests/results/pester-execution-postprocess.json', 'tests/results/pester-dispatcher.log', 'tests/results/pester-evidence-classification.json') + } + } + 'unsupported-schema' { + return [pscustomobject]@{ + gateStatus = if ($ContinueOnError) { 'notice' } else { 'fail' } + headline = 'Pester gate encountered an unsupported retained-artifact schema.' + nextActionId = 'reconcile-schema-contract' + nextAction = 'Regenerate retained artifacts with the supported schema contract or update readers before rerunning the gate.' + actionContext = @('tests/results/pester-evidence-classification.json', 'tests/execution-contract/pester-run-receipt.json', 'tests/results/pester-summary.json') + } + } + 'test-failures' { + return [pscustomobject]@{ + gateStatus = if ($ContinueOnError) { 'notice' } else { 'fail' } + headline = 'Pester gate completed and reported test failures.' + nextActionId = 'review-top-failures' + nextAction = 'Review pester-failures.json, the top-failures summary, and the failing test names before deciding whether to rerun or fix source.' + actionContext = @('tests/results/pester-failures.json', 'tests/results/pester-summary.json', 'tests/results/pester-evidence-classification.json') + } + } + default { + return [pscustomobject]@{ + gateStatus = if ($ContinueOnError) { 'notice' } else { 'fail' } + headline = 'Pester gate ended with orchestration or evidence debt.' + nextActionId = 'inspect-execution-evidence' + nextAction = 'Inspect the execution receipt, telemetry, raw artifacts, and evidence classification to isolate the real failing seam.' + actionContext = @('tests/execution-contract/pester-run-receipt.json', 'tests/results/pester-execution-telemetry.json', 'tests/results/pester-evidence-classification.json') + } + } + } +} + +$resolvedResultsDir = [System.IO.Path]::GetFullPath($ResultsDir) +if (-not (Test-Path -LiteralPath $resolvedResultsDir -PathType Container)) { + New-Item -ItemType Directory -Path $resolvedResultsDir -Force | Out-Null +} + +$resolvedClassificationPath = Resolve-OptionalPath -BasePath $resolvedResultsDir -PathValue $ClassificationPath +$resolvedSummaryPath = Resolve-OptionalPath -BasePath $resolvedResultsDir -PathValue $SummaryPath +$resolvedOutputPath = Resolve-OptionalPath -BasePath $resolvedResultsDir -PathValue $OutputPath +if (-not $resolvedOutputPath) { + $resolvedOutputPath = Join-Path $resolvedResultsDir 'pester-operator-outcome.json' +} + +$classification = Read-JsonObject -PathValue $resolvedClassificationPath +$summary = if ($resolvedSummaryPath -and (Test-Path -LiteralPath $resolvedSummaryPath -PathType Leaf)) { + Read-JsonObject -PathValue $resolvedSummaryPath +} else { + $null +} + +$descriptor = Get-OperatorOutcomeDescriptor -Classification ([string]$classification.classification) -ContinueOnError (ConvertTo-Bool $ContinueOnError) +$reasons = @($classification.reasons) +$executionPack = if ($classification.PSObject.Properties.Name -contains 'selectionExecutionPack') { [string]$classification.selectionExecutionPack } else { '' } +$payload = [ordered]@{ + schema = 'pester-operator-outcome@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + gateStatus = [string]$descriptor.gateStatus + continueOnError = ConvertTo-Bool $ContinueOnError + classification = [string]$classification.classification + headline = [string]$descriptor.headline + nextActionId = [string]$descriptor.nextActionId + nextAction = [string]$descriptor.nextAction + reasons = $reasons + reasonCount = @($reasons).Count + actionContext = @($descriptor.actionContext) + summaryPresent = [bool]$summary + selectionExecutionPack = $executionPack + contextStatus = [string]$classification.contextStatus + readinessStatus = [string]$classification.readinessStatus + selectionStatus = [string]$classification.selectionStatus + rawArtifactDownload = [string]$classification.rawArtifactDownload + dispatcherExitCode = if ($classification.PSObject.Properties.Name -contains 'dispatcherExitCode') { [int]$classification.dispatcherExitCode } else { -1 } + total = if ($summary -and $summary.PSObject.Properties.Name -contains 'total') { [int]$summary.total } else { 0 } + failed = if ($summary -and $summary.PSObject.Properties.Name -contains 'failed') { [int]$summary.failed } else { 0 } + errors = if ($summary -and $summary.PSObject.Properties.Name -contains 'errors') { [int]$summary.errors } else { 0 } + classificationPath = $resolvedClassificationPath + summaryPath = $resolvedSummaryPath +} + +$payload | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $resolvedOutputPath -Encoding UTF8 + +if ($env:GITHUB_OUTPUT) { + "gate_status=$($payload.gateStatus)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "classification=$($payload.classification)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "next_action_id=$($payload.nextActionId)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "next_action=$($payload.nextAction)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "path=$resolvedOutputPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 +} + +Write-Host '### Pester operator outcome' -ForegroundColor Cyan +Write-Host ("gateStatus : {0}" -f $payload.gateStatus) +Write-Host ("classification: {0}" -f $payload.classification) +Write-Host ("nextActionId : {0}" -f $payload.nextActionId) +Write-Host ("path : {0}" -f $resolvedOutputPath) + +exit 0 diff --git a/tools/Invoke-PesterWindowsContainerSurfaceProbe.ps1 b/tools/Invoke-PesterWindowsContainerSurfaceProbe.ps1 new file mode 100644 index 000000000..2a5a02ef7 --- /dev/null +++ b/tools/Invoke-PesterWindowsContainerSurfaceProbe.ps1 @@ -0,0 +1,154 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $false)] + [string]$ResultsDir = 'tests/results/pester-windows-container-surface', + + [Parameter(Mandatory = $false)] + [string]$OutputPath = 'pester-windows-container-surface.json', + + [Parameter(Mandatory = $false)] + [string]$PinnedImage = 'nationalinstruments/labview:2026q1-windows', + + [Parameter(Mandatory = $false)] + [string]$HostPlatformOverride = '', + + [Parameter(Mandatory = $false)] + [string]$DockerServerJson = '', + + [Parameter(Mandatory = $false)] + [string]$ImageInspectJson = '' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Resolve-OptionalPath { + param( + [Parameter(Mandatory = $true)][string]$BasePath, + [string]$PathValue + ) + + if ([string]::IsNullOrWhiteSpace($PathValue)) { + return $null + } + if ([System.IO.Path]::IsPathRooted($PathValue)) { + return [System.IO.Path]::GetFullPath($PathValue) + } + return [System.IO.Path]::GetFullPath((Join-Path $BasePath $PathValue)) +} + +function Read-JsonFromSource { + param( + [string]$InlineJson, + [string]$FilePath + ) + + if (-not [string]::IsNullOrWhiteSpace($InlineJson)) { + return $InlineJson | ConvertFrom-Json -ErrorAction Stop + } + if (-not [string]::IsNullOrWhiteSpace($FilePath) -and (Test-Path -LiteralPath $FilePath -PathType Leaf)) { + return Get-Content -LiteralPath $FilePath -Raw | ConvertFrom-Json -ErrorAction Stop + } + return $null +} + +$resolvedResultsDir = [System.IO.Path]::GetFullPath($ResultsDir) +if (-not (Test-Path -LiteralPath $resolvedResultsDir -PathType Container)) { + New-Item -ItemType Directory -Path $resolvedResultsDir -Force | Out-Null +} +$resolvedOutputPath = Resolve-OptionalPath -BasePath $resolvedResultsDir -PathValue $OutputPath +if (-not $resolvedOutputPath) { + $resolvedOutputPath = Join-Path $resolvedResultsDir 'pester-windows-container-surface.json' +} + +$hostPlatform = if (-not [string]::IsNullOrWhiteSpace($HostPlatformOverride)) { + $HostPlatformOverride +} else { + [System.Environment]::OSVersion.Platform.ToString() +} + +$status = 'unknown' +$reason = 'probe-not-run' +$server = $null +$imageInfo = $null +$dockerCommand = Get-Command docker -ErrorAction SilentlyContinue + +if ($hostPlatform -notmatch 'Win') { + $status = 'not-windows-host' + $reason = 'surface-requires-windows-host' +} elseif (-not $dockerCommand -and [string]::IsNullOrWhiteSpace($DockerServerJson)) { + $status = 'docker-cli-missing' + $reason = 'docker-cli-not-found' +} else { + try { + if ([string]::IsNullOrWhiteSpace($DockerServerJson)) { + $server = docker version --format '{{json .Server}}' | ConvertFrom-Json -ErrorAction Stop + } else { + $server = Read-JsonFromSource -InlineJson $DockerServerJson + } + } catch { + $status = 'docker-unavailable' + $reason = 'docker-server-query-failed' + } + + if ($server) { + $osType = if ($server.PSObject.Properties.Name -contains 'Os') { [string]$server.Os } elseif ($server.PSObject.Properties.Name -contains 'OSType') { [string]$server.OSType } else { '' } + if ($osType -ne 'windows') { + $status = 'docker-engine-not-windows' + $reason = 'docker-server-not-windows' + } else { + try { + if ([string]::IsNullOrWhiteSpace($ImageInspectJson)) { + $inspectRaw = docker image inspect $PinnedImage --format '{{json .}}' 2>$null + if ([string]::IsNullOrWhiteSpace($inspectRaw)) { + $status = 'ni-image-missing' + $reason = 'pinned-image-not-present' + } else { + $imageInfo = $inspectRaw | ConvertFrom-Json -ErrorAction Stop + $status = 'ready' + $reason = 'windows-container-surface-ready' + } + } else { + $imageInfo = Read-JsonFromSource -InlineJson $ImageInspectJson + $status = 'ready' + $reason = 'windows-container-surface-ready' + } + } catch { + $status = 'ni-image-missing' + $reason = 'pinned-image-not-present' + } + } + } +} + +$payload = [ordered]@{ + schema = 'pester-windows-container-surface@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + hostPlatform = [string]$hostPlatform + status = [string]$status + reason = [string]$reason + pinnedImage = [string]$PinnedImage + recommendedCommands = @( + 'npm run docker:ni:windows:bootstrap', + 'npm run compare:docker:ni:windows:probe', + 'npm run compare:docker:ni:windows' + ) + dockerServer = if ($server) { $server } else { $null } + pinnedImagePresent = [bool]$imageInfo + pinnedImageInfo = if ($imageInfo) { $imageInfo } else { $null } +} + +$payload | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $resolvedOutputPath -Encoding UTF8 + +if ($env:GITHUB_OUTPUT) { + "status=$($payload.status)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "reason=$($payload.reason)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "path=$resolvedOutputPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 +} + +Write-Host '### Pester Windows container surface' -ForegroundColor Cyan +Write-Host ("status : {0}" -f $payload.status) +Write-Host ("reason : {0}" -f $payload.reason) +Write-Host ("path : {0}" -f $resolvedOutputPath) + +exit 0 diff --git a/tools/PesterExecutionPacks.ps1 b/tools/PesterExecutionPacks.ps1 new file mode 100644 index 000000000..31d0a5fed --- /dev/null +++ b/tools/PesterExecutionPacks.ps1 @@ -0,0 +1,154 @@ +Set-StrictMode -Version Latest + +function Get-PesterExecutionPackCatalog { + $catalog = [ordered]@{ + full = [ordered]@{ + name = 'full' + description = 'Full Pester suite' + includePatterns = @() + aliases = @('all', 'default') + } + comparevi = [ordered]@{ + name = 'comparevi' + description = 'CompareVI contract and CLI coverage' + includePatterns = @('CompareVI*.ps1', 'CanonicalCli.Tests.ps1', 'Args.Tokenization.Tests.ps1') + aliases = @('compare-vi') + } + dispatcher = [ordered]@{ + name = 'dispatcher' + description = 'Dispatcher, nested execution, and invoker coverage' + includePatterns = @('Invoke-PesterTests*.ps1', 'PesterAvailability.Tests.ps1', 'NestedDispatcher*.Tests.ps1') + aliases = @('dispatch') + } + workflow = [ordered]@{ + name = 'workflow' + description = 'Workflow, artifact, and orchestration coverage' + includePatterns = @( + 'Workflow*.ps1', + 'On-FixtureValidationFail.Tests.ps1', + 'Watch.FlakyRecovery.Tests.ps1', + 'FunctionShadowing*.ps1', + 'FunctionProxy.Tests.ps1', + 'RunSummary.Tool*.ps1', + 'Action.CompositeOutputs.Tests.ps1', + 'Binding.MinRepro.Tests.ps1', + 'ArtifactTracking*.ps1', + 'Guard.*.Tests.ps1' + ) + aliases = @('orchestration') + } + fixtures = [ordered]@{ + name = 'fixtures' + description = 'Fixture validation and fixture-driven comparison coverage' + includePatterns = @( + 'Fixtures.*.ps1', + 'FixtureValidation*.ps1', + 'FixtureSummary*.ps1', + 'ViBinaryHandling.Tests.ps1', + 'FixtureValidationDiff.Tests.ps1' + ) + aliases = @('fixture') + } + psummary = [ordered]@{ + name = 'psummary' + description = 'Pester summary and failure-detail rendering coverage' + includePatterns = @('PesterSummary*.ps1', 'Write-PesterSummaryToStepSummary*.ps1', 'AggregationHints*.ps1') + aliases = @('summary') + } + schema = [ordered]@{ + name = 'schema' + description = 'Schema and schema-lite validation coverage' + includePatterns = @('Schema.*.ps1', 'SchemaLite*.ps1') + aliases = @('schemas') + } + loop = [ordered]@{ + name = 'loop' + description = 'Loop and autonomous integration control coverage' + includePatterns = @('CompareLoop*.ps1', 'Run-AutonomousIntegrationLoop*.ps1', 'LoopMetrics.Tests.ps1', 'Integration-ControlLoop*.ps1', 'IntegrationControlLoop*.ps1') + aliases = @('control-loop') + } + } + + return $catalog +} + +function Get-PesterExecutionPackNames { + return @((Get-PesterExecutionPackCatalog).Keys) +} + +function ConvertTo-PesterExecutionPackPatterns { + param( + [AllowNull()] + [AllowEmptyCollection()] + [object]$Patterns + ) + + $tokens = New-Object System.Collections.Generic.List[string] + foreach ($candidate in @($Patterns)) { + if ($null -eq $candidate) { continue } + foreach ($segment in ([string]$candidate -split "[`r`n,;]")) { + $token = $segment.Trim() + if ([string]::IsNullOrWhiteSpace($token)) { continue } + if (-not $tokens.Contains($token)) { + $tokens.Add($token) | Out-Null + } + } + } + + return @($tokens.ToArray()) +} + +function Resolve-PesterExecutionPack { + [CmdletBinding()] + param( + [AllowEmptyString()] + [string]$ExecutionPack = 'full', + [AllowNull()] + [AllowEmptyCollection()] + [object]$RefineIncludePatterns + ) + + $catalog = Get-PesterExecutionPackCatalog + $requestedPack = if ([string]::IsNullOrWhiteSpace($ExecutionPack)) { 'full' } else { $ExecutionPack.Trim().ToLowerInvariant() } + $resolved = $null + + foreach ($entry in $catalog.GetEnumerator()) { + $candidate = $entry.Value + if ($requestedPack -eq $candidate.name) { + $resolved = $candidate + break + } + if (@($candidate.aliases) -contains $requestedPack) { + $resolved = $candidate + break + } + } + + if ($null -eq $resolved) { + $supported = @( + $catalog.Values | + ForEach-Object { $_.name } | + Sort-Object + ) -join ', ' + throw "Unsupported execution pack '$ExecutionPack'. Supported packs: $supported" + } + + $basePatterns = ConvertTo-PesterExecutionPackPatterns -Patterns $resolved.includePatterns + $refinePatterns = ConvertTo-PesterExecutionPackPatterns -Patterns $RefineIncludePatterns + $effectivePatterns = New-Object System.Collections.Generic.List[string] + foreach ($token in @($basePatterns + $refinePatterns)) { + if (-not [string]::IsNullOrWhiteSpace($token) -and -not $effectivePatterns.Contains($token)) { + $effectivePatterns.Add($token) | Out-Null + } + } + + return [pscustomobject]@{ + executionPack = [string]$resolved.name + executionPackSource = if ([string]::IsNullOrWhiteSpace($ExecutionPack)) { 'default' } else { 'declared' } + executionPackDescription = [string]$resolved.description + executionPackAliases = @($resolved.aliases) + baseIncludePatterns = @($basePatterns) + refineIncludePatterns = @($refinePatterns) + effectiveIncludePatterns = @($effectivePatterns.ToArray()) + } +} diff --git a/tools/PesterFailurePayload.ps1 b/tools/PesterFailurePayload.ps1 new file mode 100644 index 000000000..d8f365079 --- /dev/null +++ b/tools/PesterFailurePayload.ps1 @@ -0,0 +1,317 @@ +Set-StrictMode -Version Latest + +function Get-PesterFailurePropertyValue { + param( + $InputObject, + [string[]]$PropertyNames + ) + + if (-not $InputObject) { return $null } + + foreach ($name in $PropertyNames) { + if ([string]::IsNullOrWhiteSpace($name)) { continue } + + if ($InputObject -is [hashtable]) { + if ($InputObject.ContainsKey($name)) { return $InputObject[$name] } + continue + } + + $prop = $InputObject.PSObject.Properties[$name] + if ($prop) { return $prop.Value } + } + + return $null +} + +function Set-PesterFailureObjectProperty { + param( + [Parameter(Mandatory = $true)]$InputObject, + [Parameter(Mandatory = $true)][string]$Name, + $Value + ) + + $property = $InputObject.PSObject.Properties[$Name] + if ($property) { + $property.Value = $Value + } else { + Add-Member -InputObject $InputObject -Name $Name -MemberType NoteProperty -Value $Value + } +} + +function Get-PesterFailureEntries { + param($FailurePayload) + + if (-not $FailurePayload) { return @() } + + $resultsProp = Get-PesterFailurePropertyValue -InputObject $FailurePayload -PropertyNames @('results') + if ($null -ne $resultsProp) { + return @($resultsProp) + } + + if ((Get-PesterFailurePropertyValue -InputObject $FailurePayload -PropertyNames @('name','Name')) -or + (Get-PesterFailurePropertyValue -InputObject $FailurePayload -PropertyNames @('result','Result'))) { + return @($FailurePayload) + } + + if ($FailurePayload -is [System.Array] -or ($FailurePayload -is [System.Collections.IEnumerable] -and $FailurePayload -isnot [string])) { + return @($FailurePayload) + } + + return @() +} + +function ConvertTo-PesterFailureEntry { + param($Entry) + + if (-not $Entry) { return $null } + + $nameValue = [string](Get-PesterFailurePropertyValue -InputObject $Entry -PropertyNames @('name','Name')) + $resultValue = [string](Get-PesterFailurePropertyValue -InputObject $Entry -PropertyNames @('result','Result')) + if ([string]::IsNullOrWhiteSpace($resultValue)) { + $resultValue = 'Failed' + } + + $durationSeconds = Get-PesterFailurePropertyValue -InputObject $Entry -PropertyNames @('duration','Duration') + $durationMilliseconds = Get-PesterFailurePropertyValue -InputObject $Entry -PropertyNames @('duration_ms','DurationMs','durationMilliseconds') + if ($null -eq $durationSeconds -and $null -ne $durationMilliseconds -and "$durationMilliseconds" -match '^-?\d+(\.\d+)?$') { + $durationSeconds = [math]::Round(([double]$durationMilliseconds / 1000), 6) + } + if ($null -eq $durationMilliseconds -and $null -ne $durationSeconds -and "$durationSeconds" -match '^-?\d+(\.\d+)?$') { + $durationMilliseconds = [math]::Round(([double]$durationSeconds * 1000), 2) + } + + $pathValue = [string](Get-PesterFailurePropertyValue -InputObject $Entry -PropertyNames @('path','Path')) + $fileValue = [string](Get-PesterFailurePropertyValue -InputObject $Entry -PropertyNames @('file','File')) + if ([string]::IsNullOrWhiteSpace($fileValue) -and -not [string]::IsNullOrWhiteSpace($pathValue)) { + $fileValue = $pathValue + } + if ([string]::IsNullOrWhiteSpace($pathValue) -and -not [string]::IsNullOrWhiteSpace($fileValue)) { + $pathValue = $fileValue + } + + $lineValue = Get-PesterFailurePropertyValue -InputObject $Entry -PropertyNames @('line','Line') + $messageValue = [string](Get-PesterFailurePropertyValue -InputObject $Entry -PropertyNames @('message','Message')) + if ([string]::IsNullOrWhiteSpace($nameValue)) { + if (-not [string]::IsNullOrWhiteSpace($pathValue)) { + $nameValue = $pathValue + } elseif (-not [string]::IsNullOrWhiteSpace($messageValue)) { + $nameValue = $messageValue + } else { + $nameValue = 'Failure' + } + } + + return [pscustomobject]@{ + name = $nameValue + result = $resultValue + duration = $durationSeconds + duration_ms = $durationMilliseconds + path = $pathValue + file = $fileValue + line = $lineValue + message = $messageValue + } +} + +function Get-PesterFailureSummaryCounts { + param($Summary) + + $total = Get-PesterFailurePropertyValue -InputObject $Summary -PropertyNames @('total','Total') + $failed = Get-PesterFailurePropertyValue -InputObject $Summary -PropertyNames @('failed','Failed') + $errors = Get-PesterFailurePropertyValue -InputObject $Summary -PropertyNames @('errors','Errors') + $skipped = Get-PesterFailurePropertyValue -InputObject $Summary -PropertyNames @('skipped','Skipped') + + return [pscustomobject]@{ + total = if ($null -ne $total -and "$total" -match '^-?\d+$') { [int]$total } else { 0 } + failed = if ($null -ne $failed -and "$failed" -match '^-?\d+$') { [int]$failed } else { 0 } + errors = if ($null -ne $errors -and "$errors" -match '^-?\d+$') { [int]$errors } else { 0 } + skipped = if ($null -ne $skipped -and "$skipped" -match '^-?\d+$') { [int]$skipped } else { 0 } + } +} + +function Read-PesterFailurePayloadFile { + param([Parameter(Mandatory = $true)][string]$PathValue) + + if (-not (Test-Path -LiteralPath $PathValue -PathType Leaf)) { + return [pscustomobject]@{ + present = $false + parseStatus = 'missing' + payload = $null + parseError = $null + } + } + + try { + $payload = Get-Content -LiteralPath $PathValue -Raw | ConvertFrom-Json -ErrorAction Stop + return [pscustomobject]@{ + present = $true + parseStatus = 'parsed' + payload = $payload + parseError = $null + } + } catch { + return [pscustomobject]@{ + present = $true + parseStatus = 'unparseable' + payload = $null + parseError = [string]$_.Exception.Message + } + } +} + +function Resolve-PesterFailureUnavailableReason { + param( + $Summary, + [string]$ParseStatus = 'parsed' + ) + + $summaryReason = [string](Get-PesterFailurePropertyValue -InputObject $Summary -PropertyNames @('failureDetailsReason')) + if (-not [string]::IsNullOrWhiteSpace($summaryReason)) { + return $summaryReason + } + + $resultsXmlStatus = [string](Get-PesterFailurePropertyValue -InputObject $Summary -PropertyNames @('resultsXmlStatus')) + $executionPostprocessStatus = [string](Get-PesterFailurePropertyValue -InputObject $Summary -PropertyNames @('executionPostprocessStatus')) + if ($executionPostprocessStatus -eq 'results-xml-truncated' -or $resultsXmlStatus -like 'truncated*') { + return 'results-xml-truncated' + } + if ($executionPostprocessStatus -eq 'invalid-results-xml' -or $resultsXmlStatus -like 'invalid*') { + return 'invalid-results-xml' + } + if ($executionPostprocessStatus -eq 'missing-results-xml' -or $resultsXmlStatus -eq 'missing') { + return 'missing-results-xml' + } + if ($ParseStatus -eq 'unparseable') { + return 'failure-payload-unparseable' + } + + return 'failure-details-unavailable' +} + +function Get-PesterFailureDetailState { + param( + $FailurePayload, + $Summary + ) + + $counts = Get-PesterFailureSummaryCounts -Summary $Summary + $entries = @(Get-PesterFailureEntries -FailurePayload $FailurePayload | ForEach-Object { ConvertTo-PesterFailureEntry -Entry $_ } | Where-Object { $null -ne $_ }) + $detailStatus = [string](Get-PesterFailurePropertyValue -InputObject $FailurePayload -PropertyNames @('detailStatus')) + $unavailableReason = [string](Get-PesterFailurePropertyValue -InputObject $FailurePayload -PropertyNames @('unavailableReason')) + + if ([string]::IsNullOrWhiteSpace($detailStatus)) { + if ($entries.Count -gt 0) { + $detailStatus = 'available' + } elseif (($counts.failed + $counts.errors) -gt 0) { + $detailStatus = 'unavailable' + } else { + $detailStatus = 'not-applicable' + } + } + + if ($detailStatus -eq 'unavailable' -and [string]::IsNullOrWhiteSpace($unavailableReason)) { + $unavailableReason = Resolve-PesterFailureUnavailableReason -Summary $Summary + } + + return [pscustomobject]@{ + detailStatus = $detailStatus + unavailableReason = $unavailableReason + detailCount = $entries.Count + entries = @($entries) + summaryCounts = $counts + } +} + +function ConvertTo-PesterFailurePayload { + param( + $FailurePayload, + $Summary, + [string]$SchemaVersion = '1.1.0', + [string]$ParseStatus = 'parsed', + [string]$ParseError + ) + + $state = Get-PesterFailureDetailState -FailurePayload $FailurePayload -Summary $Summary + $payload = [ordered]@{ + schema = 'pester-failures@v2' + schemaVersion = $SchemaVersion + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + detailStatus = $state.detailStatus + detailCount = [int]$state.detailCount + summary = [ordered]@{ + total = [int]$state.summaryCounts.total + failed = [int]$state.summaryCounts.failed + errors = [int]$state.summaryCounts.errors + skipped = [int]$state.summaryCounts.skipped + } + results = @($state.entries) + } + + if ($state.detailStatus -eq 'unavailable' -and -not [string]::IsNullOrWhiteSpace($state.unavailableReason)) { + $payload['unavailableReason'] = $state.unavailableReason + } + if ($ParseStatus -eq 'unparseable' -and -not [string]::IsNullOrWhiteSpace($ParseError)) { + $payload['sourceParseStatus'] = $ParseStatus + $payload['sourceParseError'] = $ParseError + } + + return [pscustomobject]$payload +} + +function Update-PesterSummaryWithFailurePayload { + param( + [Parameter(Mandatory = $true)]$SummaryObject, + [Parameter(Mandatory = $true)]$FailurePayload + ) + + $state = Get-PesterFailureDetailState -FailurePayload $FailurePayload -Summary $SummaryObject + Set-PesterFailureObjectProperty -InputObject $SummaryObject -Name 'failureDetailsStatus' -Value $state.detailStatus + Set-PesterFailureObjectProperty -InputObject $SummaryObject -Name 'failureDetailsCount' -Value ([int]$state.detailCount) + if ($state.detailStatus -eq 'unavailable' -and -not [string]::IsNullOrWhiteSpace($state.unavailableReason)) { + Set-PesterFailureObjectProperty -InputObject $SummaryObject -Name 'failureDetailsReason' -Value $state.unavailableReason + } elseif ($SummaryObject.PSObject.Properties['failureDetailsReason']) { + $SummaryObject.PSObject.Properties.Remove('failureDetailsReason') + } + if ($FailurePayload.PSObject.Properties['schemaVersion']) { + Set-PesterFailureObjectProperty -InputObject $SummaryObject -Name 'failureDetailsSchemaVersion' -Value ([string]$FailurePayload.schemaVersion) + } +} + +function Sync-PesterFailurePayload { + param( + [Parameter(Mandatory = $true)][string]$Directory, + [Parameter(Mandatory = $true)]$SummaryObject, + [string]$SchemaVersion = '1.1.0' + ) + + if (-not (Test-Path -LiteralPath $Directory -PathType Container)) { + New-Item -ItemType Directory -Path $Directory -Force | Out-Null + } + + $pathValue = Join-Path $Directory 'pester-failures.json' + $existing = Read-PesterFailurePayloadFile -PathValue $pathValue + $payload = ConvertTo-PesterFailurePayload -FailurePayload $existing.payload -Summary $SummaryObject -SchemaVersion $SchemaVersion -ParseStatus $existing.parseStatus -ParseError $existing.parseError + $payload | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $pathValue -Encoding UTF8 + Update-PesterSummaryWithFailurePayload -SummaryObject $SummaryObject -FailurePayload $payload + return $payload +} + +function Write-PesterFailurePayload { + param( + [Parameter(Mandatory = $true)][string]$PathValue, + [Parameter(Mandatory = $true)]$SummaryObject, + [AllowNull()] + [AllowEmptyCollection()] + [object]$FailureEntries, + [string]$SchemaVersion = '1.1.0' + ) + + $directory = Split-Path -Parent $PathValue + if ($directory -and -not (Test-Path -LiteralPath $directory -PathType Container)) { + New-Item -ItemType Directory -Path $directory -Force | Out-Null + } + + $payload = ConvertTo-PesterFailurePayload -FailurePayload $FailureEntries -Summary $SummaryObject -SchemaVersion $SchemaVersion + $payload | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $PathValue -Encoding UTF8 + return $payload +} diff --git a/tools/PesterPathHygiene.ps1 b/tools/PesterPathHygiene.ps1 new file mode 100644 index 000000000..c5cae47d6 --- /dev/null +++ b/tools/PesterPathHygiene.ps1 @@ -0,0 +1,154 @@ +Set-StrictMode -Version Latest + +function ConvertTo-PesterPathHygienePortablePath { + param([string]$PathValue) + if ([string]::IsNullOrWhiteSpace($PathValue)) { + return $null + } + return ($PathValue -replace '\\', '/') +} + +function Get-PesterPathHygieneRisks { + param([string]$PathValue) + + if ([string]::IsNullOrWhiteSpace($PathValue)) { + return @() + } + + $normalized = ConvertTo-PesterPathHygienePortablePath -PathValue $PathValue + $rules = @( + [ordered]@{ + id = 'onedrive-managed-root' + pattern = '(?i)(^|[\\/])OneDrive(?:[\\/]|$|[\s-])' + message = 'Path appears to live under a OneDrive-managed root.' + } + [ordered]@{ + id = 'dropbox-managed-root' + pattern = '(?i)(^|[\\/])Dropbox([\\/]|$)' + message = 'Path appears to live under a Dropbox-managed root.' + } + [ordered]@{ + id = 'google-drive-managed-root' + pattern = '(?i)(^|[\\/])Google Drive([\\/]|$)' + message = 'Path appears to live under a Google Drive-managed root.' + } + [ordered]@{ + id = 'icloud-drive-managed-root' + pattern = '(?i)(^|[\\/])iCloud Drive([\\/]|$)' + message = 'Path appears to live under an iCloud Drive-managed root.' + } + ) + + $risks = New-Object System.Collections.Generic.List[object] + foreach ($rule in $rules) { + if ($normalized -match $rule.pattern) { + $risks.Add([pscustomobject]@{ + id = [string]$rule.id + path = $normalized + message = [string]$rule.message + }) | Out-Null + } + } + + return @($risks.ToArray()) +} + +function Resolve-PesterPathHygienePlan { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)][string]$ResultsPath, + [Parameter(Mandatory = $true)][string]$SessionLockRoot, + [ValidateSet('auto', 'relocate', 'block', 'off')] + [string]$Mode = 'auto', + [string]$SafeRoot + ) + + $requestedResultsPath = [System.IO.Path]::GetFullPath($ResultsPath) + $requestedSessionLockRoot = [System.IO.Path]::GetFullPath($SessionLockRoot) + + $risks = New-Object System.Collections.Generic.List[object] + foreach ($risk in (Get-PesterPathHygieneRisks -PathValue $requestedResultsPath)) { + $risks.Add([pscustomobject]@{ + target = 'results' + id = $risk.id + path = $risk.path + message = $risk.message + }) | Out-Null + } + foreach ($risk in (Get-PesterPathHygieneRisks -PathValue $requestedSessionLockRoot)) { + $risks.Add([pscustomobject]@{ + target = 'session-lock' + id = $risk.id + path = $risk.path + message = $risk.message + }) | Out-Null + } + + if ($Mode -eq 'off' -or $risks.Count -eq 0) { + return [pscustomobject]@{ + mode = $Mode + status = 'clean' + requestedResultsPath = $requestedResultsPath + effectiveResultsPath = $requestedResultsPath + requestedSessionLockRoot = $requestedSessionLockRoot + effectiveSessionLockRoot = $requestedSessionLockRoot + receiptRoot = $requestedResultsPath + safeRoot = $null + risks = @($risks.ToArray()) + } + } + + $resolvedSafeRoot = if ([string]::IsNullOrWhiteSpace($SafeRoot)) { + Join-Path ([System.IO.Path]::GetTempPath()) ("compare-vi-cli-action-local-" + [Guid]::NewGuid().ToString('N')) + } else { + [System.IO.Path]::GetFullPath($SafeRoot) + } + $safeRootRisks = @(Get-PesterPathHygieneRisks -PathValue $resolvedSafeRoot) + if ($safeRootRisks.Count -gt 0) { + return [pscustomobject]@{ + mode = $Mode + status = 'path-hygiene-blocked' + requestedResultsPath = $requestedResultsPath + effectiveResultsPath = $null + requestedSessionLockRoot = $requestedSessionLockRoot + effectiveSessionLockRoot = $null + receiptRoot = $null + safeRoot = $resolvedSafeRoot + risks = @($risks.ToArray() + @( + [pscustomobject]@{ + target = 'safe-root' + id = 'unsafe-safe-root' + path = ConvertTo-PesterPathHygienePortablePath -PathValue $resolvedSafeRoot + message = 'Configured safe root is itself under a managed or synchronized path.' + } + )) + } + } + + $action = if ($Mode -eq 'block') { 'block' } else { 'relocate' } + if ($action -eq 'block') { + return [pscustomobject]@{ + mode = $Mode + status = 'path-hygiene-blocked' + requestedResultsPath = $requestedResultsPath + effectiveResultsPath = Join-Path $resolvedSafeRoot 'blocked-results' + requestedSessionLockRoot = $requestedSessionLockRoot + effectiveSessionLockRoot = Join-Path $resolvedSafeRoot 'blocked-session-lock' + receiptRoot = Join-Path $resolvedSafeRoot 'blocked-results' + safeRoot = $resolvedSafeRoot + risks = @($risks.ToArray()) + } + } + + return [pscustomobject]@{ + mode = $Mode + status = 'relocated' + requestedResultsPath = $requestedResultsPath + effectiveResultsPath = Join-Path $resolvedSafeRoot 'results' + requestedSessionLockRoot = $requestedSessionLockRoot + effectiveSessionLockRoot = Join-Path $resolvedSafeRoot 'session-lock' + receiptRoot = Join-Path $resolvedSafeRoot 'results' + safeRoot = $resolvedSafeRoot + risks = @($risks.ToArray()) + } +} diff --git a/tools/PesterServiceModelSchema.ps1 b/tools/PesterServiceModelSchema.ps1 new file mode 100644 index 000000000..6faafd9ad --- /dev/null +++ b/tools/PesterServiceModelSchema.ps1 @@ -0,0 +1,167 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Get-SchemaVersionMajor { + param([AllowNull()][AllowEmptyString()][string]$SchemaVersion) + + if ([string]::IsNullOrWhiteSpace($SchemaVersion)) { + return $null + } + + if ($SchemaVersion -match '^(?\d+)(?:\.\d+){0,2}$') { + return [int]$matches.major + } + + return $null +} + +function Read-PesterServiceModelJsonDocument { + param( + [Parameter(Mandatory = $true)][string]$PathValue, + [Parameter(Mandatory = $true)][string]$ContractName + ) + + if (-not (Test-Path -LiteralPath $PathValue -PathType Leaf)) { + return [pscustomobject]@{ + contractName = $ContractName + path = $PathValue + present = $false + valid = $false + classification = 'missing-file' + reason = "$ContractName-missing" + document = $null + actualSchema = $null + actualSchemaVersion = $null + parseError = $null + } + } + + try { + $document = Get-Content -LiteralPath $PathValue -Raw | ConvertFrom-Json -ErrorAction Stop + return [pscustomobject]@{ + contractName = $ContractName + path = $PathValue + present = $true + valid = $true + classification = 'ok' + reason = "$ContractName-ok" + document = $document + actualSchema = if ($document.PSObject.Properties.Name -contains 'schema') { [string]$document.schema } else { $null } + actualSchemaVersion = if ($document.PSObject.Properties.Name -contains 'schemaVersion') { [string]$document.schemaVersion } else { $null } + parseError = $null + } + } catch { + return [pscustomobject]@{ + contractName = $ContractName + path = $PathValue + present = $true + valid = $false + classification = 'unsupported-schema' + reason = "$ContractName-invalid-json" + document = $null + actualSchema = $null + actualSchemaVersion = $null + parseError = [string]$_.Exception.Message + } + } +} + +function Test-PesterServiceModelSchemaContract { + param( + [Parameter(Mandatory = $true)]$DocumentState, + [string]$ExpectedSchema, + [string]$SchemaVersionProperty = 'schemaVersion', + [int]$ExpectedSchemaVersionMajor = 0, + [switch]$RequireSchemaVersion + ) + + if (-not $DocumentState.present) { + return $DocumentState + } + + if (-not $DocumentState.valid) { + return $DocumentState + } + + $document = $DocumentState.document + $actualSchema = if ($document.PSObject.Properties.Name -contains 'schema') { [string]$document.schema } else { $null } + $actualSchemaVersion = if ($document.PSObject.Properties.Name -contains $SchemaVersionProperty) { [string]$document.$SchemaVersionProperty } else { $null } + + if (-not [string]::IsNullOrWhiteSpace($ExpectedSchema) -and $actualSchema -ne $ExpectedSchema) { + return [pscustomobject]@{ + contractName = $DocumentState.contractName + path = $DocumentState.path + present = $true + valid = $false + classification = 'unsupported-schema' + reason = "$($DocumentState.contractName)-unsupported-schema" + document = $document + actualSchema = $actualSchema + actualSchemaVersion = $actualSchemaVersion + parseError = $null + } + } + + if ($RequireSchemaVersion -or ($document.PSObject.Properties.Name -contains $SchemaVersionProperty)) { + if ([string]::IsNullOrWhiteSpace($actualSchemaVersion)) { + return [pscustomobject]@{ + contractName = $DocumentState.contractName + path = $DocumentState.path + present = $true + valid = $false + classification = 'unsupported-schema' + reason = "$($DocumentState.contractName)-schema-version-missing" + document = $document + actualSchema = $actualSchema + actualSchemaVersion = $actualSchemaVersion + parseError = $null + } + } + + if ($ExpectedSchemaVersionMajor -gt 0) { + $actualMajor = Get-SchemaVersionMajor -SchemaVersion $actualSchemaVersion + if ($null -eq $actualMajor) { + return [pscustomobject]@{ + contractName = $DocumentState.contractName + path = $DocumentState.path + present = $true + valid = $false + classification = 'unsupported-schema' + reason = "$($DocumentState.contractName)-invalid-schema-version" + document = $document + actualSchema = $actualSchema + actualSchemaVersion = $actualSchemaVersion + parseError = $null + } + } + + if ($actualMajor -ne $ExpectedSchemaVersionMajor) { + return [pscustomobject]@{ + contractName = $DocumentState.contractName + path = $DocumentState.path + present = $true + valid = $false + classification = 'unsupported-schema' + reason = "$($DocumentState.contractName)-unsupported-schema-version" + document = $document + actualSchema = $actualSchema + actualSchemaVersion = $actualSchemaVersion + parseError = $null + } + } + } + } + + return [pscustomobject]@{ + contractName = $DocumentState.contractName + path = $DocumentState.path + present = $true + valid = $true + classification = 'ok' + reason = "$($DocumentState.contractName)-ok" + document = $document + actualSchema = $actualSchema + actualSchemaVersion = $actualSchemaVersion + parseError = $null + } +} diff --git a/tools/Print-PesterTopFailures.ps1 b/tools/Print-PesterTopFailures.ps1 index 0265dc75f..8ce0bb84e 100644 --- a/tools/Print-PesterTopFailures.ps1 +++ b/tools/Print-PesterTopFailures.ps1 @@ -9,6 +9,11 @@ param( Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' +$failurePayloadTool = Join-Path $PSScriptRoot 'PesterFailurePayload.ps1' +if (Test-Path -LiteralPath $failurePayloadTool -PathType Leaf) { + . $failurePayloadTool +} + $repoRoot = (Resolve-Path '.').Path $uxModule = Join-Path $repoRoot 'tools' 'ConsoleUx.psm1' if (Test-Path -LiteralPath $uxModule -PathType Leaf) { @@ -30,11 +35,12 @@ if (-not (Test-Path -LiteralPath $ResultsDir -PathType Container)) { $items = @() $failJson = Join-Path $ResultsDir 'pester-failures.json' $nunitXml = Join-Path $ResultsDir 'pester-results.xml' +$failurePayloadInfo = $null if (Test-Path -LiteralPath $failJson -PathType Leaf) { - try { - $arr = Get-Content -LiteralPath $failJson -Raw | ConvertFrom-Json -ErrorAction Stop - foreach ($f in $arr) { + $failurePayloadInfo = Read-PesterFailurePayloadFile -PathValue $failJson + if ($failurePayloadInfo.parseStatus -eq 'parsed') { + foreach ($f in (Get-PesterFailureEntries -FailurePayload $failurePayloadInfo.payload)) { $nameProp = if ($f.PSObject.Properties['name']) { [string]$f.name } else { '' } $fileProp = if ($f.PSObject.Properties['file']) { [string]$f.file } else { '' } $lineProp = if ($f.PSObject.Properties['line']) { [string]$f.line } else { '' } @@ -46,8 +52,8 @@ if (Test-Path -LiteralPath $failJson -PathType Leaf) { message = $messageProp } } - } catch { - if ($dx -ne 'quiet') { Write-Warning "Failed to parse ${failJson}: $_" } + } elseif ($dx -ne 'quiet') { + Write-Warning ("Failed to parse ${failJson}: {0}" -f $failurePayloadInfo.parseError) } } elseif (Test-Path -LiteralPath $nunitXml -PathType Leaf) { try { @@ -77,7 +83,20 @@ if (Test-Path -LiteralPath $failJson -PathType Leaf) { } if (-not $items -or $items.Count -eq 0) { - if ($dx -ne 'quiet') { Write-Host '[dx] top-failures none' } + if ($dx -ne 'quiet') { + $summaryPath = Join-Path $ResultsDir 'pester-summary.json' + $summary = $null + if (Test-Path -LiteralPath $summaryPath -PathType Leaf) { + try { $summary = Get-Content -LiteralPath $summaryPath -Raw | ConvertFrom-Json -ErrorAction Stop } catch {} + } + $detailState = Get-PesterFailureDetailState -FailurePayload $(if ($failurePayloadInfo) { $failurePayloadInfo.payload } else { $null }) -Summary $summary + if ($detailState.detailStatus -eq 'unavailable') { + $reasonSuffix = if ($detailState.unavailableReason) { " reason=$($detailState.unavailableReason)" } else { '' } + Write-Host ("[dx] top-failures unavailable{0}" -f $reasonSuffix) + } else { + Write-Host '[dx] top-failures none' + } + } if ($PassThru) { return @() } return } diff --git a/tools/Replay-PesterServiceModelArtifacts.Local.ps1 b/tools/Replay-PesterServiceModelArtifacts.Local.ps1 new file mode 100644 index 000000000..0fb6cc8a1 --- /dev/null +++ b/tools/Replay-PesterServiceModelArtifacts.Local.ps1 @@ -0,0 +1,238 @@ +#Requires -Version 7.0 +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$RawArtifactDir, + + [Parameter(Mandatory = $false)] + [string]$ExecutionReceiptPath, + + [Parameter(Mandatory = $false)] + [string]$WorkspaceResultsDir = 'tests/results/pester-replay-local', + + [Parameter(Mandatory = $false)] + [switch]$SkipSessionIndex +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$schemaToolPath = Join-Path $PSScriptRoot 'PesterServiceModelSchema.ps1' +if (-not (Test-Path -LiteralPath $schemaToolPath -PathType Leaf)) { + throw "Schema tool not found: $schemaToolPath" +} +. $schemaToolPath + +function Resolve-RepoRoot { + return (Resolve-Path (Join-Path $PSScriptRoot '..')).Path +} + +function Resolve-OutputPath { + param( + [string]$RepoRoot, + [string]$PathValue + ) + + if ([System.IO.Path]::IsPathRooted($PathValue)) { + return [System.IO.Path]::GetFullPath($PathValue) + } + return [System.IO.Path]::GetFullPath((Join-Path $RepoRoot $PathValue)) +} + +function Write-JsonFile { + param( + [Parameter(Mandatory = $true)][string]$PathValue, + [Parameter(Mandatory = $true)]$Payload + ) + $dir = Split-Path -Parent $PathValue + if ($dir -and -not (Test-Path -LiteralPath $dir -PathType Container)) { + New-Item -ItemType Directory -Path $dir -Force | Out-Null + } + $Payload | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $PathValue -Encoding UTF8 +} + +$repoRoot = Resolve-RepoRoot +$resolvedRawArtifactDir = Resolve-OutputPath -RepoRoot $repoRoot -PathValue $RawArtifactDir +if (-not (Test-Path -LiteralPath $resolvedRawArtifactDir -PathType Container)) { + throw "Raw artifact directory not found: $resolvedRawArtifactDir" +} +$resolvedWorkspaceResultsDir = Resolve-OutputPath -RepoRoot $repoRoot -PathValue $WorkspaceResultsDir +$resolvedExecutionReceiptPath = if ([string]::IsNullOrWhiteSpace($ExecutionReceiptPath)) { $null } else { Resolve-OutputPath -RepoRoot $repoRoot -PathValue $ExecutionReceiptPath } + +$postprocessToolPath = Join-Path $repoRoot 'tools/Invoke-PesterExecutionPostprocess.ps1' +$telemetryToolPath = Join-Path $repoRoot 'tools/Invoke-PesterExecutionTelemetry.ps1' +$totalsToolPath = Join-Path $repoRoot 'tools/Write-PesterTotals.ps1' +$classificationToolPath = Join-Path $repoRoot 'tools/Invoke-PesterEvidenceClassification.ps1' +$operatorOutcomeToolPath = Join-Path $repoRoot 'tools/Invoke-PesterOperatorOutcome.ps1' +$provenanceToolPath = Join-Path $repoRoot 'tools/Invoke-PesterEvidenceProvenance.ps1' +$sessionIndexToolPath = Join-Path $repoRoot 'tools/Ensure-SessionIndex.ps1' + +$executionReceipt = $null +$executionReceiptState = $null +$stagedReceiptPath = $null + +if (Test-Path -LiteralPath $resolvedWorkspaceResultsDir) { + Remove-Item -LiteralPath $resolvedWorkspaceResultsDir -Recurse -Force +} +New-Item -ItemType Directory -Path $resolvedWorkspaceResultsDir -Force | Out-Null + +Get-ChildItem -LiteralPath $resolvedRawArtifactDir -Force | ForEach-Object { + Copy-Item -LiteralPath $_.FullName -Destination (Join-Path $resolvedWorkspaceResultsDir $_.Name) -Recurse -Force +} + +if ($resolvedExecutionReceiptPath) { + if (-not (Test-Path -LiteralPath $resolvedExecutionReceiptPath -PathType Leaf)) { + throw "Execution receipt not found: $resolvedExecutionReceiptPath" + } + $stagedReceiptPath = Join-Path $resolvedWorkspaceResultsDir 'pester-execution-contract/pester-run-receipt.json' + $stagedReceiptDir = Split-Path -Parent $stagedReceiptPath + if (-not (Test-Path -LiteralPath $stagedReceiptDir -PathType Container)) { + New-Item -ItemType Directory -Path $stagedReceiptDir -Force | Out-Null + } + Copy-Item -LiteralPath $resolvedExecutionReceiptPath -Destination $stagedReceiptPath -Force + $executionReceiptState = Test-PesterServiceModelSchemaContract ` + -DocumentState (Read-PesterServiceModelJsonDocument -PathValue $stagedReceiptPath -ContractName 'execution-receipt') ` + -ExpectedSchema 'pester-execution-receipt@v1' + if ($executionReceiptState.valid) { + $executionReceipt = $executionReceiptState.document + } +} + +Push-Location $repoRoot +try { + & $postprocessToolPath -ResultsDir $resolvedWorkspaceResultsDir | Out-Host + if ($LASTEXITCODE -ne 0) { + throw "Execution postprocess failed with exit code $LASTEXITCODE." + } + + & $totalsToolPath -ResultsDir $resolvedWorkspaceResultsDir | Out-Host + if ($LASTEXITCODE -ne 0) { + throw "Pester totals generation failed with exit code $LASTEXITCODE." + } + + & $telemetryToolPath -ResultsDir $resolvedWorkspaceResultsDir | Out-Host + if ($LASTEXITCODE -ne 0) { + throw "Execution telemetry generation failed with exit code $LASTEXITCODE." + } + + if (-not $SkipSessionIndex) { + & $sessionIndexToolPath -ResultsDir $resolvedWorkspaceResultsDir -SummaryJson 'pester-summary.json' | Out-Host + if ($LASTEXITCODE -ne 0) { + throw "Session index generation failed with exit code $LASTEXITCODE." + } + } + + $classificationArgs = @{ + ResultsDir = $resolvedWorkspaceResultsDir + RawArtifactDownload = 'staged' + } + if ($stagedReceiptPath) { + $classificationArgs.ExecutionReceiptPath = $stagedReceiptPath + } + if ($executionReceipt) { + if ($executionReceipt.PSObject.Properties.Name -contains 'contextStatus') { + $classificationArgs.ContextStatus = [string]$executionReceipt.contextStatus + } + if ($executionReceipt.PSObject.Properties.Name -contains 'readinessStatus') { + $classificationArgs.ReadinessStatus = [string]$executionReceipt.readinessStatus + } + if ($executionReceipt.PSObject.Properties.Name -contains 'selectionStatus') { + $classificationArgs.SelectionStatus = [string]$executionReceipt.selectionStatus + } + if ($executionReceipt.PSObject.Properties.Name -contains 'executionJobResult') { + $classificationArgs.ExecutionJobResult = [string]$executionReceipt.executionJobResult + } + if ($executionReceipt.PSObject.Properties.Name -contains 'dispatcherExitCode') { + $classificationArgs.DispatcherExitCode = [string]$executionReceipt.dispatcherExitCode + } + } + & $classificationToolPath @classificationArgs | Out-Host + if ($LASTEXITCODE -ne 0) { + throw "Evidence classification failed with exit code $LASTEXITCODE." + } + + & $operatorOutcomeToolPath -ResultsDir $resolvedWorkspaceResultsDir | Out-Host + if ($LASTEXITCODE -ne 0) { + throw "Operator outcome generation failed with exit code $LASTEXITCODE." + } + + & $provenanceToolPath ` + -ResultsDir $resolvedWorkspaceResultsDir ` + -ExecutionReceiptPath $stagedReceiptPath ` + -RawArtifactName ([System.IO.Path]::GetFileName($resolvedRawArtifactDir)) ` + -RawArtifactDownload 'local-replay' ` + -ExecutionReceiptArtifactName 'pester-execution-contract' ` + -SourceRawArtifactDir $resolvedRawArtifactDir ` + -ProvenanceKind 'local-replay' | Out-Host + if ($LASTEXITCODE -ne 0) { + throw "Evidence provenance generation failed with exit code $LASTEXITCODE." + } +} finally { + Pop-Location +} + +$classificationPath = Join-Path $resolvedWorkspaceResultsDir 'pester-evidence-classification.json' +$postprocessReportPath = Join-Path $resolvedWorkspaceResultsDir 'pester-execution-postprocess.json' +$telemetryPath = Join-Path $resolvedWorkspaceResultsDir 'pester-execution-telemetry.json' +$provenancePath = Join-Path $resolvedWorkspaceResultsDir 'pester-evidence-provenance.json' +$totalsPath = Join-Path $resolvedWorkspaceResultsDir 'pester-totals.json' +$sessionIndexPath = Join-Path $resolvedWorkspaceResultsDir 'session-index.json' +$classification = if (Test-Path -LiteralPath $classificationPath -PathType Leaf) { + (Get-Content -LiteralPath $classificationPath -Raw | ConvertFrom-Json -ErrorAction Stop).classification +} else { + 'missing' +} +$operatorOutcomePath = Join-Path $resolvedWorkspaceResultsDir 'pester-operator-outcome.json' +$operatorOutcome = if (Test-Path -LiteralPath $operatorOutcomePath -PathType Leaf) { + Get-Content -LiteralPath $operatorOutcomePath -Raw | ConvertFrom-Json -ErrorAction Stop +} else { + $null +} +$telemetry = if (Test-Path -LiteralPath $telemetryPath -PathType Leaf) { + Get-Content -LiteralPath $telemetryPath -Raw | ConvertFrom-Json -ErrorAction Stop +} else { + $null +} +$provenance = if (Test-Path -LiteralPath $provenancePath -PathType Leaf) { + Get-Content -LiteralPath $provenancePath -Raw | ConvertFrom-Json -ErrorAction Stop +} else { + $null +} + +$replayReceipt = [ordered]@{ + schema = 'pester-local-replay-receipt@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + rawArtifactDir = $resolvedRawArtifactDir + executionReceiptPath = $resolvedExecutionReceiptPath + stagedExecutionReceiptPath = $stagedReceiptPath + stagedExecutionReceiptSchemaStatus = if ($executionReceiptState) { [string]$executionReceiptState.classification } else { 'missing' } + stagedExecutionReceiptSchemaReason = if ($executionReceiptState) { [string]$executionReceiptState.reason } else { 'execution-receipt-missing' } + workspaceResultsDir = $resolvedWorkspaceResultsDir + postprocessReportPath = $postprocessReportPath + telemetryPath = $telemetryPath + telemetryPresent = [bool]$telemetry + telemetryStatus = if ($telemetry) { [string]$telemetry.telemetryStatus } else { 'telemetry-missing' } + telemetryLastKnownPhase = if ($telemetry) { [string]$telemetry.lastKnownPhase } else { $null } + telemetryEventCount = if ($telemetry) { [int]$telemetry.eventCount } else { 0 } + totalsPath = $totalsPath + sessionIndexPath = if ($SkipSessionIndex) { $null } else { $sessionIndexPath } + classificationPath = $classificationPath + classification = $classification + operatorOutcomePath = $operatorOutcomePath + operatorOutcomePresent = [bool]$operatorOutcome + operatorOutcomeGateStatus = if ($operatorOutcome) { [string]$operatorOutcome.gateStatus } else { 'missing' } + operatorOutcomeNextActionId = if ($operatorOutcome) { [string]$operatorOutcome.nextActionId } else { $null } + provenancePath = $provenancePath + provenancePresent = [bool]$provenance + provenanceKind = if ($provenance) { [string]$provenance.provenanceKind } else { 'missing' } +} +$replayReceiptPath = Join-Path $resolvedWorkspaceResultsDir 'pester-local-replay-receipt.json' +Write-JsonFile -PathValue $replayReceiptPath -Payload $replayReceipt + +Write-Host '### Pester service-model local replay' -ForegroundColor Cyan +Write-Host ("rawArtifact : {0}" -f $resolvedRawArtifactDir) +Write-Host ("workspace : {0}" -f $resolvedWorkspaceResultsDir) +Write-Host ("classify : {0}" -f $classification) +Write-Host ("receipt : {0}" -f $replayReceiptPath) + +exit 0 diff --git a/tools/Run-NIWindowsContainerCompare.ps1 b/tools/Run-NIWindowsContainerCompare.ps1 index ece5ef0bd..571c176a3 100644 --- a/tools/Run-NIWindowsContainerCompare.ps1 +++ b/tools/Run-NIWindowsContainerCompare.ps1 @@ -247,10 +247,14 @@ function Resolve-ExistingFilePath { } try { $resolved = Resolve-Path -LiteralPath $effectiveInput -ErrorAction Stop - if (-not (Test-Path -LiteralPath $resolved.Path -PathType Leaf)) { + $providerPath = [string]$resolved.ProviderPath + if ([string]::IsNullOrWhiteSpace($providerPath)) { + throw ("Path is not a provider-backed file: {0}" -f $effectiveInput) + } + if (-not (Test-Path -LiteralPath $providerPath -PathType Leaf)) { throw ("Path is not a file: {0}" -f $effectiveInput) } - return $resolved.Path + return [System.IO.Path]::GetFullPath($providerPath) } catch { throw ("Unable to resolve -{0} file path '{1}'." -f $ParameterName, $effectiveInput) } @@ -291,14 +295,15 @@ function Resolve-OutputReportPath { [string]$PathValue, [Parameter(Mandatory)][string]$Extension ) + $resolveAbsolutePath = { + param([Parameter(Mandatory)][string]$Candidate) + $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($Candidate) + } if ([string]::IsNullOrWhiteSpace($PathValue)) { - $defaultRoot = Join-Path (Resolve-Path '.').Path 'tests/results/ni-windows-container' + $defaultRoot = Join-Path (& $resolveAbsolutePath '.') 'tests/results/ni-windows-container' return (Join-Path $defaultRoot ("compare-report.{0}" -f $Extension)) } - if ([System.IO.Path]::IsPathRooted($PathValue)) { - return [System.IO.Path]::GetFullPath($PathValue) - } - return [System.IO.Path]::GetFullPath((Join-Path (Resolve-Path '.').Path $PathValue)) + return & $resolveAbsolutePath $PathValue } function Get-DockerServerOsType { @@ -339,6 +344,182 @@ function Convert-HostFileToContainerPath { return (Join-Path $containerDir (Split-Path -Leaf $HostFilePath)) } +function Test-PathRequiresWindowsDockerLocalStage { + param([AllowNull()][string]$PathValue) + + if ([string]::IsNullOrWhiteSpace($PathValue)) { + return $false + } + $normalized = [string]$PathValue + return $normalized.StartsWith('\\') +} + +function Resolve-NativeProviderPath { + param([Parameter(Mandatory)][string]$PathValue) + + $resolved = Resolve-Path -LiteralPath $PathValue -ErrorAction Stop + $providerPath = [string]$resolved.ProviderPath + if ([string]::IsNullOrWhiteSpace($providerPath)) { + $providerPath = [string]$resolved.Path + } + if ([string]::IsNullOrWhiteSpace($providerPath)) { + throw ("Unable to resolve native provider path: {0}" -f $PathValue) + } + return [System.IO.Path]::GetFullPath($providerPath) +} + +function New-WindowsDockerMountStage { + param( + [Parameter(Mandatory)][string]$BaseViPath, + [Parameter(Mandatory)][string]$HeadViPath, + [Parameter(Mandatory)][string]$ReportPath + ) + + $stageRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("comparevi-windows-docker-stage-{0}" -f ([guid]::NewGuid().ToString('N'))) + $inputRoot = Join-Path $stageRoot 'inputs' + $outputRoot = Join-Path $stageRoot 'output' + New-Item -ItemType Directory -Path $inputRoot -Force | Out-Null + New-Item -ItemType Directory -Path $outputRoot -Force | Out-Null + + $baseExtension = [System.IO.Path]::GetExtension($BaseViPath) + $headExtension = [System.IO.Path]::GetExtension($HeadViPath) + $stagedBaseViPath = Join-Path $inputRoot ('Base' + ($(if ($baseExtension) { $baseExtension } else { '' }))) + $stagedHeadViPath = Join-Path $inputRoot ('Head' + ($(if ($headExtension) { $headExtension } else { '' }))) + $stagedReportPath = Join-Path $outputRoot (Split-Path -Leaf $ReportPath) + + Copy-Item -LiteralPath $BaseViPath -Destination $stagedBaseViPath -Force + Copy-Item -LiteralPath $HeadViPath -Destination $stagedHeadViPath -Force + + return [ordered]@{ + enabled = $true + reason = 'windows-docker-unc-mount-stage' + stageRoot = (Resolve-NativeProviderPath -PathValue $stageRoot) + inputRoot = (Resolve-NativeProviderPath -PathValue $inputRoot) + outputRoot = (Resolve-NativeProviderPath -PathValue $outputRoot) + requestedBaseViPath = $BaseViPath + requestedHeadViPath = $HeadViPath + requestedReportPath = $ReportPath + stagedBaseViPath = (Resolve-NativeProviderPath -PathValue $stagedBaseViPath) + stagedHeadViPath = (Resolve-NativeProviderPath -PathValue $stagedHeadViPath) + stagedReportPath = [System.IO.Path]::GetFullPath($stagedReportPath) + } +} + +function Convert-WindowsDockerStagePathToRequestedPath { + param( + [AllowNull()][hashtable]$Stage, + [AllowNull()][string]$PathValue + ) + + if ($null -eq $Stage -or [string]::IsNullOrWhiteSpace($PathValue)) { + return $PathValue + } + + $stageOutputRoot = [string]$Stage.outputRoot + if ([string]::IsNullOrWhiteSpace($stageOutputRoot)) { + return $PathValue + } + + $comparison = [System.StringComparison]::OrdinalIgnoreCase + if (-not ([string]$PathValue).StartsWith($stageOutputRoot, $comparison)) { + return $PathValue + } + + $requestedReportDirectory = Split-Path -Parent ([string]$Stage.requestedReportPath) + $suffix = ([string]$PathValue).Substring($stageOutputRoot.Length).TrimStart('\') + if ([string]::IsNullOrWhiteSpace($suffix)) { + return $requestedReportDirectory + } + return (Join-Path $requestedReportDirectory $suffix) +} + +function Sync-WindowsDockerMountStageArtifacts { + param( + [Parameter(Mandatory)][hashtable]$Stage, + [Parameter(Mandatory)][object]$ExportResult + ) + + $requestedReportPath = [string]$Stage.requestedReportPath + $requestedReportDirectory = Split-Path -Parent $requestedReportPath + if (-not (Test-Path -LiteralPath $requestedReportDirectory -PathType Container)) { + New-Item -ItemType Directory -Path $requestedReportDirectory -Force | Out-Null + } + + $reportCopied = $false + $exportCopied = $false + $stagedReportPath = [string]$Stage.stagedReportPath + if (Test-Path -LiteralPath $stagedReportPath -PathType Leaf) { + Copy-Item -LiteralPath $stagedReportPath -Destination $requestedReportPath -Force + $reportCopied = $true + + $stageAssetDir = Join-Path (Split-Path -Parent $stagedReportPath) ("{0}_files" -f [System.IO.Path]::GetFileNameWithoutExtension($stagedReportPath)) + if (Test-Path -LiteralPath $stageAssetDir -PathType Container) { + $assetDestination = Join-Path $requestedReportDirectory (Split-Path -Leaf $stageAssetDir) + if (Test-Path -LiteralPath $assetDestination -PathType Container) { + Remove-Item -LiteralPath $assetDestination -Recurse -Force -ErrorAction SilentlyContinue + } + Copy-Item -LiteralPath $stageAssetDir -Destination $assetDestination -Recurse -Force + } + } + + $stageExportDir = Join-Path ([string]$Stage.outputRoot) 'container-export' + $requestedExportDir = Join-Path $requestedReportDirectory 'container-export' + if (Test-Path -LiteralPath $stageExportDir -PathType Container) { + if (Test-Path -LiteralPath $requestedExportDir -PathType Container) { + Remove-Item -LiteralPath $requestedExportDir -Recurse -Force -ErrorAction SilentlyContinue + } + Copy-Item -LiteralPath $stageExportDir -Destination $requestedReportDirectory -Recurse -Force + $exportCopied = $true + } + + $mappedCopiedPaths = @( + foreach ($copiedPath in @($ExportResult.copiedPaths)) { + Convert-WindowsDockerStagePathToRequestedPath -Stage $Stage -PathValue ([string]$copiedPath) + } + ) + $mappedCopyAttempts = @( + foreach ($attempt in @($ExportResult.copyAttempts)) { + [ordered]@{ + sourcePath = [string]$attempt.sourcePath + destinationPath = (Convert-WindowsDockerStagePathToRequestedPath -Stage $Stage -PathValue ([string]$attempt.destinationPath)) + exitCode = [int]$attempt.exitCode + artifactPresent = [bool]$attempt.artifactPresent + recoveredFromNonZeroExit = [bool]$attempt.recoveredFromNonZeroExit + recoveredFromHostReport = [bool]$attempt.recoveredFromHostReport + recoveryKind = [string]$attempt.recoveryKind + } + } + ) + + return [ordered]@{ + reportCopied = $reportCopied + exportCopied = $exportCopied + exportResult = [ordered]@{ + exportDir = (Convert-WindowsDockerStagePathToRequestedPath -Stage $Stage -PathValue ([string]$ExportResult.exportDir)) + copiedPaths = $mappedCopiedPaths + copyAttempts = $mappedCopyAttempts + copyStatus = [string]$ExportResult.copyStatus + recoveredCopyCount = [int]$ExportResult.recoveredCopyCount + reportPathExtracted = (Convert-WindowsDockerStagePathToRequestedPath -Stage $Stage -PathValue ([string]$ExportResult.reportPathExtracted)) + } + } +} + +function Remove-WindowsDockerMountStage { + param([AllowNull()][hashtable]$Stage) + + if ($null -eq $Stage) { + return + } + $stageRoot = [string]$Stage.stageRoot + if ([string]::IsNullOrWhiteSpace($stageRoot)) { + return + } + if (Test-Path -LiteralPath $stageRoot -PathType Container) { + Remove-Item -LiteralPath $stageRoot -Recurse -Force -ErrorAction Stop + } +} + function New-ContainerCommand { return @' $ErrorActionPreference = "Stop" @@ -955,6 +1136,18 @@ $capture = [ordered]@{ baseVi = $null headVi = $null reportPath = $null + staging = [ordered]@{ + enabled = $false + reason = '' + stageRoot = '' + inputRoot = '' + outputRoot = '' + activeBaseViPath = '' + activeHeadViPath = '' + activeReportPath = '' + outputSyncStatus = 'not-required' + cleanupStatus = 'not-attempted' + } labviewPath = $null flags = @() command = $null @@ -1014,6 +1207,8 @@ $reportDirectoryForExport = '' $additionalExportPaths = @() $previousCompareLabVIEWPath = $null $restoreCompareLabVIEWPath = $false +$activeReportPathForExport = '' +$windowsMountStage = $null try { Assert-Tool -Name 'docker' @@ -1023,14 +1218,10 @@ try { throw ("Runtime guard script not found: {0}" -f $runtimeGuardPath) } $runtimeSnapshot = if ([string]::IsNullOrWhiteSpace($RuntimeSnapshotPath)) { - $defaultRoot = Join-Path (Resolve-Path '.').Path 'tests/results/ni-windows-container' + $defaultRoot = Join-Path ($ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath('.')) 'tests/results/ni-windows-container' Join-Path $defaultRoot 'runtime-determinism.json' } else { - if ([System.IO.Path]::IsPathRooted($RuntimeSnapshotPath)) { - [System.IO.Path]::GetFullPath($RuntimeSnapshotPath) - } else { - [System.IO.Path]::GetFullPath((Join-Path (Resolve-Path '.').Path $RuntimeSnapshotPath)) - } + $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($RuntimeSnapshotPath) } & pwsh -NoLogo -NoProfile -File $runtimeGuardPath ` @@ -1099,6 +1290,32 @@ try { Remove-Item -LiteralPath $resolvedReportPath -Force -ErrorAction SilentlyContinue } + $activeBaseViPath = $baseViPath + $activeHeadViPath = $headViPath + $activeReportPath = $resolvedReportPath + if ( + (Test-PathRequiresWindowsDockerLocalStage -PathValue $baseViPath) -or + (Test-PathRequiresWindowsDockerLocalStage -PathValue $headViPath) -or + (Test-PathRequiresWindowsDockerLocalStage -PathValue $resolvedReportPath) + ) { + $windowsMountStage = New-WindowsDockerMountStage -BaseViPath $baseViPath -HeadViPath $headViPath -ReportPath $resolvedReportPath + $activeBaseViPath = [string]$windowsMountStage.stagedBaseViPath + $activeHeadViPath = [string]$windowsMountStage.stagedHeadViPath + $activeReportPath = [string]$windowsMountStage.stagedReportPath + $capture.staging = [ordered]@{ + enabled = $true + reason = [string]$windowsMountStage.reason + stageRoot = [string]$windowsMountStage.stageRoot + inputRoot = [string]$windowsMountStage.inputRoot + outputRoot = [string]$windowsMountStage.outputRoot + activeBaseViPath = $activeBaseViPath + activeHeadViPath = $activeHeadViPath + activeReportPath = $activeReportPath + outputSyncStatus = 'pending' + cleanupStatus = 'pending' + } + } + $capturePath = Join-Path $reportDirectory 'ni-windows-container-capture.json' $stdoutPath = Join-Path $reportDirectory 'ni-windows-container-stdout.txt' $stderrPath = Join-Path $reportDirectory 'ni-windows-container-stderr.txt' @@ -1130,14 +1347,15 @@ try { $mounts = @{} $mountIndex = 0 $mountRef = [ref]$mountIndex - $containerBaseVi = Convert-HostFileToContainerPath -HostFilePath $baseViPath -MountMap $mounts -MountIndex $mountRef - $containerHeadVi = Convert-HostFileToContainerPath -HostFilePath $headViPath -MountMap $mounts -MountIndex $mountRef - $containerReportPath = Convert-HostFileToContainerPath -HostFilePath $resolvedReportPath -MountMap $mounts -MountIndex $mountRef + $containerBaseVi = Convert-HostFileToContainerPath -HostFilePath $activeBaseViPath -MountMap $mounts -MountIndex $mountRef + $containerHeadVi = Convert-HostFileToContainerPath -HostFilePath $activeHeadViPath -MountMap $mounts -MountIndex $mountRef + $containerReportPath = Convert-HostFileToContainerPath -HostFilePath $activeReportPath -MountMap $mounts -MountIndex $mountRef $containerName = 'ni-compare-{0}' -f ([guid]::NewGuid().ToString('N').Substring(0, 12)) $containerNameForCleanup = $containerName $containerReportPathForExport = $containerReportPath - $reportDirectoryForExport = $reportDirectory + $reportDirectoryForExport = Split-Path -Parent $activeReportPath + $activeReportPathForExport = $activeReportPath $containerCommand = New-ContainerCommand $encodedContainerCommand = Convert-ToEncodedCommand -CommandText $containerCommand @@ -1258,17 +1476,34 @@ try { $exportResult = Export-ContainerArtifacts ` -ContainerName $containerNameForCleanup ` -ContainerReportPath $containerReportPathForExport ` - -HostReportPath $resolvedReportPath ` + -HostReportPath $activeReportPathForExport ` -ReportDirectory $reportDirectoryForExport ` -AdditionalContainerPaths $additionalExportPaths + $effectiveExportResult = $exportResult + if ($windowsMountStage) { + try { + $syncResult = Sync-WindowsDockerMountStageArtifacts -Stage $windowsMountStage -ExportResult $exportResult + $effectiveExportResult = $syncResult.exportResult + $capture.staging.outputSyncStatus = if ($syncResult.reportCopied -or $syncResult.exportCopied) { 'synchronized' } else { 'no-artifacts' } + } catch { + $capture.staging.outputSyncStatus = 'sync-failed' + $capture.status = 'error' + $capture.classification = 'staging-sync-error' + $capture.message = ("Windows Docker staging sync failed: {0}" -f $_.Exception.Message) + if (($null -eq $capture.exitCode) -or ([int]$capture.exitCode -eq 0)) { + $capture.exitCode = 1 + $finalExitCode = 1 + } + } + } $capture.containerArtifacts = [ordered]@{ - exportDir = [string]$exportResult.exportDir - copiedPaths = @($exportResult.copiedPaths) - copyAttempts = @($exportResult.copyAttempts) - copyStatus = [string]$exportResult.copyStatus - recoveredCopyCount = [int]$exportResult.recoveredCopyCount + exportDir = [string]$effectiveExportResult.exportDir + copiedPaths = @($effectiveExportResult.copiedPaths) + copyAttempts = @($effectiveExportResult.copyAttempts) + copyStatus = [string]$effectiveExportResult.copyStatus + recoveredCopyCount = [int]$effectiveExportResult.recoveredCopyCount } - $capture.reportAnalysis = Get-ReportAnalysis -ExtractedReportPath ([string]$exportResult.reportPathExtracted) + $capture.reportAnalysis = Get-ReportAnalysis -ExtractedReportPath ([string]$effectiveExportResult.reportPathExtracted) } if (-not [string]::IsNullOrWhiteSpace($containerNameForCleanup)) { @@ -1279,6 +1514,18 @@ try { [Environment]::SetEnvironmentVariable('COMPARE_LABVIEW_PATH', $previousCompareLabVIEWPath, 'Process') } + if ($windowsMountStage) { + try { + Remove-WindowsDockerMountStage -Stage $windowsMountStage + $capture.staging.cleanupStatus = 'removed' + } catch { + $capture.staging.cleanupStatus = 'cleanup-failed' + if ([string]::IsNullOrWhiteSpace([string]$capture.message)) { + $capture.message = ("Windows Docker staging cleanup failed: {0}" -f $_.Exception.Message) + } + } + } + $runtimeStatusForClassification = '' $runtimeReasonForClassification = '' if ($capture.runtimeDeterminism) { diff --git a/tools/Run-PesterExecutionOnly.Local.ps1 b/tools/Run-PesterExecutionOnly.Local.ps1 index 8cb2bca14..722c78c96 100644 --- a/tools/Run-PesterExecutionOnly.Local.ps1 +++ b/tools/Run-PesterExecutionOnly.Local.ps1 @@ -6,6 +6,8 @@ param( [string]$ContextReceiptPath, [string]$ReadinessReceiptPath, [string]$SelectionReceiptPath, + [ValidateSet('full', 'comparevi', 'dispatcher', 'workflow', 'fixtures', 'psummary', 'schema', 'loop')] + [string]$ExecutionPack = 'full', [ValidateSet('auto', 'include', 'exclude')] [string]$IntegrationMode = 'exclude', [string[]]$IncludePatterns, @@ -25,7 +27,10 @@ param( [int]$SessionLockQueueWaitSeconds = 15, [int]$SessionLockQueueMaxAttempts = 40, [int]$SessionLockStaleSeconds = 300, - [int]$SessionHeartbeatSeconds = 15 + [int]$SessionHeartbeatSeconds = 15, + [ValidateSet('auto', 'relocate', 'block', 'off')] + [string]$PathHygieneMode = 'auto', + [string]$PathHygieneSafeRoot ) Set-StrictMode -Version Latest @@ -257,20 +262,40 @@ function Write-JsonFile { $Payload | ConvertTo-Json -Depth 10 | Set-Content -LiteralPath $PathValue -Encoding UTF8 } +function Write-ExecutionReceiptFiles { + param( + [Parameter(Mandatory = $true)][string]$ReceiptRoot, + [Parameter(Mandatory = $true)]$Receipt + ) + + $resolvedReceiptRoot = [System.IO.Path]::GetFullPath($ReceiptRoot) + $receiptPath = Join-Path $resolvedReceiptRoot 'pester-run-receipt.json' + $contractReceiptPath = Join-Path $resolvedReceiptRoot 'pester-execution-contract' 'pester-run-receipt.json' + Write-JsonFile -PathValue $receiptPath -Payload $Receipt + Write-JsonFile -PathValue $contractReceiptPath -Payload $Receipt + return [pscustomobject]@{ + receiptPath = $receiptPath + contractReceiptPath = $contractReceiptPath + } +} + $repoRoot = Resolve-RepoRoot -$resolvedResultsPath = Resolve-OutputPath -RepoRoot $repoRoot -PathValue $ResultsPath +$requestedResultsPath = Resolve-OutputPath -RepoRoot $repoRoot -PathValue $ResultsPath $resolvedContextReceiptPath = Resolve-OutputPath -RepoRoot $repoRoot -PathValue $ContextReceiptPath $resolvedReadinessReceiptPath = Resolve-OutputPath -RepoRoot $repoRoot -PathValue $ReadinessReceiptPath $resolvedSelectionReceiptPath = Resolve-OutputPath -RepoRoot $repoRoot -PathValue $SelectionReceiptPath -$resolvedSessionLockRoot = if ([string]::IsNullOrWhiteSpace($SessionLockRoot)) { - Join-Path $resolvedResultsPath '_session_lock' +$requestedSessionLockRoot = if ([string]::IsNullOrWhiteSpace($SessionLockRoot)) { + Join-Path $requestedResultsPath '_session_lock' } else { Resolve-OutputPath -RepoRoot $repoRoot -PathValue $SessionLockRoot } +$resolvedResultsPath = $requestedResultsPath +$resolvedSessionLockRoot = $requestedSessionLockRoot $contextReceipt = $null $readinessReceipt = $null $selectionReceipt = $null +$executionPackResolution = $null $preparedFixtures = $null $dispatcherExitCode = -1 $postprocessStatus = 'seam-defect' @@ -283,9 +308,104 @@ $lvComparePath = $null $dotnetReady = $false $sessionLockPath = $null $sessionLockId = $null +$summaryPresent = $false +$receiptRoot = $resolvedResultsPath +$receiptPaths = $null +$pathHygienePlan = $null +$pathHygieneRecord = $null Push-Location $repoRoot try { + . (Join-Path $repoRoot 'tools/PesterExecutionPacks.ps1') + . (Join-Path $repoRoot 'tools/PesterPathHygiene.ps1') + + $pathHygienePlan = Resolve-PesterPathHygienePlan -ResultsPath $requestedResultsPath -SessionLockRoot $requestedSessionLockRoot -Mode $PathHygieneMode -SafeRoot $PathHygieneSafeRoot + if ([string]::IsNullOrWhiteSpace([string]$pathHygienePlan.receiptRoot)) { + $receiptRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("compare-vi-cli-action-path-hygiene-" + [Guid]::NewGuid().ToString('N')) + } else { + $receiptRoot = [System.IO.Path]::GetFullPath([string]$pathHygienePlan.receiptRoot) + } + if (-not [string]::IsNullOrWhiteSpace([string]$pathHygienePlan.effectiveResultsPath)) { + $resolvedResultsPath = [System.IO.Path]::GetFullPath([string]$pathHygienePlan.effectiveResultsPath) + } + if (-not [string]::IsNullOrWhiteSpace([string]$pathHygienePlan.effectiveSessionLockRoot)) { + $resolvedSessionLockRoot = [System.IO.Path]::GetFullPath([string]$pathHygienePlan.effectiveSessionLockRoot) + } + + $pathHygieneRecord = [ordered]@{ + mode = [string]$pathHygienePlan.mode + status = [string]$pathHygienePlan.status + requestedResultsPath = ConvertTo-PortablePath $requestedResultsPath + effectiveResultsPath = ConvertTo-PortablePath $resolvedResultsPath + requestedSessionLockRoot = ConvertTo-PortablePath $requestedSessionLockRoot + effectiveSessionLockRoot = ConvertTo-PortablePath $resolvedSessionLockRoot + receiptRoot = ConvertTo-PortablePath $receiptRoot + safeRoot = ConvertTo-PortablePath ([string]$pathHygienePlan.safeRoot) + risks = @($pathHygienePlan.risks) + } + + if ([string]$pathHygienePlan.status -eq 'path-hygiene-blocked') { + $executionStatus = 'path-hygiene-blocked' + $executionJobResult = 'skipped' + $postprocessStatus = 'not-run' + $repository = Resolve-RepositorySlug -RepoRoot $repoRoot + $blockedReceipt = [ordered]@{ + schema = 'pester-execution-receipt@v1' + generatedAtUtc = [DateTime]::UtcNow.ToString('o') + source = 'local-harness' + repository = $repository + contextStatus = 'local-ready' + contextReceiptPath = ConvertTo-PortablePath $resolvedContextReceiptPath + contextReceiptPresent = $false + standingPriorityIssue = $null + readinessStatus = 'local-ready' + readinessReceiptPath = ConvertTo-PortablePath $resolvedReadinessReceiptPath + readinessReceiptPresent = $false + selectionStatus = 'local-ready' + selectionReceiptPath = ConvertTo-PortablePath $resolvedSelectionReceiptPath + selectionReceiptPresent = $false + dispatcherExitCode = $dispatcherExitCode + postprocessStatus = $postprocessStatus + resultsXmlStatus = $null + executionJobResult = $executionJobResult + summaryPresent = $summaryPresent + status = $executionStatus + pathHygieneStatus = [string]$pathHygienePlan.status + rawArtifactName = 'pester-run-raw-local' + localHarness = [ordered]@{ + dotnetReady = $false + lvComparePath = $null + fixtureBase = $null + fixtureHead = $null + timeoutSeconds = 0 + emitFailuresJsonAlways = $false + detectLeaks = $false + failOnLeaks = $false + killLeaks = $false + leakGraceSeconds = $LeakGraceSeconds + cleanLabVIEWBefore = $false + cleanAfter = $false + trackArtifacts = $false + sessionLockGroup = $SessionLockGroup + sessionLockRoot = ConvertTo-PortablePath $resolvedSessionLockRoot + sessionLockPath = $null + postprocessReportPath = $null + preSnapshotPath = $null + dispatcherLogPath = $null + pathHygiene = $pathHygieneRecord + } + } + $receiptPaths = Write-ExecutionReceiptFiles -ReceiptRoot $receiptRoot -Receipt $blockedReceipt + + Write-Host '### Local Pester execution harness' -ForegroundColor Cyan + Write-Host ("status : {0}" -f $executionStatus) + Write-Host ("requested : {0}" -f $requestedResultsPath) + Write-Host ("effective : {0}" -f ($resolvedResultsPath ?? '')) + Write-Host ("receipt : {0}" -f $receiptPaths.receiptPath) + Write-Host ("contract : {0}" -f $receiptPaths.contractReceiptPath) + throw ("Local path hygiene blocked unsafe synchronized or externally managed roots. Receipt: {0}" -f $receiptPaths.receiptPath) + } + New-Item -ItemType Directory -Path $resolvedResultsPath -Force | Out-Null New-Item -ItemType Directory -Path $resolvedSessionLockRoot -Force | Out-Null @@ -296,16 +416,27 @@ try { $repository = if ($contextReceipt) { [string]$contextReceipt.repository } else { Resolve-RepositorySlug -RepoRoot $repoRoot } $standingPriorityIssue = if ($contextReceipt) { $contextReceipt.standingPriority.issueNumber } else { $null } + $effectiveExecutionPack = if ($selectionReceipt -and -not $PSBoundParameters.ContainsKey('ExecutionPack')) { + [string]$selectionReceipt.selection.executionPack + } else { + $ExecutionPack + } + $effectiveIntegrationMode = if ($selectionReceipt -and -not $PSBoundParameters.ContainsKey('IntegrationMode')) { [string]$selectionReceipt.selection.integrationMode } else { $IntegrationMode } $effectiveIncludePatterns = if ($selectionReceipt -and -not $PSBoundParameters.ContainsKey('IncludePatterns')) { - @($selectionReceipt.selection.includePatterns) + if ($selectionReceipt.selection.PSObject.Properties.Name -contains 'refineIncludePatterns') { + @($selectionReceipt.selection.refineIncludePatterns) + } else { + @($selectionReceipt.selection.includePatterns) + } } else { @($IncludePatterns) } + $executionPackResolution = Resolve-PesterExecutionPack -ExecutionPack $effectiveExecutionPack -RefineIncludePatterns $effectiveIncludePatterns $fixtureRequired = if ($selectionReceipt) { ConvertTo-Bool $selectionReceipt.selection.fixtureRequired } else { @@ -435,9 +566,10 @@ try { $invokeParams = @{ TestsPath = $TestsPath ResultsPath = $resolvedResultsPath + ExecutionPack = $executionPackResolution.executionPack IntegrationMode = $effectiveIntegrationMode } - $effectiveIncludePatternsList = @($effectiveIncludePatterns | Where-Object { -not [string]::IsNullOrWhiteSpace([string]$_) }) + $effectiveIncludePatternsList = @($executionPackResolution.refineIncludePatterns | Where-Object { -not [string]::IsNullOrWhiteSpace([string]$_) }) if ($effectiveIncludePatternsList.Count -gt 0) { $invokeParams.IncludePatterns = $effectiveIncludePatternsList } @@ -453,9 +585,14 @@ try { $dispatcherPath = Join-Path $repoRoot 'Invoke-PesterTests.ps1' $logPath = Join-Path $resolvedResultsPath 'pester-dispatcher.log' + $dispatcherOutputTrace = Join-Path $resolvedResultsPath 'dispatcher-github-output.txt' + $originalGitHubOutput = $env:GITHUB_OUTPUT if (Test-Path -LiteralPath $logPath) { Remove-Item -LiteralPath $logPath -Force } + if (Test-Path -LiteralPath $dispatcherOutputTrace) { + Remove-Item -LiteralPath $dispatcherOutputTrace -Force + } try { $previousErrorActionPreference = $ErrorActionPreference @@ -467,6 +604,11 @@ try { $dispatcherExitCode = if ($null -ne $LASTEXITCODE -and [int]$LASTEXITCODE -ne 0) { [int]$LASTEXITCODE } else { 1 } } finally { $ErrorActionPreference = $previousErrorActionPreference + if ($null -ne $originalGitHubOutput) { + $env:GITHUB_OUTPUT = $originalGitHubOutput + } else { + Remove-Item -Path Env:GITHUB_OUTPUT -ErrorAction SilentlyContinue + } if ($heartbeatJob) { Stop-Job -Id $heartbeatJob.Id -ErrorAction SilentlyContinue | Out-Null Remove-Job -Id $heartbeatJob.Id -Force -ErrorAction SilentlyContinue | Out-Null @@ -480,6 +622,8 @@ try { } } + "exit_code=$dispatcherExitCode" | Out-File -FilePath $dispatcherOutputTrace -Append -Encoding utf8 + $postprocessToolPath = Join-Path $repoRoot 'tools/Invoke-PesterExecutionPostprocess.ps1' if (-not (Test-Path -LiteralPath $postprocessToolPath -PathType Leaf)) { throw "Postprocess tool not found: $postprocessToolPath" @@ -495,9 +639,24 @@ try { $resultsXmlStatus = [string]$postprocessReport.resultsXmlStatus } + $telemetryToolPath = Join-Path $repoRoot 'tools/Invoke-PesterExecutionTelemetry.ps1' + if (-not (Test-Path -LiteralPath $telemetryToolPath -PathType Leaf)) { + throw "Telemetry tool not found: $telemetryToolPath" + } + & $telemetryToolPath -ResultsDir $resolvedResultsPath | Out-Host + if ($LASTEXITCODE -ne 0) { + throw "Pester execution telemetry failed with exit code $LASTEXITCODE." + } + $telemetryReportPath = Join-Path $resolvedResultsPath 'pester-execution-telemetry.json' + $telemetryReport = if (Test-Path -LiteralPath $telemetryReportPath -PathType Leaf) { + Read-JsonFile -PathValue $telemetryReportPath + } else { + $null + } + $summaryPath = Join-Path $resolvedResultsPath 'pester-summary.json' $summaryPresent = Test-Path -LiteralPath $summaryPath - if ($postprocessStatus -in @('results-xml-truncated', 'invalid-results-xml', 'missing-results-xml')) { + if ($postprocessStatus -in @('results-xml-truncated', 'invalid-results-xml', 'missing-results-xml', 'unsupported-schema')) { $executionStatus = $postprocessStatus $executionJobResult = 'failure' } elseif ($summaryPresent -and $dispatcherExitCode -eq 0) { @@ -526,15 +685,25 @@ try { selectionStatus = if ($selectionReceipt) { [string]$selectionReceipt.status } else { 'local-ready' } selectionReceiptPath = ConvertTo-PortablePath $resolvedSelectionReceiptPath selectionReceiptPresent = [bool]$selectionReceipt + selectionExecutionPack = [string]$executionPackResolution.executionPack + selectionExecutionPackSource = [string]$executionPackResolution.executionPackSource selectionIntegrationMode = $effectiveIntegrationMode selectionFixtureRequired = $fixtureRequired - includePatterns = @($effectiveIncludePatternsList) + baseIncludePatterns = @($executionPackResolution.baseIncludePatterns) + refineIncludePatterns = @($executionPackResolution.refineIncludePatterns) + effectiveIncludePatterns = @($executionPackResolution.effectiveIncludePatterns) + includePatterns = @($executionPackResolution.effectiveIncludePatterns) dispatcherExitCode = $dispatcherExitCode + telemetryPresent = [bool]$telemetryReport + telemetryStatus = if ($telemetryReport) { [string]$telemetryReport.telemetryStatus } else { '' } + telemetryLastKnownPhase = if ($telemetryReport) { [string]$telemetryReport.lastKnownPhase } else { '' } + telemetryEventCount = if ($telemetryReport) { [int]$telemetryReport.eventCount } else { 0 } postprocessStatus = $postprocessStatus resultsXmlStatus = $resultsXmlStatus executionJobResult = $executionJobResult summaryPresent = $summaryPresent status = $executionStatus + pathHygieneStatus = if ($pathHygienePlan) { [string]$pathHygienePlan.status } else { 'clean' } rawArtifactName = 'pester-run-raw-local' localHarness = [ordered]@{ dotnetReady = $dotnetReady @@ -554,22 +723,24 @@ try { sessionLockRoot = ConvertTo-PortablePath $resolvedSessionLockRoot sessionLockPath = ConvertTo-PortablePath $sessionLockPath postprocessReportPath = ConvertTo-PortablePath $postprocessReportPath + telemetryReportPath = ConvertTo-PortablePath $telemetryReportPath preSnapshotPath = ConvertTo-PortablePath $preSnapshotPath dispatcherLogPath = ConvertTo-PortablePath $logPath + pathHygiene = $pathHygieneRecord } } - $receiptPath = Join-Path $resolvedResultsPath 'pester-run-receipt.json' - $contractReceiptPath = Join-Path $resolvedResultsPath 'pester-execution-contract' 'pester-run-receipt.json' - Write-JsonFile -PathValue $receiptPath -Payload $receipt - Write-JsonFile -PathValue $contractReceiptPath -Payload $receipt + $receiptPaths = Write-ExecutionReceiptFiles -ReceiptRoot $receiptRoot -Receipt $receipt Write-Host '### Local Pester execution harness' -ForegroundColor Cyan + Write-Host ("executionPack : {0}" -f $executionPackResolution.executionPack) Write-Host ("status : {0}" -f $executionStatus) Write-Host ("exitCode : {0}" -f $dispatcherExitCode) Write-Host ("summary : {0}" -f $summaryPresent) - Write-Host ("receipt : {0}" -f $receiptPath) - Write-Host ("contract : {0}" -f $contractReceiptPath) + $telemetryStatusDisplay = if ($telemetryReport) { [string]$telemetryReport.telemetryStatus } else { 'missing' } + Write-Host ("telemetry : {0}" -f $telemetryStatusDisplay) + Write-Host ("receipt : {0}" -f $receiptPaths.receiptPath) + Write-Host ("contract : {0}" -f $receiptPaths.contractReceiptPath) } finally { if ($preparedFixtures -and $preparedFixtures.tempDir -and (Test-Path -LiteralPath $preparedFixtures.tempDir -PathType Container)) { diff --git a/tools/Test-WindowsNI2026q1HostPreflight.ps1 b/tools/Test-WindowsNI2026q1HostPreflight.ps1 index 2dbb3b83b..68f1e658d 100644 --- a/tools/Test-WindowsNI2026q1HostPreflight.ps1 +++ b/tools/Test-WindowsNI2026q1HostPreflight.ps1 @@ -34,10 +34,7 @@ $ErrorActionPreference = 'Stop' function Resolve-AbsolutePath { param([Parameter(Mandatory)][string]$Path) - if ([System.IO.Path]::IsPathRooted($Path)) { - return [System.IO.Path]::GetFullPath($Path) - } - return [System.IO.Path]::GetFullPath((Join-Path (Get-Location).Path $Path)) + return $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($Path) } function Write-GitHubOutput { diff --git a/tools/Write-PesterTopFailures.ps1 b/tools/Write-PesterTopFailures.ps1 index 07adcad44..ad41e4ff2 100644 --- a/tools/Write-PesterTopFailures.ps1 +++ b/tools/Write-PesterTopFailures.ps1 @@ -5,22 +5,57 @@ [CmdletBinding()] param( [string]$ResultsDir = 'tests/results', - [int]$Top = 5 + [int]$Top = 5, + [string]$OperatorOutcomePath = 'pester-operator-outcome.json' ) Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' if (-not $env:GITHUB_STEP_SUMMARY) { return } +$failurePayloadTool = Join-Path $PSScriptRoot 'PesterFailurePayload.ps1' +if (Test-Path -LiteralPath $failurePayloadTool -PathType Leaf) { + . $failurePayloadTool +} + function Add-Lines([string[]]$lines) { $lines -join "`n" | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append -Encoding utf8 } +function Get-OperatorOutcome([string]$ResultsDir, [string]$OutcomePath) { + if ([string]::IsNullOrWhiteSpace($OutcomePath)) { return $null } + $resolvedPath = if ([System.IO.Path]::IsPathRooted($OutcomePath)) { $OutcomePath } else { Join-Path $ResultsDir $OutcomePath } + if (-not (Test-Path -LiteralPath $resolvedPath -PathType Leaf)) { return $null } + try { + return (Get-Content -LiteralPath $resolvedPath -Raw | ConvertFrom-Json -ErrorAction Stop) + } catch { + return $null + } +} +function Get-FailureSummaryMeta([string]$ResultsDir) { + $summaryPath = Join-Path $ResultsDir 'pester-summary.json' + if (-not (Test-Path -LiteralPath $summaryPath -PathType Leaf)) { return $null } + try { + $summary = Get-Content -LiteralPath $summaryPath -Raw | ConvertFrom-Json -ErrorAction Stop + return [pscustomobject]@{ + failed = [int]($summary.failed ?? 0) + errors = [int]($summary.errors ?? 0) + resultsXmlStatus = if ($summary.PSObject.Properties['resultsXmlStatus']) { [string]$summary.resultsXmlStatus } else { '' } + failureDetailsStatus = if ($summary.PSObject.Properties['failureDetailsStatus']) { [string]$summary.failureDetailsStatus } else { '' } + failureDetailsReason = if ($summary.PSObject.Properties['failureDetailsReason']) { [string]$summary.failureDetailsReason } else { '' } + } + } catch { + return $null + } +} + +$operatorOutcome = Get-OperatorOutcome -ResultsDir $ResultsDir -OutcomePath $OperatorOutcomePath $failJson = Join-Path $ResultsDir 'pester-failures.json' $nunitXml = Join-Path $ResultsDir 'pester-results.xml' $items = @() +$failurePayloadInfo = $null if (Test-Path -LiteralPath $failJson) { - try { - $arr = Get-Content -LiteralPath $failJson -Raw | ConvertFrom-Json -ErrorAction Stop - foreach ($f in $arr) { + $failurePayloadInfo = Read-PesterFailurePayloadFile -PathValue $failJson + if ($failurePayloadInfo.parseStatus -eq 'parsed') { + foreach ($f in (Get-PesterFailureEntries -FailurePayload $failurePayloadInfo.payload)) { $file = '' $line = '' if ($f.PSObject.Properties.Name -contains 'file') { $file = [string]$f.file } @@ -31,7 +66,7 @@ if (Test-Path -LiteralPath $failJson) { if ($f.PSObject.Properties.Name -contains 'name') { $name = [string]$f.name } $items += [pscustomobject]@{ name=$name; file=$file; line=$line; message=$msg } } - } catch {} + } } elseif (Test-Path -LiteralPath $nunitXml) { try { @@ -53,7 +88,28 @@ elseif (Test-Path -LiteralPath $nunitXml) { } if (-not $items -or $items.Count -eq 0) { - Add-Lines @('### Top Failures','- (none)') + $summaryMeta = Get-FailureSummaryMeta -ResultsDir $ResultsDir + $detailState = Get-PesterFailureDetailState -FailurePayload $(if ($failurePayloadInfo) { $failurePayloadInfo.payload } else { $null }) -Summary $summaryMeta + if ($summaryMeta -and (($summaryMeta.failed + $summaryMeta.errors) -gt 0)) { + $statusSuffix = if ($summaryMeta.resultsXmlStatus) { " (resultsXmlStatus=$($summaryMeta.resultsXmlStatus))" } else { '' } + $reasonSuffix = if ($detailState.unavailableReason) { "; reason=$($detailState.unavailableReason)" } else { '' } + $lines = @( + '### Top Failures', + ("- failure details unavailable; summary reports {0} failed/error cases{1}{2}" -f ($summaryMeta.failed + $summaryMeta.errors), $statusSuffix, $reasonSuffix) + ) + if ($operatorOutcome -and [string]$operatorOutcome.classification -ne 'ok') { + $lines += ("- gate outcome: {0} ({1})" -f [string]$operatorOutcome.classification, [string]$operatorOutcome.gateStatus) + $lines += ("- next action: {0}" -f [string]$operatorOutcome.nextAction) + } + Add-Lines $lines + } else { + $lines = @('### Top Failures','- (none)') + if ($operatorOutcome -and [string]$operatorOutcome.classification -ne 'ok') { + $lines += ("- gate outcome: {0} ({1})" -f [string]$operatorOutcome.classification, [string]$operatorOutcome.gateStatus) + $lines += ("- next action: {0}" -f [string]$operatorOutcome.nextAction) + } + Add-Lines $lines + } return } @@ -67,5 +123,9 @@ for ($i=0; $i -lt $take; $i++) { $lines += ("- {0}{1}" -f $title,$loc) if ($msg) { $lines += (" - {0}" -f $msg) } } +if ($operatorOutcome -and [string]$operatorOutcome.classification -ne 'ok') { + $lines += '' + $lines += ("- gate outcome: {0} ({1})" -f [string]$operatorOutcome.classification, [string]$operatorOutcome.gateStatus) + $lines += ("- next action: {0}" -f [string]$operatorOutcome.nextAction) +} Add-Lines $lines - diff --git a/tools/Write-PesterTotals.ps1 b/tools/Write-PesterTotals.ps1 new file mode 100644 index 000000000..b6fbf256f --- /dev/null +++ b/tools/Write-PesterTotals.ps1 @@ -0,0 +1,63 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $false)] + [string]$ResultsDir = 'tests/results', + + [Parameter(Mandatory = $false)] + [string]$SummaryPath = 'pester-summary.json', + + [Parameter(Mandatory = $false)] + [string]$OutputPath = 'pester-totals.json' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$resolvedResultsDir = [System.IO.Path]::GetFullPath($ResultsDir) +if (-not (Test-Path -LiteralPath $resolvedResultsDir -PathType Container)) { + New-Item -ItemType Directory -Path $resolvedResultsDir -Force | Out-Null +} + +$summaryJsonPath = if ([System.IO.Path]::IsPathRooted($SummaryPath)) { + [System.IO.Path]::GetFullPath($SummaryPath) +} else { + Join-Path $resolvedResultsDir $SummaryPath +} +$totalsPath = if ([System.IO.Path]::IsPathRooted($OutputPath)) { + [System.IO.Path]::GetFullPath($OutputPath) +} else { + Join-Path $resolvedResultsDir $OutputPath +} + +$payload = [ordered]@{ + schema = 'pester-totals/v1' + includeIntegration = $null + status = 'missing-summary' +} + +if (Test-Path -LiteralPath $summaryJsonPath -PathType Leaf) { + try { + $summary = Get-Content -LiteralPath $summaryJsonPath -Raw | ConvertFrom-Json -ErrorAction Stop + $payload.total = $summary.total + $payload.passed = $summary.passed + $payload.failed = $summary.failed + $payload.errors = $summary.errors + $payload.duration_s = $summary.duration_s + $payload.status = if (([int]$summary.failed + [int]$summary.errors) -gt 0) { 'fail' } else { 'ok' } + } catch { + $payload.status = 'unknown' + } +} + +$payload | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $totalsPath -Encoding UTF8 + +if ($env:GITHUB_OUTPUT) { + "path=$totalsPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + "status=$($payload.status)" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 +} + +Write-Host '### Pester totals' -ForegroundColor Cyan +Write-Host ("status : {0}" -f $payload.status) +Write-Host ("path : {0}" -f $totalsPath) + +exit 0 diff --git a/tools/priority/__tests__/comparevi-local-program-ci.test.mjs b/tools/priority/__tests__/comparevi-local-program-ci.test.mjs new file mode 100644 index 000000000..3886d8ba8 --- /dev/null +++ b/tools/priority/__tests__/comparevi-local-program-ci.test.mjs @@ -0,0 +1,239 @@ +#!/usr/bin/env node + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { readFileSync } from 'node:fs'; + +import { buildPostLocalPromotionEscalation, buildRequirementCandidate, mergeSharedSurfaceEscalations, rankProgramRequirements, selectProgramNextStep } from '../comparevi-local-program-ci.mjs'; + +const repoRoot = process.cwd(); + +function readRepoFile(relativePath) { + return readFileSync(path.join(repoRoot, relativePath), 'utf8'); +} + +test('rankProgramRequirements prefers active requirement work over later passive packet work', () => { + const ranked = rankProgramRequirements([ + buildRequirementCandidate( + { id: 'vi-history-local-proof', label: 'VI History Local Proof' }, + 'tests/results/_agent/vi-history-local-proof/local-ci/vi-history-local-ci-report.json', + 'tests/results/_agent/vi-history-local-proof/local-ci/vi-history-local-next-step.json', + { + req_id: 'REQ-VHLP-002', + priority: 'High', + status: 'Gap', + phase: 'foundation', + score: 1200, + why_now: 'VI History refinement profile work remains.', + requirement: 'Profiles stay stable.', + test_id: 'TEST-VHLP-002', + code_refs: ['tools/Invoke-VIHistoryLocalRefinement.ps1'], + suggested_loop: ['Run the VI History refinement surface.'], + active_now: false + } + ), + buildRequirementCandidate( + { id: 'pester-service-model', label: 'Pester Service Model' }, + 'tests/results/_agent/pester-service-model/local-ci/pester-service-model-local-ci-report.json', + 'tests/results/_agent/pester-service-model/local-ci/pester-service-model-next-step.json', + { + req_id: 'REQ-PSM-015', + priority: 'High', + status: 'Regression', + phase: 'execution-governance', + score: 1100, + why_now: 'Execution regression requires attention.', + requirement: 'Dispatch side effects stay separated.', + test_id: 'TEST-PSM-015', + code_refs: ['Invoke-PesterTests.ps1'], + suggested_loop: ['Fix the execution split.'], + active_now: true + } + ) + ]); + + assert.equal(ranked[0].packet_id, 'pester-service-model'); + assert.equal(ranked[0].req_id, 'REQ-PSM-015'); +}); + +test('mergeSharedSurfaceEscalations collapses packet escalations into one Windows-surface handoff', () => { + const merged = mergeSharedSurfaceEscalations([ + { + packet_id: 'pester-service-model', + packet_label: 'Pester Service Model', + source_next_step_path: 'tests/results/_agent/pester-service-model/local-ci/pester-service-model-next-step.json', + escalation_id: 'windows-container-live-proof', + governing_requirement: 'REQ-PSM-027', + blocked_requirement: 'REQ-PSM-025', + proof_check_id: 'windows-container-surface', + status: 'required', + mode: 'escalate', + why_now: 'The next truthful Pester proof surface is unavailable.', + reason: 'Current host is not Windows, so the Docker Desktop + NI Windows image proof surface cannot be exercised here.', + required_surface: 'windows-docker-desktop-ni-image', + current_surface_status: 'not-windows-host', + current_host_platform: 'Unix', + receipt_path: 'tests/results/_agent/pester-service-model/local-ci/windows-container-surface/pester-windows-container-surface.json', + suggested_loop: ['Move to Windows.', 'Probe the shared surface.'], + recommended_commands: ['npm run docker:ni:windows:bootstrap', 'npm run compare:docker:ni:windows:probe'], + stop_conditions: ['Stop when the probe reports ready.'] + }, + { + packet_id: 'vi-history-local-proof', + packet_label: 'VI History Local Proof', + source_next_step_path: 'tests/results/_agent/vi-history-local-proof/local-ci/vi-history-local-next-step.json', + escalation_id: 'windows-docker-desktop-ni-image', + governing_requirement: 'REQ-VHLP-006', + blocked_requirement: 'REQ-VHLP-001', + proof_check_id: 'windows-workflow-replay', + status: 'required', + mode: 'escalate', + why_now: 'The next truthful VI History proof surface is unavailable.', + reason: 'Current host is not Windows, so the VI History Windows workflow replay lane cannot be exercised here.', + required_surface: 'windows-docker-desktop-ni-image', + current_surface_status: 'not-windows-host', + current_host_platform: 'Unix', + receipt_path: 'tests/results/_agent/vi-history-local-proof/local-ci/windows-surface/vi-history-windows-surface.json', + suggested_loop: ['Move to Windows.', 'Run the VI History replay lane.'], + recommended_commands: ['npm run docker:ni:windows:bootstrap', 'npm run compare:docker:ni:windows:probe', 'npm run priority:workflow:replay:windows:vi-history'], + stop_conditions: ['Stop when the replay lane passes.'] + }, + { + packet_id: 'windows-docker-shared-surface', + packet_label: 'Windows Docker Shared Surface', + source_next_step_path: 'tests/results/_agent/windows-docker-shared-surface/local-ci/windows-docker-shared-surface-next-step.json', + escalation_id: 'windows-docker-desktop-ni-image', + governing_requirement: 'REQ-WDSS-005', + blocked_requirement: 'REQ-WDSS-001', + proof_check_id: 'windows-surface', + status: 'required', + mode: 'escalate', + why_now: 'The shared Windows surface itself is unavailable from the current host.', + reason: 'Current host is not Windows, so the shared Windows Docker Desktop + NI image surface cannot be exercised here.', + required_surface: 'windows-docker-desktop-ni-image', + current_surface_status: 'not-windows-host', + current_host_platform: 'Unix', + receipt_path: 'tests/results/_agent/windows-docker-shared-surface/local-ci/windows-surface/pester-windows-container-surface.json', + suggested_loop: ['Move to Windows.', 'Probe the shared Windows surface.'], + recommended_commands: ['npm run docker:ni:windows:bootstrap', 'npm run compare:docker:ni:windows:probe'], + stop_conditions: ['Stop when the shared surface probe reaches ready.'] + } + ]); + + assert.equal(merged.length, 1); + assert.equal(merged[0].required_surface, 'windows-docker-desktop-ni-image'); + assert.deepEqual(merged[0].packet_ids, ['pester-service-model', 'vi-history-local-proof', 'windows-docker-shared-surface']); + assert.deepEqual(merged[0].governing_requirements, ['REQ-PSM-027', 'REQ-VHLP-006', 'REQ-WDSS-005']); + assert.deepEqual(merged[0].blocked_requirements, ['REQ-PSM-025', 'REQ-VHLP-001', 'REQ-WDSS-001']); + assert.deepEqual(merged[0].recommended_commands, [ + 'npm run docker:ni:windows:bootstrap', + 'npm run compare:docker:ni:windows:probe', + 'npm run priority:workflow:replay:windows:vi-history' + ]); +}); + +test('selectProgramNextStep prefers a requirement before a shared escalation', () => { + const requirement = { + type: 'requirement', + packet_id: 'vi-history-local-proof', + packet_label: 'VI History Local Proof', + req_id: 'REQ-VHLP-002' + }; + const escalation = { + type: 'escalation', + required_surface: 'windows-docker-desktop-ni-image' + }; + + assert.equal(selectProgramNextStep([requirement], [escalation]).type, 'requirement'); + assert.equal(selectProgramNextStep([], [escalation]).type, 'escalation'); +}); + +test('selectProgramNextStep falls back to a post-local promotion escalation', () => { + const promotion = buildPostLocalPromotionEscalation([ + { + id: 'pester-service-model', + label: 'Pester Service Model', + report_path: 'tests/results/_agent/pester-service-model/local-ci/pester-service-model-local-ci-report.json', + next_step_path: 'tests/results/_agent/pester-service-model/local-ci/pester-service-model-next-step.json' + }, + { + id: 'vi-history-local-proof', + label: 'VI History Local Proof', + report_path: 'tests/results/_agent/vi-history-local-proof/local-ci/vi-history-local-ci-report.json', + next_step_path: 'tests/results/_agent/vi-history-local-proof/local-ci/vi-history-local-next-step.json' + } + ]); + + assert.equal(selectProgramNextStep([], [], promotion).type, 'escalation'); + assert.equal(selectProgramNextStep([], [], promotion).required_surface, 'integration-or-hosted-proof'); + assert.deepEqual(selectProgramNextStep([], [], promotion).governing_requirements, ['REQ-LPAP-003']); +}); + +test('program-level local proof contract is documented and exposed through package.json', () => { + const packageJson = JSON.parse(readRepoFile('package.json')); + const programDoc = readRepoFile('docs/knowledgebase/Local-Proof-Autonomy-Program.md'); + const programArch = readRepoFile('docs/architecture/local-proof-autonomy-program-control-plane.md'); + const programSrs = readRepoFile('docs/requirements-local-proof-autonomy-program-srs.md'); + const programRtm = readRepoFile('docs/rtm-local-proof-autonomy-program.csv'); + const programPlan = readRepoFile('docs/testing/local-proof-autonomy-program-test-plan.md'); + const pesterSrs = readRepoFile('docs/requirements-pester-service-model-srs.md'); + const pesterRtm = readRepoFile('docs/rtm-pester-service-model.csv'); + const viSrs = readRepoFile('docs/requirements-vi-history-local-proof-srs.md'); + const viRtm = readRepoFile('docs/rtm-vi-history-local-proof.csv'); + const windowsSrs = readRepoFile('docs/requirements-windows-docker-shared-surface-srs.md'); + const windowsRtm = readRepoFile('docs/rtm-windows-docker-shared-surface.csv'); + + assert.equal( + packageJson.scripts['priority:program:local-ci'], + 'node tools/priority/comparevi-local-program-ci.mjs' + ); + assert.equal( + packageJson.scripts['priority:program:next-step'], + 'node tools/priority/comparevi-local-program-ci.mjs --print-next-step' + ); + assert.equal( + packageJson.scripts['priority:windows-surface:local-ci'], + 'node tools/priority/windows-docker-shared-surface-local-ci.mjs' + ); + assert.equal( + packageJson.scripts['priority:windows-surface:next-step'], + 'node tools/priority/windows-docker-shared-surface-local-ci.mjs --print-next-step' + ); + + assert.match(programDoc, /Pester Service Model/i); + assert.match(programDoc, /VI History Local Proof/i); + assert.match(programDoc, /Windows Docker Shared Surface/i); + assert.match(programDoc, /shared `windows-docker-desktop-ni-image` surface/i); + assert.match(programDoc, /priority:program:local-ci/); + assert.match(programDoc, /comparevi-local-program-next-step\.json/); + assert.match(programDoc, /priority:windows-surface:local-ci/); + assert.match(programDoc, /post-local promotion escalation/i); + assert.match(programDoc, /integration or hosted proof/i); + + assert.match(programSrs, /REQ-LPAP-001/); + assert.match(programSrs, /REQ-LPAP-003/); + assert.match(programSrs, /REQ-LPAP-004/); + assert.match(programSrs, /promotion escalation instead of `null`/i); + assert.match(programSrs, /run-scoped audit-surface bundle workspaces/i); + assert.match(programRtm, /REQ-LPAP-003/); + assert.match(programRtm, /TEST-LPAP-003/); + assert.match(programRtm, /REQ-LPAP-004/); + assert.match(programRtm, /TEST-LPAP-004/); + assert.match(programPlan, /TEST-LPAP-001/); + assert.match(programPlan, /TEST-LPAP-003/); + assert.match(programPlan, /TEST-LPAP-004/); + assert.match(programArch, /Post-local promotion surface/); + assert.match(programArch, /integration or hosted proof escalation/i); + assert.match(programArch, /Bundle workspace safety surface/); + assert.match(programArch, /surface-bundle\/run-\*/i); + + assert.match(pesterSrs, /REQ-PSM-028/); + assert.match(pesterRtm, /REQ-PSM-028/); + assert.match(viSrs, /REQ-VHLP-007/); + assert.match(viRtm, /REQ-VHLP-007/); + assert.match(windowsSrs, /REQ-WDSS-006/); + assert.match(windowsSrs, /REQ-WDSS-008/); + assert.match(windowsRtm, /REQ-WDSS-006/); + assert.match(windowsRtm, /REQ-WDSS-008/); +}); diff --git a/tools/priority/__tests__/pester-service-model-local-ci.test.mjs b/tools/priority/__tests__/pester-service-model-local-ci.test.mjs new file mode 100644 index 000000000..a11ced4d1 --- /dev/null +++ b/tools/priority/__tests__/pester-service-model-local-ci.test.mjs @@ -0,0 +1,208 @@ +#!/usr/bin/env node + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { applyAutonomyPolicy, deriveEscalations, determinePhase, parseCsv, parseRequirementNumber, rankProofRegressions, rankRequirementGaps, selectNextStep } from '../pester-service-model-local-ci.mjs'; + +test('parseRequirementNumber extracts numeric requirement ids', () => { + assert.equal(parseRequirementNumber('REQ-PSM-011'), 11); + assert.equal(parseRequirementNumber('REQ-PSM-022'), 22); +}); + +test('determinePhase groups requirement ids into the expected planning phases', () => { + assert.equal(determinePhase(11), 'foundation'); + assert.equal(determinePhase(14), 'execution-governance'); + assert.equal(determinePhase(18), 'promotion-governance'); + assert.equal(determinePhase(21), 'evidence-governance'); + assert.equal(determinePhase(22), 'autonomy'); +}); + +test('parseCsv handles quoted RTM rows with commas', () => { + const rows = parseCsv([ + 'ReqID,Requirement,Source,Priority,TestID,TestArtifact,CodeRef,Status', + 'REQ-PSM-011,"Execution keeps failure-detail artifacts semantically consistent, even under degradation",docs/requirements.md,High,TEST-PSM-011,"Planned local repro",Invoke-PesterTests.ps1,Gap' + ].join('\n')); + + assert.equal(rows.length, 1); + assert.equal(rows[0].ReqID, 'REQ-PSM-011'); + assert.match(rows[0].Requirement, /semantically consistent/); +}); + +test('rankRequirementGaps prefers earliest high-priority runtime-adjacent gaps', () => { + const ranked = rankRequirementGaps([ + { + ReqID: 'REQ-PSM-018', + Requirement: 'Promotion evidence is retained for baseline comparison.', + Source: 'docs/requirements.md', + Priority: 'High', + TestID: 'TEST-PSM-018', + TestArtifact: 'Planned promotion coverage', + CodeRef: 'docs/knowledgebase/Pester-Service-Model.md;.github/workflows/pester-service-model-release-evidence.yml', + Status: 'Gap' + }, + { + ReqID: 'REQ-PSM-011', + Requirement: 'Execution keeps failure-detail artifacts semantically consistent with summary counts.', + Source: 'docs/requirements.md', + Priority: 'High', + TestID: 'TEST-PSM-011', + TestArtifact: 'Planned local repro and contract coverage', + CodeRef: 'Invoke-PesterTests.ps1;tools/Invoke-PesterExecutionFinalize.ps1', + Status: 'Gap' + }, + { + ReqID: 'REQ-PSM-013', + Requirement: 'The local harness detects unsafe OneDrive-managed roots.', + Source: 'docs/requirements.md', + Priority: 'High', + TestID: 'TEST-PSM-013', + TestArtifact: 'Planned local path-hygiene coverage', + CodeRef: 'tools/Run-PesterExecutionOnly.Local.ps1;tools/Session-Lock.ps1', + Status: 'Gap' + } + ]); + + assert.equal(ranked[0].req_id, 'REQ-PSM-011'); + assert.equal(ranked[0].phase, 'foundation'); + assert.match(ranked[0].why_now, /highest-ranked unresolved gap|unresolved High foundation gap/i); + assert.equal(ranked[1].req_id, 'REQ-PSM-013'); + assert.equal(ranked[2].req_id, 'REQ-PSM-018'); +}); + +test('applyAutonomyPolicy prioritizes active worktree matches and adds bounded local guidance', () => { + const ranked = rankRequirementGaps([ + { + ReqID: 'REQ-PSM-011', + Requirement: 'Execution keeps failure-detail artifacts semantically consistent with summary counts.', + Source: 'docs/requirements.md', + Priority: 'High', + TestID: 'TEST-PSM-011', + TestArtifact: 'Planned local repro and contract coverage', + CodeRef: 'Invoke-PesterTests.ps1;tools/Invoke-PesterExecutionFinalize.ps1', + Status: 'Gap' + }, + { + ReqID: 'REQ-PSM-018', + Requirement: 'Promotion evidence is retained for baseline comparison.', + Source: 'docs/requirements.md', + Priority: 'High', + TestID: 'TEST-PSM-018', + TestArtifact: 'Planned promotion coverage', + CodeRef: 'docs/knowledgebase/Pester-Service-Model.md;.github/workflows/pester-service-model-release-evidence.yml', + Status: 'Gap' + } + ]); + const policy = JSON.parse(fs.readFileSync(path.join(process.cwd(), 'tools/priority/pester-service-model-autonomy-policy.json'), 'utf8')); + const guided = applyAutonomyPolicy(ranked, policy, ['Invoke-PesterTests.ps1']); + + assert.equal(guided[0].req_id, 'REQ-PSM-011'); + assert.equal(guided[0].active_now, true); + assert.equal(guided[0].mode, 'local-first'); + assert.ok(guided[0].preferred_commands.length > 0); + assert.ok(guided[0].stop_conditions.length > 0); + assert.ok(guided[0].escalate_when.length > 0); + assert.equal(guided[1].active_now, false); +}); + +test('rankProofRegressions reopens implemented requirements when representative proof checks fail', () => { + const regressions = rankProofRegressions([ + { + id: 'representative-replay', + owner_requirement: 'REQ-PSM-024', + status: 'fail', + blocking: true, + summary: 'Representative replay crashed on a schema-lite retained run.' + } + ], [ + { + ReqID: 'REQ-PSM-024', + Requirement: 'Representative retained-artifact replay normalizes legacy retained runs.', + Source: 'docs/requirements.md', + Priority: 'High', + TestID: 'TEST-PSM-024', + TestArtifact: 'Representative replay proof', + CodeRef: 'tools/Replay-PesterServiceModelArtifacts.Local.ps1;tools/Invoke-PesterEvidenceClassification.ps1', + Status: 'Implemented' + } + ]); + + assert.equal(regressions.length, 1); + assert.equal(regressions[0].req_id, 'REQ-PSM-024'); + assert.equal(regressions[0].status, 'Regression'); + assert.equal(regressions[0].proof_check_id, 'representative-replay'); + assert.match(regressions[0].why_now, /regressed/i); +}); + +test('deriveEscalations emits a machine-readable next step for the Windows-container advisory surface', () => { + const escalations = deriveEscalations([ + { + id: 'windows-container-surface', + owner_requirement: 'REQ-PSM-025', + status: 'advisory', + blocking: false, + summary: 'Windows-container surface is not-windows-host; use the recommended Docker Desktop + NI image commands when a live local proof is required.', + surface_status: 'not-windows-host', + host_platform: 'Unix', + receipt_path: 'tests/results/_agent/pester-service-model/local-ci/windows-container-surface/pester-windows-container-surface.json', + recommended_commands: [ + 'npm run docker:ni:windows:bootstrap', + 'npm run compare:docker:ni:windows:probe', + 'npm run compare:docker:ni:windows' + ] + } + ]); + + assert.equal(escalations.length, 1); + assert.equal(escalations[0].type, 'escalation'); + assert.equal(escalations[0].governing_requirement, 'REQ-PSM-027'); + assert.equal(escalations[0].blocked_requirement, 'REQ-PSM-025'); + assert.equal(escalations[0].required_surface, 'windows-docker-desktop-ni-image'); + assert.match(escalations[0].reason, /not Windows/i); + assert.ok(escalations[0].recommended_commands.length > 0); +}); + +test('selectNextStep prefers requirements first and otherwise yields escalation guidance', () => { + const requirement = { + req_id: 'REQ-PSM-012', + priority: 'High', + status: 'Gap', + phase: 'execution-governance', + score: 1234, + why_now: 'Need named execution-pack coverage.', + requirement: 'Named execution pack contract exists.', + test_id: 'TEST-PSM-012', + code_refs: ['tools/PesterExecutionPacks.ps1'], + suggested_loop: ['Add or tighten local coverage first.'] + }; + const escalation = { + type: 'escalation', + escalation_id: 'windows-container-live-proof' + }; + + const nextRequirementStep = selectNextStep(requirement, [escalation]); + assert.equal(nextRequirementStep.type, 'requirement'); + assert.equal(nextRequirementStep.req_id, 'REQ-PSM-012'); + + const nextEscalationStep = selectNextStep(null, [escalation]); + assert.equal(nextEscalationStep.type, 'escalation'); + assert.equal(nextEscalationStep.escalation_id, 'windows-container-live-proof'); +}); + +test('autonomy policy file exists and defines local guidance', () => { + const policyPath = path.join(process.cwd(), 'tools/priority/pester-service-model-autonomy-policy.json'); + const policy = JSON.parse(fs.readFileSync(policyPath, 'utf8')); + + assert.equal(policy.schema_version, '1.0.0'); + assert.equal(policy.phase_guidance.foundation.mode, 'local-first'); + assert.ok(policy.phase_guidance.foundation.preferred_commands.length > 0); +}); + +test('Pester local CI uses a run-scoped audit bundle root', () => { + const source = fs.readFileSync(path.join(process.cwd(), 'tools/priority/pester-service-model-local-ci.mjs'), 'utf8'); + + assert.match(source, /createRunScopedBundleRoot/); + assert.match(source, /surface-bundle/); + assert.match(source, /run-\$\{Date\.now\(\)\}-\$\{process\.pid\}/); +}); diff --git a/tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs b/tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs index a1e5c944f..217914b6b 100644 --- a/tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs +++ b/tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs @@ -17,14 +17,51 @@ test('package.json exposes the local execution harness as a first-class entrypoi packageJson.scripts['tests:execution:local'], 'pwsh -NoLogo -NoProfile -File tools/Run-PesterExecutionOnly.Local.ps1' ); + assert.equal( + packageJson.scripts['tests:replay:local'], + 'pwsh -NoLogo -NoProfile -File tools/Replay-PesterServiceModelArtifacts.Local.ps1' + ); + assert.equal( + packageJson.scripts['tests:replay:representative'], + 'pwsh -NoLogo -NoProfile -Command "& \'tools/Replay-PesterServiceModelArtifacts.Local.ps1\' -RawArtifactDir \'tests/fixtures/pester-service-model/legacy-results-xml-truncated/raw\' -ExecutionReceiptPath \'tests/fixtures/pester-service-model/legacy-results-xml-truncated/pester-run-receipt.json\' -WorkspaceResultsDir \'tests/results/pester-replay-representative\'"' + ); + assert.equal( + packageJson.scripts['tests:windows-surface:probe'], + 'pwsh -NoLogo -NoProfile -File tools/Invoke-PesterWindowsContainerSurfaceProbe.ps1' + ); + assert.equal( + packageJson.scripts['tests:pack:comparevi'], + 'pwsh -NoLogo -NoProfile -File tools/Run-PesterExecutionOnly.Local.ps1 -ExecutionPack comparevi' + ); + assert.equal( + packageJson.scripts['tests:pack:dispatcher'], + 'pwsh -NoLogo -NoProfile -File tools/Run-PesterExecutionOnly.Local.ps1 -ExecutionPack dispatcher' + ); + assert.equal( + packageJson.scripts['tests:pack:workflow'], + 'pwsh -NoLogo -NoProfile -File tools/Run-PesterExecutionOnly.Local.ps1 -ExecutionPack workflow' + ); + assert.equal( + packageJson.scripts['priority:pester:next-step'], + 'node tools/priority/pester-service-model-local-ci.mjs --print-next-step' + ); }); test('local execution harness owns lock lifecycle, preflight, dispatch, and receipt generation', () => { const harness = readRepoFile('tools/Run-PesterExecutionOnly.Local.ps1'); + const packs = readRepoFile('tools/PesterExecutionPacks.ps1'); const invoker = readRepoFile('scripts/Pester-Invoker.psm1'); + assert.match(harness, /\[string\]\$ExecutionPack = 'full'/); + assert.match(harness, /PesterExecutionPacks\.ps1/); + assert.match(harness, /Resolve-PesterExecutionPack -ExecutionPack \$effectiveExecutionPack/); assert.match(harness, /\[string\]\$SessionLockRoot/); - assert.match(harness, /\$resolvedSessionLockRoot = if \(\[string\]::IsNullOrWhiteSpace\(\$SessionLockRoot\)\)/); + assert.match(harness, /\$requestedSessionLockRoot = if \(\[string\]::IsNullOrWhiteSpace\(\$SessionLockRoot\)\)/); + assert.match(harness, /\$resolvedSessionLockRoot = \$requestedSessionLockRoot/); + assert.match(harness, /\[ValidateSet\('auto', 'relocate', 'block', 'off'\)\]\s*\[string\]\$PathHygieneMode = 'auto'/); + assert.match(harness, /PesterPathHygiene\.ps1/); + assert.match(harness, /Resolve-PesterPathHygienePlan -ResultsPath \$requestedResultsPath -SessionLockRoot \$requestedSessionLockRoot -Mode \$PathHygieneMode -SafeRoot \$PathHygieneSafeRoot/); + assert.match(harness, /path-hygiene-blocked/); assert.match(harness, /-Action Acquire -Group \$SessionLockGroup -LockRoot \$resolvedSessionLockRoot/); assert.match(harness, /-Action Release/); assert.match(harness, /SESSION_LOCK_ROOT = \$resolvedSessionLockRoot/); @@ -34,37 +71,65 @@ test('local execution harness owns lock lifecycle, preflight, dispatch, and rece assert.match(harness, /Resolve-LVComparePath/); assert.match(harness, /Invoke-PesterTests\.ps1/); assert.match(harness, /Invoke-PesterExecutionPostprocess\.ps1/); + assert.match(harness, /Invoke-PesterExecutionTelemetry\.ps1/); + assert.match(harness, /unsupported-schema/); + assert.match(harness, /dispatcher-github-output\.txt/); + assert.match(harness, /\$originalGitHubOutput = \$env:GITHUB_OUTPUT/); assert.match(harness, /pester-execution-receipt@v1/); assert.match(harness, /pester-execution-contract/); assert.match(harness, /source = 'local-harness'/); assert.match(harness, /results-xml-truncated/); + assert.match(harness, /telemetryStatus/); + assert.match(harness, /telemetryLastKnownPhase/); + assert.match(harness, /telemetryEventCount/); + assert.match(harness, /selectionExecutionPack = \[string\]\$executionPackResolution\.executionPack/); + assert.match(harness, /effectiveIncludePatterns = @\(\$executionPackResolution\.effectiveIncludePatterns\)/); assert.match(harness, /summaryPresent/); assert.match(harness, /sessionLockRoot = ConvertTo-PortablePath \$resolvedSessionLockRoot/); + assert.match(harness, /pathHygiene = \$pathHygieneRecord/); + assert.match(packs, /function Resolve-PesterExecutionPack/); + assert.match(packs, /comparevi/); + assert.match(packs, /dispatcher/); + assert.match(packs, /workflow/); assert.match(invoker, /ConvertTo-Json -Depth 8 -Compress/); }); test('dispatcher path delegates summary, artifact, and session-index side effects to the execution finalize helper', () => { const dispatcher = readRepoFile('Invoke-PesterTests.ps1'); const finalize = readRepoFile('tools/Invoke-PesterExecutionFinalize.ps1'); + const publication = readRepoFile('tools/Invoke-PesterExecutionPublication.ps1'); assert.match(dispatcher, /Invoke-PesterExecutionFinalize\.ps1/); assert.match(dispatcher, /pester-execution-finalize-context@v1/); assert.match(dispatcher, /Invoke-ExecutionFinalizeHelper\s+-SummaryText\s+\$summary\s+-SummaryPayload\s+\$jsonObj\s+-ArtifactTrail\s+\$script:artifactTrail/); assert.match(dispatcher, /Invoke-ExecutionFinalizeHelper\s+-ReuseExistingContext/); + assert.match(dispatcher, /function New-ExecutionPublicationPayload/); + assert.match(dispatcher, /Leak detected: pester-leak-report\.json will be emitted during finalize\./); + assert.doesNotMatch(dispatcher, /\$env:GITHUB_STEP_SUMMARY/); assert.doesNotMatch(dispatcher, /Write-ArtifactManifest\s+-Directory/); assert.doesNotMatch(dispatcher, /Write-SessionIndex\s+-ResultsDirectory/); + assert.match(finalize, /Write-LeakReportFromPayload/); + assert.match(finalize, /publicationToolPath = Join-Path \$PSScriptRoot 'Invoke-PesterExecutionPublication\.ps1'/); assert.match(finalize, /pester-artifacts\.json/); assert.match(finalize, /session-index\.json/); assert.match(finalize, /compare-report\.html/); assert.match(finalize, /results-index\.html/); + assert.match(publication, /Write-PesterSummaryToStepSummary\.ps1/); + assert.match(publication, /Write-SessionIndexSummary\.ps1/); + assert.match(publication, /pester-execution-publication@v1/); }); test('knowledgebase documents the local harness as the workflow-shell-free execution entrypoint', () => { const doc = readRepoFile('docs/knowledgebase/Pester-Service-Model.md'); assert.match(doc, /Run-PesterExecutionOnly\.Local\.ps1/); + assert.match(doc, /Replay-PesterServiceModelArtifacts\.Local\.ps1/); + assert.match(doc, /tests:replay:representative/); + assert.match(doc, /tests:windows-surface:probe/); assert.match(doc, /without the workflow shell/i); assert.match(doc, /lock,\s+LV guard,\s+fixture prep,\s+dispatcher profile,\s+dispatch,\s+execution postprocess,\s+and local execution receipt/i); + assert.match(doc, /tests:pack:comparevi/); + assert.match(doc, /tests:replay:local/); }); test('execution-layer assurance packet traces the local harness in the SRS, RTM, and test plan', () => { @@ -79,3 +144,158 @@ test('execution-layer assurance packet traces the local harness in the SRS, RTM, assert.match(plan, /Run-PesterExecutionOnly\.Local\.ps1/); assert.match(plan, /Local harness contract tests pass/i); }); + +test('assurance packet records forward execution-pack, path-hygiene, replay, and side-effect requirements', () => { + const srs = readRepoFile('docs/requirements-pester-service-model-srs.md'); + const rtm = readRepoFile('docs/rtm-pester-service-model.csv'); + const plan = readRepoFile('docs/testing/pester-service-model-test-plan.md'); + const doc = readRepoFile('docs/knowledgebase/Pester-Service-Model.md'); + + assert.match(srs, /REQ-PSM-012/); + assert.match(srs, /named execution pack or test group/i); + assert.match(srs, /REQ-PSM-013/); + assert.match(srs, /OneDrive-managed directories/i); + assert.match(srs, /REQ-PSM-014/); + assert.match(srs, /reproducible locally from retained artifacts/i); + assert.match(srs, /REQ-PSM-015/); + assert.match(srs, /Dispatcher responsibilities shall stop at declared test execution/i); + assert.match(srs, /REQ-PSM-016/); + assert.match(srs, /durable progress telemetry/i); + assert.match(srs, /REQ-PSM-017/); + assert.match(srs, /schema contracts with version-governed readers/i); + assert.match(srs, /REQ-PSM-018/); + assert.match(srs, /retained requirement-to-run evidence/i); + assert.match(srs, /REQ-PSM-019/); + assert.match(srs, /Stable operator-facing entrypoints/i); + assert.match(srs, /REQ-PSM-020/); + assert.match(srs, /retain provenance/i); + assert.match(srs, /REQ-PSM-021/); + assert.match(srs, /machine-readable operator outcomes/i); + assert.match(srs, /REQ-PSM-022/); + assert.match(srs, /ranked backlog and a selected next requirement/i); + assert.match(srs, /REQ-PSM-023/); + assert.match(srs, /active-worktree signals, and stop conditions/i); + assert.match(srs, /REQ-PSM-024/); + assert.match(srs, /representative retained-artifact replay/i); + assert.match(srs, /REQ-PSM-025/); + assert.match(srs, /Docker Desktop Windows engine and the pinned NI Windows image/i); + assert.match(srs, /REQ-PSM-026/); + assert.match(srs, /reopen implemented requirements when representative proof checks regress/i); + assert.match(srs, /REQ-PSM-027/); + assert.match(srs, /machine-readable escalation step/i); + assert.match(srs, /REQ-PSM-028/); + assert.match(srs, /shared local proof program selector/i); + + assert.match(rtm, /REQ-PSM-012/); + assert.match(rtm, /TEST-PSM-012/); + assert.match(rtm, /REQ-PSM-013/); + assert.match(rtm, /TEST-PSM-013/); + assert.match(rtm, /REQ-PSM-014/); + assert.match(rtm, /TEST-PSM-014/); + assert.match(rtm, /REQ-PSM-015/); + assert.match(rtm, /TEST-PSM-015/); + assert.match(rtm, /REQ-PSM-016/); + assert.match(rtm, /TEST-PSM-016/); + assert.match(rtm, /REQ-PSM-017/); + assert.match(rtm, /TEST-PSM-017/); + assert.match(rtm, /REQ-PSM-018/); + assert.match(rtm, /TEST-PSM-018/); + assert.match(rtm, /REQ-PSM-019/); + assert.match(rtm, /TEST-PSM-019/); + assert.match(rtm, /REQ-PSM-020/); + assert.match(rtm, /TEST-PSM-020/); + assert.match(rtm, /REQ-PSM-021/); + assert.match(rtm, /TEST-PSM-021/); + assert.match(rtm, /REQ-PSM-022/); + assert.match(rtm, /TEST-PSM-022/); + assert.match(rtm, /REQ-PSM-023/); + assert.match(rtm, /TEST-PSM-023/); + assert.match(rtm, /REQ-PSM-024/); + assert.match(rtm, /TEST-PSM-024/); + assert.match(rtm, /REQ-PSM-025/); + assert.match(rtm, /TEST-PSM-025/); + assert.match(rtm, /REQ-PSM-026/); + assert.match(rtm, /TEST-PSM-026/); + assert.match(rtm, /REQ-PSM-027/); + assert.match(rtm, /TEST-PSM-027/); + assert.match(rtm, /REQ-PSM-028/); + assert.match(rtm, /TEST-PSM-028/); + + assert.match(plan, /TEST-PSM-012[\s\S]*named-pack and execution-group coverage/i); + assert.match(plan, /TEST-PSM-013[\s\S]*local path-hygiene coverage/i); + assert.match(plan, /Run-PesterExecutionOnly\.Local\.PathHygiene\.Tests\.ps1/); + assert.match(plan, /TEST-PSM-014[\s\S]*retained-artifact replay coverage/i); + assert.match(plan, /Replay-PesterServiceModelArtifacts\.Local\.Tests\.ps1/); + assert.match(plan, /TEST-PSM-015[\s\S]*side-effect ownership coverage/i); + assert.match(plan, /TEST-PSM-016[\s\S]*durable progress telemetry coverage/i); + assert.match(plan, /TEST-PSM-017[\s\S]*schema-governance coverage/i); + assert.match(plan, /TEST-PSM-018[\s\S]*promotion-comparison coverage/i); + assert.match(plan, /TEST-PSM-019[\s\S]*named entrypoint coverage/i); + assert.match(plan, /TEST-PSM-020[\s\S]*provenance coverage/i); + assert.match(plan, /TEST-PSM-021[\s\S]*operator-outcome coverage/i); + assert.match(plan, /TEST-PSM-022[\s\S]*local autonomy-loop coverage/i); + assert.match(plan, /TEST-PSM-023[\s\S]*autonomy-policy and stop-condition coverage/i); + assert.match(plan, /TEST-PSM-024[\s\S]*representative retained-artifact replay coverage/i); + assert.match(plan, /Replay-PesterServiceModelRepresentativeArtifact\.Tests\.ps1/); + assert.match(plan, /TEST-PSM-025[\s\S]*windows-container surface coverage/i); + assert.match(plan, /Invoke-PesterWindowsContainerSurfaceProbe\.Tests\.ps1/); + assert.match(plan, /TEST-PSM-026[\s\S]*proof-check aware autonomy coverage/i); + assert.match(plan, /TEST-PSM-027[\s\S]*next-step escalation coverage/i); + assert.match(plan, /TEST-PSM-028[\s\S]*shared local-program selector coverage/i); + + assert.match(doc, /named execution pack or test group/i); + assert.match(doc, /OneDrive-like paths are path-hygiene risk/i); + assert.match(doc, /PesterPathHygiene\.ps1/); + assert.match(doc, /PathHygieneMode/i); + assert.match(doc, /reproducible locally from mounted artifacts/i); + assert.match(doc, /Invoke-PesterEvidenceClassification\.ps1/); + assert.match(doc, /dispatcher-events\.ndjson/i); + assert.match(doc, /pester-execution-telemetry\.json/i); + assert.match(doc, /tests:replay:local/); + assert.match(doc, /Invoke-PesterTests\.ps1[\s\S]*operator-facing side effects directly/i); + assert.match(doc, /durable progress telemetry/i); + assert.match(doc, /schema versions explicitly/i); + assert.match(doc, /unsupported-schema/i); + assert.match(doc, /retained promotion evidence compares representative named packs/i); + assert.match(doc, /pester-service-model-promotion-comparison\.json/i); + assert.match(doc, /stable named entrypoints or wrappers/i); + assert.match(doc, /retain provenance/i); + assert.match(doc, /pester-evidence-provenance\.json/i); + assert.match(doc, /release-evidence-provenance\.json/i); + assert.match(doc, /promotion-dossier-provenance\.json/i); + assert.match(doc, /classification, reason chain, and next-step context/i); + assert.match(doc, /pester-operator-outcome\.json/i); + assert.match(doc, /ranked requirement backlog and a selected next requirement/i); + assert.match(doc, /preferred commands, stop conditions, and escalation conditions/i); + assert.match(doc, /representative retained-artifact replay/i); + assert.match(doc, /pester-windows-container-surface\.json/i); + assert.match(doc, /Windows-container surrogate/i); + assert.match(doc, /reachable Windows host bridge/i); + assert.match(doc, /reopen implemented requirements when proof checks regress/i); + assert.match(doc, /pester-service-model-next-step\.json/i); + assert.match(doc, /machine-readable escalation step/i); + assert.match(doc, /comparevi-local-program-next-step\.json/i); + assert.match(doc, /shared `windows-docker-desktop-ni-image` escalations should merge/i); +}); + +test('assurance packet records failure-detail producer consistency as an implemented execution requirement', () => { + const srs = readRepoFile('docs/requirements-pester-service-model-srs.md'); + const rtm = readRepoFile('docs/rtm-pester-service-model.csv'); + const plan = readRepoFile('docs/testing/pester-service-model-test-plan.md'); + const doc = readRepoFile('docs/knowledgebase/Pester-Service-Model.md'); + const dispatcher = readRepoFile('Invoke-PesterTests.ps1'); + const finalize = readRepoFile('tools/Invoke-PesterExecutionFinalize.ps1'); + + assert.match(srs, /REQ-PSM-011/); + assert.match(srs, /explicit machine-readable unavailable-details state/i); + assert.match(rtm, /REQ-PSM-011/); + assert.match(rtm, /TEST-PSM-011/); + assert.match(rtm, /PesterFailureProducerConsistency\.Tests\.ps1/); + assert.match(rtm, /Implemented/); + assert.match(plan, /Failure-detail producer consistency coverage/i); + assert.match(plan, /PesterFailureProducerConsistency\.Tests\.ps1/); + assert.match(doc, /failureDetailsStatus/i); + assert.match(doc, /pester-failures@v2/i); + assert.match(dispatcher, /Sync-PesterFailurePayload -Directory \$resultsDir -SummaryObject \$jsonObj/); + assert.match(finalize, /Sync-PesterFailurePayload -Directory \$resultsDir -SummaryObject \$summaryPayloadToWrite/); +}); diff --git a/tools/priority/__tests__/pester-service-model-release-evidence-provenance.test.mjs b/tools/priority/__tests__/pester-service-model-release-evidence-provenance.test.mjs new file mode 100644 index 000000000..d2baaeffa --- /dev/null +++ b/tools/priority/__tests__/pester-service-model-release-evidence-provenance.test.mjs @@ -0,0 +1,120 @@ +#!/usr/bin/env node + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; + +const repoRoot = process.cwd(); + +function runNode(scriptRelativePath, args, extraEnv = {}) { + return execFileSync( + process.execPath, + [path.join(repoRoot, scriptRelativePath), ...args], + { + cwd: repoRoot, + encoding: 'utf8', + env: { + ...process.env, + GITHUB_REPOSITORY: 'LabVIEW-Community-CI-CD/compare-vi-cli-action', + GITHUB_WORKFLOW: 'Pester service-model release evidence', + GITHUB_EVENT_NAME: 'workflow_dispatch', + GITHUB_RUN_ID: '1234567890', + GITHUB_RUN_ATTEMPT: '2', + GITHUB_REF: 'refs/heads/integration/pester-service-model', + GITHUB_REF_NAME: 'integration/pester-service-model', + GITHUB_SHA: '0123456789abcdef0123456789abcdef01234567', + GITHUB_SERVER_URL: 'https://github.com', + ...extraEnv + } + } + ); +} + +test('release evidence materializer emits provenance for retained hosted inputs and bundle outputs', async (t) => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'psm-release-evidence-')); + t.after(() => fs.rmSync(tempRoot, { recursive: true, force: true })); + + const baseDir = path.join(tempRoot, 'base'); + const outputDir = path.join(tempRoot, 'bundle'); + fs.mkdirSync(baseDir, { recursive: true }); + fs.writeFileSync(path.join(baseDir, 'coverage.xml'), '\n', 'utf8'); + fs.writeFileSync(path.join(baseDir, 'docs-link-check.json'), '{"links":[]}\n', 'utf8'); + + runNode( + 'tools/priority/materialize-pester-service-model-release-evidence.mjs', + [ + '--repo-root', repoRoot, + '--base-dir', baseDir, + '--output-dir', outputDir, + '--version', 'v9.9.9', + '--upstream-issue', '2069', + '--fork-issue', '2078', + '--fork-basis-commit', 'deadbeef', + '--fork-basis-url', 'https://example.test/fork-basis' + ] + ); + + const provenancePath = path.join(outputDir, 'release-evidence-provenance.json'); + assert.ok(fs.existsSync(provenancePath)); + const provenance = JSON.parse(fs.readFileSync(provenancePath, 'utf8')); + assert.equal(provenance.schema, 'pester-derived-provenance@v1'); + assert.equal(provenance.provenanceKind, 'release-evidence'); + assert.equal(provenance.subject.baselineVersion, 'v9.9.9'); + assert.equal(provenance.runContext.runId, '1234567890'); + assert.equal(provenance.runContext.refName, 'integration/pester-service-model'); + assert.ok(provenance.sourceInputs.some((entry) => entry.role === 'coverage-xml' && entry.present)); + assert.ok(provenance.sourceInputs.some((entry) => entry.role === 'docs-link-check' && entry.present)); + assert.ok(provenance.sourceInputs.some((entry) => entry.role === 'promotion-comparison' && entry.present)); + assert.ok(provenance.derivedOutputs.some((entry) => entry.role === 'release-record' && entry.present)); + assert.ok(provenance.derivedOutputs.some((entry) => entry.role === 'requirements-srs' && entry.present)); + assert.ok(fs.existsSync(path.join(outputDir, 'pester-service-model-promotion-comparison.json'))); +}); + +test('promotion dossier render emits provenance that points back to the release-evidence bundle', async (t) => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'psm-promotion-dossier-')); + t.after(() => fs.rmSync(tempRoot, { recursive: true, force: true })); + + const baseDir = path.join(tempRoot, 'base'); + const outputDir = path.join(tempRoot, 'bundle'); + fs.mkdirSync(baseDir, { recursive: true }); + fs.writeFileSync(path.join(baseDir, 'coverage.xml'), '\n', 'utf8'); + fs.writeFileSync(path.join(baseDir, 'docs-link-check.json'), '{"links":[]}\n', 'utf8'); + + runNode( + 'tools/priority/materialize-pester-service-model-release-evidence.mjs', + [ + '--repo-root', repoRoot, + '--base-dir', baseDir, + '--output-dir', outputDir, + '--version', 'v9.9.9' + ] + ); + runNode( + 'tools/priority/render-pester-service-model-promotion-dossier.mjs', + [ + '--repo-root', repoRoot, + '--release-evidence-dir', outputDir, + '--upstream-issue', '2069', + '--fork-issue', '2078' + ] + ); + + const provenancePath = path.join(outputDir, 'promotion-dossier-provenance.json'); + const dossierPath = path.join(outputDir, 'promotion-dossier.md'); + assert.ok(fs.existsSync(dossierPath)); + assert.ok(fs.existsSync(provenancePath)); + const provenance = JSON.parse(fs.readFileSync(provenancePath, 'utf8')); + assert.equal(provenance.schema, 'pester-derived-provenance@v1'); + assert.equal(provenance.provenanceKind, 'promotion-dossier'); + const dossier = fs.readFileSync(dossierPath, 'utf8'); + assert.match(dossier, /Representative Pack Comparisons/); + assert.match(dossier, /dispatcher-first-slice-baseline-vs-service-model/); + assert.match(dossier, /23818978524/); + assert.match(dossier, /23795198442/); + assert.ok(provenance.sourceInputs.some((entry) => entry.role === 'release-evidence-provenance' && entry.present)); + assert.ok(provenance.sourceInputs.some((entry) => entry.role === 'promotion-comparison' && entry.present)); + assert.ok(provenance.derivedOutputs.some((entry) => entry.role === 'promotion-dossier' && entry.present)); +}); diff --git a/tools/priority/__tests__/pester-service-model-release-evidence-workflow-contract.test.mjs b/tools/priority/__tests__/pester-service-model-release-evidence-workflow-contract.test.mjs index 47829c126..23c2f7b74 100644 --- a/tools/priority/__tests__/pester-service-model-release-evidence-workflow-contract.test.mjs +++ b/tools/priority/__tests__/pester-service-model-release-evidence-workflow-contract.test.mjs @@ -24,7 +24,13 @@ test('pester service-model release-evidence workflow retains hosted promotion ev assert.match(workflow, /coverage\.xml/); assert.match(workflow, /Docs link check \/ lychee/); assert.doesNotMatch(workflow, /fork-lane-local-assurance-ci\.mjs/); + assert.match(workflow, /pester-service-model-provenance\.mjs/); + assert.match(workflow, /pester-service-model-promotion-comparison\.json/); + assert.match(workflow, /pester-promotion-comparison-v1\.schema\.json/); + assert.match(workflow, /pester-service-model-release-evidence-provenance\.test\.mjs/); assert.match(workflow, /materialize-pester-service-model-release-evidence\.mjs/); assert.match(workflow, /render-pester-service-model-promotion-dossier\.mjs/); + assert.match(workflow, /release-evidence-provenance\.json/); + assert.match(workflow, /promotion-dossier-provenance\.json/); assert.match(workflow, /Upload release-evidence bundle/); }); diff --git a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs index 796d74dfb..24a088974 100644 --- a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs +++ b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs @@ -86,10 +86,15 @@ test('pester selection owns pack shaping and dispatcher profile resolution befor assert.match(workflow, /group:\s+pester-selection-\$\{\{\s*github\.event\.inputs\.sample_id \|\| inputs\.sample_id \|\| github\.ref\s*\}\}/); assert.match(workflow, /receipt_status:/); assert.match(workflow, /receipt_artifact_name:/); + assert.match(workflow, /execution_pack:/); assert.match(workflow, /Normalize include_integration/); - assert.match(workflow, /Shape include patterns/); + assert.match(workflow, /Resolve execution pack/); + assert.match(workflow, /Resolve-PesterExecutionPack/); assert.match(workflow, /Resolve dispatcher profile/); assert.match(workflow, /pester-selection-receipt@v1/); + assert.match(workflow, /executionPack/); + assert.match(workflow, /refineIncludePatterns/); + assert.match(workflow, /effectiveIncludePatterns/); assert.match(workflow, /integrationMode/); assert.match(workflow, /fixtureRequired/); assert.match(workflow, /Upload selection receipt/); @@ -115,16 +120,30 @@ test('pester run is execution-only and validates context, readiness, and selecti assert.match(workflow, /Download selection receipt artifact/); assert.match(workflow, /Validate selection receipt/); assert.match(workflow, /pester-selection\.json/); + assert.match(workflow, /execution_pack=/); + assert.match(workflow, /refine_include_patterns_json=/); assert.match(workflow, /selection-blocked/); assert.match(workflow, /Run Pester tests via local dispatcher/); + assert.match(workflow, /\$bound\.ExecutionPack = \$executionPack/); assert.match(workflow, /Postprocess execution results/); assert.match(workflow, /Invoke-PesterExecutionPostprocess\.ps1/); + assert.match(workflow, /Materialize execution telemetry/); + assert.match(workflow, /Invoke-PesterExecutionTelemetry\.ps1/); assert.match(workflow, /\$ErrorActionPreference = 'Continue'/); + assert.match(workflow, /\$stepOutputPath = \$env:GITHUB_OUTPUT/); + assert.match(workflow, /dispatcher-github-output\.txt/); assert.match(workflow, /"exit_code=\$exitCode"/); assert.match(workflow, /Write-Warning \("Dispatcher exited with code \{0\}" -f \$exitCode\)/); + assert.match(workflow, /if \(\$dispatcherExitCode -eq '' -and \(Test-Path -LiteralPath \$dispatcherOutputTrace\)\)/); assert.match(workflow, /results-xml-truncated/); assert.match(workflow, /invalid-results-xml/); assert.match(workflow, /missing-results-xml/); + assert.match(workflow, /unsupported-schema/); + assert.match(workflow, /selectionExecutionPack/); + assert.match(workflow, /telemetryStatus/); + assert.match(workflow, /telemetryLastKnownPhase/); + assert.match(workflow, /telemetryEventCount/); + assert.match(workflow, /pester-execution-telemetry\.json/); assert.match(workflow, /pester-run-receipt\.json/); assert.match(workflow, /Upload raw Pester execution artifact/); assert.match(workflow, /Emit execution contract/); @@ -140,40 +159,62 @@ test('pester run is execution-only and validates context, readiness, and selecti test('pester evidence distinguishes context-blocked, selection-blocked, and readiness-blocked skips from seam defects', () => { const workflow = readRepoFile('.github/workflows/pester-evidence.yml'); + const classificationTool = readRepoFile('tools/Invoke-PesterEvidenceClassification.ps1'); + const totalsTool = readRepoFile('tools/Write-PesterTotals.ps1'); assert.match(workflow, /name:\s+Pester evidence/); assert.match(workflow, /runs-on:\s+ubuntu-latest/); assert.match(workflow, /execution_receipt_artifact_name:/); assert.match(workflow, /Download execution receipt artifact/); assert.match(workflow, /Download raw execution artifact/); + assert.match(workflow, /name:\s+\$\{\{\s*steps\.artifact_name\.outputs\.name\s*\}\}\s*\n\s+path:\s+tests\/results/); assert.match(workflow, /Validate execution receipt artifact/); - assert.match(workflow, /execution-receipt-missing/); + assert.match(workflow, /PesterServiceModelSchema\.ps1/); + assert.match(workflow, /status=unsupported-schema/); + assert.match(workflow, /execution_pack=/); assert.match(workflow, /Write-PesterSummaryToStepSummary\.ps1/); assert.match(workflow, /Ensure-SessionIndex\.ps1/); + assert.match(workflow, /Write-PesterTotals\.ps1/); + assert.match(workflow, /Invoke-PesterEvidenceClassification\.ps1/); + assert.match(workflow, /Invoke-PesterOperatorOutcome\.ps1/); assert.match(workflow, /Invoke-DevDashboard\.ps1/); - assert.match(workflow, /classification = 'seam-defect'/); - assert.match(workflow, /\$classification = 'context-blocked'/); - assert.match(workflow, /\$classification = 'selection-blocked'/); - assert.match(workflow, /\$classification = 'readiness-blocked'/); - assert.match(workflow, /\$classification = 'results-xml-truncated'/); - assert.match(workflow, /\$classification = 'invalid-results-xml'/); - assert.match(workflow, /\$classification = 'missing-results-xml'/); - assert.match(workflow, /execution-receipt-results-xml-truncated/); - assert.match(workflow, /execution-receipt-invalid-results-xml/); - assert.match(workflow, /execution-receipt-missing-results-xml/); - assert.match(workflow, /results-xml-status=/); - assert.match(workflow, /\$contextStatus -ne 'ready'/); - assert.match(workflow, /\$selectionStatus -ne 'ready'/); - assert.match(workflow, /\$executionReceiptStatus -eq 'selection-blocked'/); - assert.match(workflow, /\$executionReceiptStatus -eq 'context-blocked'/); - assert.match(workflow, /\$executionReceiptStatus -eq 'results-xml-truncated'/); - assert.match(workflow, /\$executionReceiptStatus -eq 'invalid-results-xml'/); - assert.match(workflow, /\$executionReceiptStatus -eq 'missing-results-xml'/); - assert.match(workflow, /\$readinessStatus -ne 'ready' -and \$executionJobResult -in @\('skipped','cancelled'\)/); - assert.match(workflow, /raw-artifact-download=/); - assert.match(workflow, /execution-receipt-seam-defect/); assert.match(workflow, /Upload evidence artifact/); assert.match(workflow, /Propagate gate outcome/); + assert.match(workflow, /classification=\$\{\{\s*steps\.classify\.outputs\.classification\s*\}\}/); + assert.match(workflow, /next_action=\$\{\{\s*steps\.operator_outcome\.outputs\.next_action\s*\}\}/); + assert.match(workflow, /pester-operator-outcome\.json/); + assert.match(workflow, /pester-evidence-classification\.json/); + assert.match(workflow, /Invoke-PesterEvidenceProvenance\.ps1/); + assert.match(workflow, /pester-evidence-provenance\.json/); + assert.match(classificationTool, /execution-receipt-missing/); + assert.match(classificationTool, /classification = 'seam-defect'/); + assert.match(classificationTool, /function Get-OptionalStringProperty/); + assert.match(classificationTool, /selectionExecutionPack = Get-OptionalStringProperty/); + assert.match(classificationTool, /\$classification = 'context-blocked'/); + assert.match(classificationTool, /\$classification = 'selection-blocked'/); + assert.match(classificationTool, /\$classification = 'readiness-blocked'/); + assert.match(classificationTool, /\$classification = 'results-xml-truncated'/); + assert.match(classificationTool, /\$classification = 'invalid-results-xml'/); + assert.match(classificationTool, /\$classification = 'missing-results-xml'/); + assert.match(classificationTool, /\$classification = 'unsupported-schema'/); + assert.match(classificationTool, /execution-job-unsupported-schema/); + assert.match(classificationTool, /execution-receipt-unsupported-schema/); + assert.match(classificationTool, /execution-receipt-results-xml-truncated/); + assert.match(classificationTool, /execution-receipt-invalid-results-xml/); + assert.match(classificationTool, /execution-receipt-missing-results-xml/); + assert.match(classificationTool, /results-xml-status=/); + assert.match(classificationTool, /\$effectiveContextStatus -ne 'ready'/); + assert.match(classificationTool, /\$effectiveSelectionStatus -ne 'ready'/); + assert.match(classificationTool, /\$executionReceiptStatus -eq 'selection-blocked'/); + assert.match(classificationTool, /\$executionReceiptStatus -eq 'context-blocked'/); + assert.match(classificationTool, /\$executionReceiptStatus -eq 'results-xml-truncated'/); + assert.match(classificationTool, /\$executionReceiptStatus -eq 'invalid-results-xml'/); + assert.match(classificationTool, /\$executionReceiptStatus -eq 'missing-results-xml'/); + assert.match(classificationTool, /\$effectiveReadinessStatus -ne 'ready' -and \$effectiveExecutionJobResult -in @\('skipped', 'cancelled'\)/); + assert.match(classificationTool, /raw-artifact-download=/); + assert.match(classificationTool, /execution-receipt-seam-defect/); + assert.match(totalsTool, /schema = 'pester-totals\/v1'/); + assert.match(totalsTool, /status = 'missing-summary'/); }); test('knowledgebase documents the additive service model and keeps the monolith as the current baseline', () => { @@ -189,6 +230,11 @@ test('knowledgebase documents the additive service model and keeps the monolith assert.match(doc, /Selection resolves integration mode, include patterns, and dispatcher profile into a receipt/i); assert.match(doc, /readiness receipt/i); assert.match(doc, /execution receipt/i); + assert.match(doc, /durable progress telemetry/i); + assert.match(doc, /pester-execution-telemetry\.json/i); + assert.match(doc, /pester-evidence-provenance\.json/i); + assert.match(doc, /release-evidence-provenance\.json/i); + assert.match(doc, /promotion-dossier-provenance\.json/i); assert.match(doc, /existing required gate remains in place/i); }); diff --git a/tools/priority/__tests__/vi-history-local-ci.test.mjs b/tools/priority/__tests__/vi-history-local-ci.test.mjs new file mode 100644 index 000000000..1b1a14e39 --- /dev/null +++ b/tools/priority/__tests__/vi-history-local-ci.test.mjs @@ -0,0 +1,219 @@ +#!/usr/bin/env node + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { applyAutonomyPolicy, deriveEscalations, determinePhase, parseCsv, parseRequirementNumber, rankProofRegressions, rankRequirementGaps, runLiveHistoryCandidateProof, selectNextStep } from '../vi-history-local-ci.mjs'; + +test('parseRequirementNumber extracts numeric VI History local-proof ids', () => { + assert.equal(parseRequirementNumber('REQ-VHLP-001'), 1); + assert.equal(parseRequirementNumber('REQ-VHLP-006'), 6); +}); + +test('determinePhase groups VI History requirements into foundation and autonomy', () => { + assert.equal(determinePhase(1), 'foundation'); + assert.equal(determinePhase(4), 'foundation'); + assert.equal(determinePhase(5), 'autonomy'); +}); + +test('parseCsv handles quoted VI History RTM rows', () => { + const rows = parseCsv([ + 'ReqID,Requirement,Source,Priority,TestID,TestArtifact,CodeRef,Status', + 'REQ-VHLP-002,"Local refinement profiles remain stable, including windows-mirror-proof",docs/requirements.md,High,TEST-VHLP-002,"Planned local profile coverage",tools/Invoke-VIHistoryLocalRefinement.ps1,Gap' + ].join('\n')); + + assert.equal(rows.length, 1); + assert.equal(rows[0].ReqID, 'REQ-VHLP-002'); + assert.match(rows[0].Requirement, /windows-mirror-proof/); +}); + +test('rankRequirementGaps prefers earliest high-priority VI History local gaps', () => { + const ranked = rankRequirementGaps([ + { + ReqID: 'REQ-VHLP-006', + Requirement: 'Escalation packet exists for shared Windows surface.', + Source: 'docs/requirements.md', + Priority: 'High', + TestID: 'TEST-VHLP-006', + TestArtifact: 'Planned escalation coverage', + CodeRef: 'tools/priority/vi-history-local-ci.mjs', + Status: 'Gap' + }, + { + ReqID: 'REQ-VHLP-001', + Requirement: 'Windows workflow replay lane exists.', + Source: 'docs/requirements.md', + Priority: 'High', + TestID: 'TEST-VHLP-001', + TestArtifact: 'Planned replay coverage', + CodeRef: 'tools/priority/windows-workflow-replay-lane.mjs', + Status: 'Gap' + } + ]); + + assert.equal(ranked[0].req_id, 'REQ-VHLP-001'); + assert.equal(ranked[0].phase, 'foundation'); +}); + +test('applyAutonomyPolicy prioritizes active VI History worktree matches', () => { + const ranked = rankRequirementGaps([ + { + ReqID: 'REQ-VHLP-002', + Requirement: 'Local refinement profiles remain stable.', + Source: 'docs/requirements.md', + Priority: 'High', + TestID: 'TEST-VHLP-002', + TestArtifact: 'Planned local profile coverage', + CodeRef: 'tools/Invoke-VIHistoryLocalRefinement.ps1', + Status: 'Gap' + } + ]); + const policy = JSON.parse(fs.readFileSync(path.join(process.cwd(), 'tools/priority/vi-history-local-proof-autonomy-policy.json'), 'utf8')); + const guided = applyAutonomyPolicy(ranked, policy, ['tools/Invoke-VIHistoryLocalRefinement.ps1']); + + assert.equal(guided[0].active_now, true); + assert.equal(guided[0].mode, 'local-first'); +}); + +test('rankProofRegressions reopens implemented VI History requirements when proof checks fail', () => { + const regressions = rankProofRegressions([ + { + id: 'windows-workflow-replay', + owner_requirement: 'REQ-VHLP-001', + status: 'fail', + blocking: true, + summary: 'Windows workflow replay lane failed.' + } + ], [ + { + ReqID: 'REQ-VHLP-001', + Requirement: 'Windows workflow replay lane exists.', + Source: 'docs/requirements.md', + Priority: 'High', + TestID: 'TEST-VHLP-001', + TestArtifact: 'Replay lane proof', + CodeRef: 'tools/priority/windows-workflow-replay-lane.mjs', + Status: 'Implemented' + } + ]); + + assert.equal(regressions.length, 1); + assert.equal(regressions[0].status, 'Regression'); + assert.equal(regressions[0].proof_check_id, 'windows-workflow-replay'); +}); + +test('deriveEscalations emits a shared Windows-surface escalation for VI History', () => { + const escalations = deriveEscalations([ + { + id: 'windows-workflow-replay', + owner_requirement: 'REQ-VHLP-001', + status: 'advisory', + blocking: false, + summary: 'VI History Windows workflow replay is unavailable from the current host.', + current_surface_status: 'unavailable', + current_host_platform: 'Unix', + receipt_path: 'tests/results/docker-tools-parity/workflow-replay/vi-history-scenarios-windows-receipt.json', + recommended_commands: [ + 'npm run docker:ni:windows:bootstrap', + 'npm run compare:docker:ni:windows:probe', + 'npm run priority:workflow:replay:windows:vi-history' + ] + } + ]); + + assert.equal(escalations.length, 1); + assert.equal(escalations[0].governing_requirement, 'REQ-VHLP-006'); + assert.equal(escalations[0].blocked_requirement, 'REQ-VHLP-001'); + assert.equal(escalations[0].required_surface, 'windows-docker-desktop-ni-image'); +}); + +test('deriveEscalations emits a clone-backed live-history escalation for VI History', () => { + const escalations = deriveEscalations([ + { + id: 'live-history-candidate', + owner_requirement: 'REQ-VHLP-009', + status: 'advisory', + blocking: false, + summary: 'The governed clone-backed VI History candidate is not cloned locally yet.', + current_surface_status: 'missing-clone', + current_host_platform: 'Unix', + receipt_path: 'tests/results/_agent/vi-history-local-proof/local-ci/live-candidate/vi-history-live-candidate-readiness.json', + reason: 'No local clone was found.', + recommended_commands: [ + 'git clone https://github.com/ni/labview-icon-editor.git /tmp/labview-icon-editor' + ] + } + ]); + + assert.equal(escalations.length, 1); + assert.equal(escalations[0].governing_requirement, 'REQ-VHLP-009'); + assert.equal(escalations[0].blocked_requirement, 'REQ-VHLP-008'); + assert.equal(escalations[0].required_surface, 'clone-backed-live-history-candidate'); +}); + +test('runLiveHistoryCandidateProof validates a clone-backed target with real git history', async () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'vi-history-candidate-')); + const repoDir = path.join(tempRoot, 'repo'); + const resultsDir = path.join(tempRoot, 'results'); + const candidatePath = path.join(tempRoot, 'candidate.json'); + const targetPath = path.join(repoDir, 'Tooling', 'deployment', 'VIP_Pre-Uninstall Custom Action.vi'); + + fs.mkdirSync(path.dirname(targetPath), { recursive: true }); + spawnSync('git', ['init', '-b', 'develop', repoDir], { encoding: 'utf8' }); + fs.writeFileSync(targetPath, 'v1', 'utf8'); + spawnSync('git', ['-C', repoDir, 'add', '.'], { encoding: 'utf8' }); + spawnSync('git', ['-C', repoDir, '-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-m', 'initial'], { encoding: 'utf8' }); + fs.writeFileSync(targetPath, 'v2', 'utf8'); + spawnSync('git', ['-C', repoDir, 'add', '.'], { encoding: 'utf8' }); + spawnSync('git', ['-C', repoDir, '-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-m', 'second'], { encoding: 'utf8' }); + + fs.writeFileSync(candidatePath, JSON.stringify({ + $schema: '../../docs/schemas/vi-history-live-candidate-v1.schema.json', + schemaVersion: '1.0.0', + id: 'test-live-candidate', + repoSlug: 'example/repo', + repoUrl: 'https://github.com/example/repo', + defaultBranch: 'develop', + cloneRootEnvVar: 'COMPAREVI_VI_HISTORY_CANDIDATE_ROOT', + preferredLocalCloneRoots: [repoDir], + targetViPath: 'Tooling/deployment/VIP_Pre-Uninstall Custom Action.vi', + historyExpectation: { minCommits: 2 }, + iterationRationale: 'test' + }, null, 2), 'utf8'); + + const check = await runLiveHistoryCandidateProof(process.cwd(), resultsDir, path.relative(process.cwd(), candidatePath)); + + assert.equal(check.status, 'pass'); + assert.equal(check.id, 'live-history-candidate'); + assert.match(check.summary, /ready for local iteration/i); +}); + +test('selectNextStep prefers requirements before VI History escalations', () => { + const requirement = { + req_id: 'REQ-VHLP-002', + priority: 'High', + status: 'Gap', + phase: 'foundation', + score: 1234, + why_now: 'Need refinement profile coverage.', + requirement: 'Profiles stay stable.', + test_id: 'TEST-VHLP-002', + code_refs: ['tools/Invoke-VIHistoryLocalRefinement.ps1'], + suggested_loop: ['Add local profile coverage.'] + }; + const escalation = { type: 'escalation', escalation_id: 'windows-docker-desktop-ni-image' }; + + assert.equal(selectNextStep(requirement, [escalation]).type, 'requirement'); + assert.equal(selectNextStep(null, [escalation]).type, 'escalation'); +}); + +test('VI History local CI uses a run-scoped audit bundle root', () => { + const source = fs.readFileSync(path.join(process.cwd(), 'tools/priority/vi-history-local-ci.mjs'), 'utf8'); + + assert.match(source, /createRunScopedBundleRoot/); + assert.match(source, /surface-bundle/); + assert.match(source, /run-\$\{Date\.now\(\)\}-\$\{process\.pid\}/); +}); diff --git a/tools/priority/__tests__/vi-history-local-proof-contract.test.mjs b/tools/priority/__tests__/vi-history-local-proof-contract.test.mjs new file mode 100644 index 000000000..470f63613 --- /dev/null +++ b/tools/priority/__tests__/vi-history-local-proof-contract.test.mjs @@ -0,0 +1,114 @@ +#!/usr/bin/env node + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { readFileSync } from 'node:fs'; + +const repoRoot = process.cwd(); + +function readRepoFile(relativePath) { + return readFileSync(path.join(repoRoot, relativePath), 'utf8'); +} + +test('package.json exposes explicit VI History local proof entrypoints', () => { + const packageJson = JSON.parse(readRepoFile('package.json')); + + assert.equal( + packageJson.scripts['priority:workflow:replay:windows:vi-history'], + 'node tools/priority/windows-workflow-replay-lane.mjs --mode vi-history-scenarios-windows' + ); + assert.equal( + packageJson.scripts['history:local:proof'], + 'pwsh -NoLogo -NoProfile -File tools/Invoke-VIHistoryLocalRefinement.ps1 -Profile proof' + ); + assert.equal( + packageJson.scripts['history:local:refine'], + 'pwsh -NoLogo -NoProfile -File tools/Invoke-VIHistoryLocalRefinement.ps1 -Profile dev-fast' + ); + assert.equal( + packageJson.scripts['history:local:operator:review'], + 'pwsh -NoLogo -NoProfile -File tools/Invoke-VIHistoryLocalOperatorSession.ps1 -Profile dev-fast' + ); + assert.equal( + packageJson.scripts['priority:vi-history:local-ci'], + 'node tools/priority/vi-history-local-ci.mjs' + ); + assert.equal( + packageJson.scripts['priority:vi-history:next-step'], + 'node tools/priority/vi-history-local-ci.mjs --print-next-step' + ); +}); + +test('VI History local-proof packet traces requirements, tests, and shared Windows-surface escalation', () => { + const srs = readRepoFile('docs/requirements-vi-history-local-proof-srs.md'); + const rtm = readRepoFile('docs/rtm-vi-history-local-proof.csv'); + const plan = readRepoFile('docs/testing/vi-history-local-proof-test-plan.md'); + const doc = readRepoFile('docs/knowledgebase/VI-History-Local-Proof.md'); + const arch = readRepoFile('docs/architecture/vi-history-local-proof-control-plane.md'); + const localCi = readRepoFile('tools/priority/vi-history-local-ci.mjs'); + + assert.match(srs, /REQ-VHLP-001/); + assert.match(srs, /windows workflow-replay lane/i); + assert.match(srs, /REQ-VHLP-002/); + assert.match(srs, /windows-mirror-proof/i); + assert.match(srs, /REQ-VHLP-003/); + assert.match(srs, /operator-session wrappers/i); + assert.match(srs, /REQ-VHLP-004/); + assert.match(srs, /workflow-readiness envelope/i); + assert.match(srs, /REQ-VHLP-005/); + assert.match(srs, /machine-readable report, ranked backlog, and next step/i); + assert.match(srs, /REQ-VHLP-006/); + assert.match(srs, /windows-docker-desktop-ni-image/i); + assert.match(srs, /reachable Windows host bridge/i); + assert.match(srs, /REQ-VHLP-007/); + assert.match(srs, /shared local proof program selector/i); + assert.match(srs, /REQ-VHLP-008/); + assert.match(srs, /ni\/labview-icon-editor/i); + assert.match(srs, /VIP_Pre-Uninstall Custom Action\.vi/); + assert.match(srs, /REQ-VHLP-009/); + assert.match(srs, /clone exists locally, contains the target VI, and exposes real git history/i); + + assert.match(rtm, /REQ-VHLP-006/); + assert.match(rtm, /TEST-VHLP-006/); + assert.match(rtm, /REQ-VHLP-007/); + assert.match(rtm, /TEST-VHLP-007/); + assert.match(rtm, /REQ-VHLP-008/); + assert.match(rtm, /TEST-VHLP-008/); + assert.match(rtm, /REQ-VHLP-009/); + assert.match(rtm, /TEST-VHLP-009/); + + assert.match(plan, /TEST-VHLP-001/); + assert.match(plan, /TEST-VHLP-006/); + assert.match(plan, /TEST-VHLP-007/); + assert.match(plan, /machine-readable escalation step/i); + assert.match(plan, /reachable Windows host bridge/i); + assert.match(plan, /TEST-VHLP-008/); + assert.match(plan, /ni\/labview-icon-editor/i); + assert.match(plan, /TEST-VHLP-009/); + assert.match(plan, /clone presence, target path presence, and git history/i); + + assert.match(doc, /priority:workflow:replay:windows:vi-history/); + assert.match(doc, /history:local:proof/); + assert.match(doc, /vi-history-local-next-step\.json/); + assert.match(doc, /comparevi-local-program-next-step\.json/); + assert.match(doc, /shared `windows-docker-desktop-ni-image` proof surface/i); + assert.match(doc, /reachable Windows Desktop is available behind WSL/i); + assert.match(doc, /vi-history-live-candidate\.json/); + assert.match(doc, /ni\/labview-icon-editor/i); + assert.match(doc, /VIP_Pre-Uninstall Custom Action\.vi/); + assert.match(doc, /UNC-backed WSL checkout/i); + assert.match(doc, /staged into a Windows-local mount root/i); + + assert.match(arch, /Windows workflow replay surface/); + assert.match(arch, /Local autonomy surface/); + assert.match(arch, /Clone-backed live-history candidate surface/); + + assert.match(localCi, /REQ-VHLP-006/); + assert.match(localCi, /windows-docker-desktop-ni-image/); + assert.match(localCi, /priority:workflow:replay:windows:vi-history/); + assert.match(localCi, /windows-host-bridge/i); + assert.match(localCi, /REQ-VHLP-009/); + assert.match(localCi, /clone-backed-live-history-candidate/); + assert.match(localCi, /vi-history-live-candidate-readiness\.json/); +}); diff --git a/tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs b/tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs new file mode 100644 index 000000000..ca5329b36 --- /dev/null +++ b/tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs @@ -0,0 +1,100 @@ +#!/usr/bin/env node + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { readFileSync } from 'node:fs'; + +const repoRoot = process.cwd(); + +function readRepoFile(relativePath) { + return readFileSync(path.join(repoRoot, relativePath), 'utf8'); +} + +test('package.json exposes explicit Windows shared-surface entrypoints', () => { + const packageJson = JSON.parse(readRepoFile('package.json')); + + assert.equal( + packageJson.scripts['priority:windows-surface:local-ci'], + 'node tools/priority/windows-docker-shared-surface-local-ci.mjs' + ); + assert.equal( + packageJson.scripts['priority:windows-surface:next-step'], + 'node tools/priority/windows-docker-shared-surface-local-ci.mjs --print-next-step' + ); + assert.equal( + packageJson.scripts['tests:windows-surface:probe'], + 'pwsh -NoLogo -NoProfile -File tools/Invoke-PesterWindowsContainerSurfaceProbe.ps1' + ); +}); + +test('Windows shared-surface packet traces requirements, tests, and shared-program integration', () => { + const srs = readRepoFile('docs/requirements-windows-docker-shared-surface-srs.md'); + const rtm = readRepoFile('docs/rtm-windows-docker-shared-surface.csv'); + const plan = readRepoFile('docs/testing/windows-docker-shared-surface-test-plan.md'); + const doc = readRepoFile('docs/knowledgebase/Windows-Docker-Shared-Surface.md'); + const arch = readRepoFile('docs/architecture/windows-docker-shared-surface-control-plane.md'); + const programDoc = readRepoFile('docs/knowledgebase/Local-Proof-Autonomy-Program.md'); + const localCi = readRepoFile('tools/priority/windows-docker-shared-surface-local-ci.mjs'); + + assert.match(srs, /REQ-WDSS-001/); + assert.match(srs, /Docker Desktop Windows engine/i); + assert.match(srs, /REQ-WDSS-003/); + assert.match(srs, /OneDrive-managed paths/i); + assert.match(srs, /REQ-WDSS-006/); + assert.match(srs, /shared local proof program selector/i); + assert.match(srs, /REQ-WDSS-007/); + assert.match(srs, /reachable Windows host bridge/i); + assert.match(srs, /REQ-WDSS-008/); + assert.match(srs, /UNC-backed WSL/i); + assert.match(srs, /Windows-local mount root/i); + + assert.match(rtm, /REQ-WDSS-003/); + assert.match(rtm, /TEST-WDSS-003/); + assert.match(rtm, /REQ-WDSS-006/); + assert.match(rtm, /TEST-WDSS-006/); + assert.match(rtm, /REQ-WDSS-007/); + assert.match(rtm, /TEST-WDSS-007/); + assert.match(rtm, /REQ-WDSS-008/); + assert.match(rtm, /TEST-WDSS-008/); + + assert.match(plan, /TEST-WDSS-001/); + assert.match(plan, /TEST-WDSS-002/); + assert.match(plan, /TEST-WDSS-003/); + assert.match(plan, /TEST-WDSS-006/); + assert.match(plan, /OneDrive-like managed roots/i); + assert.match(plan, /TEST-WDSS-007/); + assert.match(plan, /reachable Windows host bridge/i); + assert.match(plan, /TEST-WDSS-008/); + assert.match(plan, /UNC-backed WSL staging coverage/i); + + assert.match(doc, /priority:windows-surface:local-ci/); + assert.match(doc, /tests:windows-surface:probe/); + assert.match(doc, /docker:ni:windows:bootstrap/); + assert.match(doc, /OneDrive-like managed roots/i); + assert.match(doc, /reachable Windows host bridge/i); + assert.match(doc, /ExecutionPolicy Bypass/i); + assert.match(doc, /stage container-bound inputs and output targets/i); + assert.match(doc, /ni-windows-container-capture\.json/); + + assert.match(arch, /Readiness probe surface/); + assert.match(arch, /Path-hygiene surface/); + assert.match(arch, /shared Windows surface should stay packetized separately/i); + assert.match(arch, /Bridge surface/); + assert.match(arch, /Windows-local staging surface/); + assert.match(arch, /UNC-backed WSL repo paths should never be passed straight to Docker bind\s+mounts/i); + + assert.match(programDoc, /Windows Docker Shared Surface/i); + + assert.match(localCi, /REQ-WDSS-003/); + assert.match(localCi, /local-safe-root/); + assert.match(localCi, /windows-docker-desktop-ni-image/); + assert.match(localCi, /Invoke-PesterWindowsContainerSurfaceProbe\.ps1/); + assert.match(localCi, /Test-WindowsNI2026q1HostPreflight\.ps1/); + assert.match(localCi, /windows-host-bridge-unavailable/); + + const compareScript = readRepoFile('tools/Run-NIWindowsContainerCompare.ps1'); + assert.match(compareScript, /Test-PathRequiresWindowsDockerLocalStage/); + assert.match(compareScript, /outputSyncStatus/); + assert.match(compareScript, /cleanupStatus/); +}); diff --git a/tools/priority/__tests__/windows-docker-shared-surface-local-ci.test.mjs b/tools/priority/__tests__/windows-docker-shared-surface-local-ci.test.mjs new file mode 100644 index 000000000..a0da3f7ec --- /dev/null +++ b/tools/priority/__tests__/windows-docker-shared-surface-local-ci.test.mjs @@ -0,0 +1,205 @@ +#!/usr/bin/env node + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { applyAutonomyPolicy, deriveEscalations, determinePhase, parseCsv, parseRequirementNumber, rankProofRegressions, rankRequirementGaps, runPathHygieneProof, selectNextStep } from '../windows-docker-shared-surface-local-ci.mjs'; + +test('parseRequirementNumber extracts numeric shared-surface ids', () => { + assert.equal(parseRequirementNumber('REQ-WDSS-001'), 1); + assert.equal(parseRequirementNumber('REQ-WDSS-006'), 6); +}); + +test('determinePhase groups shared-surface requirements into foundation and autonomy', () => { + assert.equal(determinePhase(1), 'foundation'); + assert.equal(determinePhase(3), 'foundation'); + assert.equal(determinePhase(4), 'autonomy'); + assert.equal(determinePhase(7), 'autonomy'); +}); + +test('parseCsv handles quoted shared-surface RTM rows', () => { + const rows = parseCsv([ + 'ReqID,Requirement,Source,Priority,TestID,TestArtifact,CodeRef,Status', + 'REQ-WDSS-003,"Shared surface detects OneDrive-managed roots",docs/requirements.md,High,TEST-WDSS-003,"Planned path-hygiene coverage",tools/priority/windows-docker-shared-surface-local-ci.mjs,Gap' + ].join('\n')); + + assert.equal(rows.length, 1); + assert.equal(rows[0].ReqID, 'REQ-WDSS-003'); + assert.match(rows[0].Requirement, /OneDrive-managed roots/); +}); + +test('rankRequirementGaps prefers earliest high-priority shared-surface gaps', () => { + const ranked = rankRequirementGaps([ + { + ReqID: 'REQ-WDSS-005', + Requirement: 'Escalation packet exists for shared Windows surface.', + Source: 'docs/requirements.md', + Priority: 'High', + TestID: 'TEST-WDSS-005', + TestArtifact: 'Planned escalation coverage', + CodeRef: 'tools/priority/windows-docker-shared-surface-local-ci.mjs', + Status: 'Gap' + }, + { + ReqID: 'REQ-WDSS-001', + Requirement: 'Shared-surface readiness probe exists.', + Source: 'docs/requirements.md', + Priority: 'High', + TestID: 'TEST-WDSS-001', + TestArtifact: 'Planned probe coverage', + CodeRef: 'tools/Invoke-PesterWindowsContainerSurfaceProbe.ps1', + Status: 'Gap' + } + ]); + + assert.equal(ranked[0].req_id, 'REQ-WDSS-001'); + assert.equal(ranked[0].phase, 'foundation'); +}); + +test('applyAutonomyPolicy prioritizes active shared-surface worktree matches', () => { + const ranked = rankRequirementGaps([ + { + ReqID: 'REQ-WDSS-003', + Requirement: 'Shared surface detects OneDrive-managed roots.', + Source: 'docs/requirements.md', + Priority: 'High', + TestID: 'TEST-WDSS-003', + TestArtifact: 'Planned path-hygiene coverage', + CodeRef: 'tools/priority/windows-docker-shared-surface-local-ci.mjs', + Status: 'Gap' + } + ]); + const policy = JSON.parse(fs.readFileSync(path.join(process.cwd(), 'tools/priority/windows-docker-shared-surface-autonomy-policy.json'), 'utf8')); + const guided = applyAutonomyPolicy(ranked, policy, ['tools/priority/windows-docker-shared-surface-local-ci.mjs']); + + assert.equal(guided[0].active_now, true); + assert.equal(guided[0].mode, 'local-first'); +}); + +test('rankProofRegressions reopens implemented shared-surface requirements when proof checks fail', () => { + const regressions = rankProofRegressions([ + { + id: 'windows-surface', + owner_requirement: 'REQ-WDSS-001', + status: 'fail', + blocking: true, + summary: 'Shared Windows surface probe failed.' + } + ], [ + { + ReqID: 'REQ-WDSS-001', + Requirement: 'Shared-surface readiness probe exists.', + Source: 'docs/requirements.md', + Priority: 'High', + TestID: 'TEST-WDSS-001', + TestArtifact: 'Probe proof', + CodeRef: 'tools/Invoke-PesterWindowsContainerSurfaceProbe.ps1', + Status: 'Implemented' + } + ]); + + assert.equal(regressions.length, 1); + assert.equal(regressions[0].status, 'Regression'); + assert.equal(regressions[0].proof_check_id, 'windows-surface'); +}); + +test('deriveEscalations emits a path-hygiene relocation escalation for shared Windows surface', () => { + const escalations = deriveEscalations([ + { + id: 'path-hygiene', + owner_requirement: 'REQ-WDSS-003', + status: 'advisory', + blocking: false, + summary: 'The shared Windows surface is currently rooted in a synchronized path.', + current_surface_status: 'unsafe-synced-root', + current_host_platform: 'Windows', + receipt_path: 'tests/results/_agent/windows-docker-shared-surface/local-ci/path-hygiene/windows-docker-shared-surface-path-hygiene.json', + reason: 'OneDrive-like managed roots can mutate artifacts during live proof.', + recommended_commands: ['Move the repo to a safe local root.'] + } + ]); + + assert.equal(escalations.length, 1); + assert.equal(escalations[0].governing_requirement, 'REQ-WDSS-003'); + assert.equal(escalations[0].required_surface, 'local-safe-root'); +}); + +test('deriveEscalations emits a shared Windows-surface escalation when host is unavailable', () => { + const escalations = deriveEscalations([ + { + id: 'windows-surface', + owner_requirement: 'REQ-WDSS-001', + status: 'advisory', + blocking: false, + summary: 'The shared Windows surface is unavailable from the current host.', + current_surface_status: 'not-windows-host', + current_host_platform: 'Unix', + receipt_path: 'tests/results/_agent/windows-docker-shared-surface/local-ci/windows-surface/pester-windows-container-surface.json', + reason: 'Current host is not Windows.', + recommended_commands: ['npm run docker:ni:windows:bootstrap'] + } + ]); + + assert.equal(escalations.length, 1); + assert.equal(escalations[0].governing_requirement, 'REQ-WDSS-005'); + assert.equal(escalations[0].required_surface, 'windows-docker-desktop-ni-image'); +}); + +test('deriveEscalations keeps bridge-unavailable shared-surface advisories on the Windows surface escalation', () => { + const escalations = deriveEscalations([ + { + id: 'windows-host-preflight', + owner_requirement: 'REQ-WDSS-002', + status: 'advisory', + blocking: false, + summary: 'Deterministic Windows host preflight is unavailable.', + current_surface_status: 'windows-host-bridge-unavailable', + current_host_platform: 'Unix', + reason: 'No reachable Windows host bridge is available.', + recommended_commands: ['npm run docker:ni:windows:bootstrap'] + } + ]); + + assert.equal(escalations.length, 1); + assert.equal(escalations[0].governing_requirement, 'REQ-WDSS-005'); + assert.equal(escalations[0].required_surface, 'windows-docker-desktop-ni-image'); +}); + +test('runPathHygieneProof detects OneDrive-like roots', async () => { + const riskyRoot = path.join(process.cwd(), 'tests', 'results', '_agent', 'OneDrive - Contoso', 'windows-shared-surface'); + const proof = await runPathHygieneProof(riskyRoot, path.join(riskyRoot, 'results')); + + assert.equal(proof.status, 'advisory'); + assert.equal(proof.id, 'path-hygiene'); + assert.match(proof.summary, /synchronized or externally managed path/i); +}); + +test('selectNextStep prefers a requirement before a shared-surface escalation', () => { + const requirement = { + type: 'requirement', + req_id: 'REQ-WDSS-003', + priority: 'High', + status: 'Gap', + phase: 'foundation', + score: 1234, + why_now: 'Need path hygiene coverage.', + requirement: 'Shared surface stays off synced roots.', + test_id: 'TEST-WDSS-003', + code_refs: ['tools/priority/windows-docker-shared-surface-local-ci.mjs'], + suggested_loop: ['Add path hygiene coverage.'] + }; + const escalation = { type: 'escalation', required_surface: 'windows-docker-desktop-ni-image' }; + + assert.equal(selectNextStep(requirement, [escalation]).type, 'requirement'); + assert.equal(selectNextStep(null, [escalation]).type, 'escalation'); +}); + +test('Windows shared-surface local CI uses a run-scoped audit bundle root', () => { + const source = fs.readFileSync(path.join(process.cwd(), 'tools/priority/windows-docker-shared-surface-local-ci.mjs'), 'utf8'); + + assert.match(source, /createRunScopedBundleRoot/); + assert.match(source, /surface-bundle/); + assert.match(source, /run-\$\{Date\.now\(\)\}-\$\{process\.pid\}/); +}); diff --git a/tools/priority/__tests__/windows-host-bridge.test.mjs b/tools/priority/__tests__/windows-host-bridge.test.mjs new file mode 100644 index 000000000..2458e22fd --- /dev/null +++ b/tools/priority/__tests__/windows-host-bridge.test.mjs @@ -0,0 +1,132 @@ +#!/usr/bin/env node + +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { + buildWindowsNodeBridgeSpec, + buildWindowsPath, + buildWindowsPowerShellFileBridgeSpec, + detectWindowsHostBridge, + resolveRepoWindowsPath, +} from '../windows-host-bridge.mjs'; + +test('resolveRepoWindowsPath translates the repository root through wslpath', () => { + const result = resolveRepoWindowsPath('/tmp/repo', (command, args) => { + assert.equal(command, 'wslpath'); + assert.deepEqual(args, ['-w', '/tmp/repo']); + return { + status: 0, + stdout: '\\\\wsl.localhost\\Ubuntu\\tmp\\repo\r\n', + stderr: '', + error: null, + }; + }); + + assert.equal(result, '\\\\wsl.localhost\\Ubuntu\\tmp\\repo'); +}); + +test('detectWindowsHostBridge reports a reachable Windows bridge from Unix when PowerShell and node.exe are available', () => { + const bridge = detectWindowsHostBridge('/tmp/repo', { + platform: 'linux', + pathExists(candidate) { + return candidate === '/mnt/c/Program Files/PowerShell/7/pwsh.exe'; + }, + runProcessFn(command, args) { + if (command === 'wslpath') { + return { + status: 0, + stdout: '\\\\wsl.localhost\\Ubuntu\\tmp\\repo\n', + stderr: '', + error: null, + }; + } + + if ( + command === '/mnt/c/Program Files/PowerShell/7/pwsh.exe' && + args.includes('$PSVersionTable.PSVersion.ToString()') + ) { + return { + status: 0, + stdout: '7.5.0\n', + stderr: '', + error: null, + }; + } + + if ( + command === '/mnt/c/Program Files/PowerShell/7/pwsh.exe' && + args.includes('(Get-Command node.exe -ErrorAction Stop).Source') + ) { + return { + status: 0, + stdout: 'C:\\Program Files\\nodejs\\node.exe\r\n', + stderr: '', + error: null, + }; + } + + throw new Error(`Unexpected probe: ${command} ${args.join(' ')}`); + }, + }); + + assert.equal(bridge.status, 'reachable'); + assert.equal(bridge.bridge_mode, 'wsl-windows'); + assert.equal(bridge.coordinator_host_platform, 'Unix'); + assert.equal(bridge.current_host_platform, 'Windows'); + assert.equal(bridge.repo_root_windows, '\\\\wsl.localhost\\Ubuntu\\tmp\\repo'); + assert.equal(bridge.windows_pwsh_path, '/mnt/c/Program Files/PowerShell/7/pwsh.exe'); + assert.equal(bridge.windows_node_path, 'C:\\Program Files\\nodejs\\node.exe'); +}); + +test('detectWindowsHostBridge reports native mode on Windows coordinators', () => { + const bridge = detectWindowsHostBridge('C:\\repo', { platform: 'win32' }); + + assert.equal(bridge.status, 'native'); + assert.equal(bridge.bridge_mode, 'native-windows'); + assert.equal(bridge.coordinator_host_platform, 'Windows'); + assert.equal(bridge.current_host_platform, 'Windows'); + assert.equal(bridge.repo_root_windows, 'C:\\repo'); + assert.equal(bridge.windows_pwsh_path, 'pwsh'); +}); + +test('buildWindowsPath joins UNC and relative Windows segments correctly', () => { + const result = buildWindowsPath('\\\\wsl.localhost\\Ubuntu\\tmp\\repo', 'tools/priority/windows-host-bridge.mjs'); + assert.equal(result, '\\\\wsl.localhost\\Ubuntu\\tmp\\repo\\tools\\priority\\windows-host-bridge.mjs'); +}); + +test('buildWindowsPowerShellFileBridgeSpec produces an execution-policy-bypass bridge command', () => { + const spec = buildWindowsPowerShellFileBridgeSpec({ + bridge: { + repo_root_windows: '\\\\wsl.localhost\\Ubuntu\\tmp\\repo', + windows_pwsh_path: '/mnt/c/Program Files/PowerShell/7/pwsh.exe', + }, + scriptRelativePath: 'tools/Invoke-PesterWindowsContainerSurfaceProbe.ps1', + scriptArgs: ['-ResultsDir', 'C:\\Temp\\comparevi'], + }); + + assert.equal(spec.command, '/mnt/c/Program Files/PowerShell/7/pwsh.exe'); + assert.deepEqual(spec.args.slice(0, 4), ['-NoLogo', '-NoProfile', '-ExecutionPolicy', 'Bypass']); + assert.match(spec.script_path_windows, /Invoke-PesterWindowsContainerSurfaceProbe\.ps1$/); + assert.match(spec.args.at(-1), /Set-Location -LiteralPath/); + assert.match(spec.args.at(-1), /-ResultsDir/); + assert.doesNotMatch(spec.args.at(-1), /'-ResultsDir'/); +}); + +test('buildWindowsNodeBridgeSpec requires node.exe and produces a Windows-node bridge command', () => { + const spec = buildWindowsNodeBridgeSpec({ + bridge: { + repo_root_windows: '\\\\wsl.localhost\\Ubuntu\\tmp\\repo', + windows_pwsh_path: '/mnt/c/Program Files/PowerShell/7/pwsh.exe', + windows_node_path: 'C:\\Program Files\\nodejs\\node.exe', + }, + scriptRelativePath: 'tools/priority/windows-workflow-replay-lane.mjs', + scriptArgs: ['--mode', 'vi-history-scenarios-windows'], + }); + + assert.equal(spec.command, '/mnt/c/Program Files/PowerShell/7/pwsh.exe'); + assert.equal(spec.node_path_windows, 'C:\\Program Files\\nodejs\\node.exe'); + assert.match(spec.script_path_windows, /windows-workflow-replay-lane\.mjs$/); + assert.match(spec.args.at(-1), /node\.exe/); + assert.match(spec.args.at(-1), /--mode/); +}); diff --git a/tools/priority/comparevi-local-program-ci.mjs b/tools/priority/comparevi-local-program-ci.mjs new file mode 100644 index 000000000..f62071aa2 --- /dev/null +++ b/tools/priority/comparevi-local-program-ci.mjs @@ -0,0 +1,450 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; +import Ajv2020 from 'ajv/dist/2020.js'; +import addFormats from 'ajv-formats'; + +import { runPesterServiceModelLocalCi } from './pester-service-model-local-ci.mjs'; +import { runVIHistoryLocalCi } from './vi-history-local-ci.mjs'; +import { runWindowsDockerSharedSurfaceLocalCi } from './windows-docker-shared-surface-local-ci.mjs'; + +const repoRootDefault = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); +const DEFAULT_RESULTS_DIR = path.join('tests', 'results', '_agent', 'local-proof-program', 'local-ci'); +const DEFAULT_REPORT = path.join(DEFAULT_RESULTS_DIR, 'comparevi-local-program-ci-report.json'); +const DEFAULT_SUMMARY = path.join(DEFAULT_RESULTS_DIR, 'comparevi-local-program-ci-summary.md'); +const DEFAULT_NEXT_STEP = path.join(DEFAULT_RESULTS_DIR, 'comparevi-local-program-next-step.json'); + +const HELP = [ + 'Usage: node tools/priority/comparevi-local-program-ci.mjs [options]', + '', + 'Options:', + ` --repo-root (default: ${repoRootDefault})`, + ` --results-dir (default: ${DEFAULT_RESULTS_DIR})`, + ` --output (default: ${DEFAULT_REPORT})`, + ` --summary-output (default: ${DEFAULT_SUMMARY})`, + ` --next-step-output (default: ${DEFAULT_NEXT_STEP})`, + ' --print-next-step print the selected next step to stdout', + ' --help, -h' +]; + +const PACKETS = Object.freeze([ + Object.freeze({ + id: 'pester-service-model', + label: 'Pester Service Model', + reportDir: path.join('tests', 'results', '_agent', 'pester-service-model', 'local-ci'), + reportFile: 'pester-service-model-local-ci-report.json', + nextStepFile: 'pester-service-model-next-step.json' + }), + Object.freeze({ + id: 'vi-history-local-proof', + label: 'VI History Local Proof', + reportDir: path.join('tests', 'results', '_agent', 'vi-history-local-proof', 'local-ci'), + reportFile: 'vi-history-local-ci-report.json', + nextStepFile: 'vi-history-local-next-step.json' + }), + Object.freeze({ + id: 'windows-docker-shared-surface', + label: 'Windows Docker Shared Surface', + reportDir: path.join('tests', 'results', '_agent', 'windows-docker-shared-surface', 'local-ci'), + reportFile: 'windows-docker-shared-surface-local-ci-report.json', + nextStepFile: 'windows-docker-shared-surface-next-step.json' + }) +]); + +function parseArgs(argv = process.argv) { + const options = { + repoRoot: repoRootDefault, + resultsDir: DEFAULT_RESULTS_DIR, + outputPath: DEFAULT_REPORT, + summaryPath: DEFAULT_SUMMARY, + nextStepOutputPath: DEFAULT_NEXT_STEP, + printNextStep: false, + help: false + }; + + const args = argv.slice(2); + for (let i = 0; i < args.length; i += 1) { + const token = args[i]; + const next = args[i + 1]; + if (token === '--help' || token === '-h') { + options.help = true; + continue; + } + if (token === '--print-next-step') { + options.printNextStep = true; + continue; + } + if (['--repo-root', '--results-dir', '--output', '--summary-output', '--next-step-output'].includes(token)) { + if (!next || next.startsWith('-')) { + throw new Error(`Missing value for ${token}`); + } + i += 1; + if (token === '--repo-root') options.repoRoot = path.resolve(next); + if (token === '--results-dir') options.resultsDir = next; + if (token === '--output') options.outputPath = next; + if (token === '--summary-output') options.summaryPath = next; + if (token === '--next-step-output') options.nextStepOutputPath = next; + continue; + } + throw new Error(`Unknown option: ${token}`); + } + + return options; +} + +function printHelp() { + for (const line of HELP) console.log(line); +} + +async function ensureDir(filePath) { + await fs.mkdir(filePath, { recursive: true }); +} + +async function readJson(filePath) { + return JSON.parse(await fs.readFile(filePath, 'utf8')); +} + +function relativeFrom(repoRoot, filePath) { + return path.relative(repoRoot, filePath).split(path.sep).join('/'); +} + +async function writeJson(filePath, payload) { + await ensureDir(path.dirname(filePath)); + await fs.writeFile(filePath, JSON.stringify(payload, null, 2), 'utf8'); +} + +function weightPriority(priority) { + if (priority === 'High') return 300; + if (priority === 'Medium') return 150; + return 50; +} + +function weightStatus(status) { + if (status === 'Regression') return 1000; + if (status === 'Gap') return 600; + return 200; +} + +function weightPhase(phase) { + if (phase === 'foundation') return 250; + if (phase === 'execution-governance') return 220; + if (phase === 'promotion-governance') return 180; + if (phase === 'evidence-governance') return 160; + return 100; +} + +function buildRequirementCandidate(packet, reportPath, nextStepPath, nextStep) { + const sourceScore = Number.isFinite(Number(nextStep.score)) ? Number(nextStep.score) : 0; + return { + type: 'requirement', + packet_id: packet.id, + packet_label: packet.label, + source_report_path: reportPath, + source_next_step_path: nextStepPath, + ...nextStep, + program_score: + (nextStep.active_now ? 10000 : 0) + + weightStatus(nextStep.status) + + weightPriority(nextStep.priority) + + weightPhase(nextStep.phase) + + sourceScore + }; +} + +function rankProgramRequirements(requirements) { + return [...requirements] + .sort((left, right) => { + if (right.program_score !== left.program_score) return right.program_score - left.program_score; + if (left.packet_id !== right.packet_id) return left.packet_id.localeCompare(right.packet_id); + return left.req_id.localeCompare(right.req_id); + }) + .map((entry, index) => ({ ...entry, program_rank: index + 1 })); +} + +function dedupeStrings(values) { + const ordered = []; + const seen = new Set(); + for (const value of values) { + if (!value || seen.has(value)) continue; + seen.add(value); + ordered.push(value); + } + return ordered; +} + +function mergeSharedSurfaceEscalations(escalations) { + const groups = new Map(); + for (const escalation of escalations) { + const key = escalation.required_surface; + if (!groups.has(key)) groups.set(key, []); + groups.get(key).push(escalation); + } + + return [...groups.entries()] + .map(([requiredSurface, group]) => { + const packetIds = group.map((entry) => entry.packet_id); + const packetLabels = group.map((entry) => entry.packet_label); + const hostPlatforms = dedupeStrings(group.map((entry) => entry.current_host_platform)); + const surfaceStatuses = dedupeStrings(group.map((entry) => entry.current_surface_status)); + const mergedWhyNow = group.length > 1 + ? `Multiple local proof packets require the shared ${requiredSurface} surface before the next truthful local proof.` + : group[0].why_now; + const mergedReason = hostPlatforms.length === 1 && hostPlatforms[0] === 'Unix' + ? `Current host is not Windows, so the shared ${requiredSurface} surface cannot be satisfied here.` + : dedupeStrings(group.map((entry) => entry.reason)).join(' '); + return { + type: 'escalation', + escalation_id: requiredSurface, + status: 'required', + mode: 'escalate', + why_now: mergedWhyNow, + reason: mergedReason, + required_surface: requiredSurface, + current_surface_status: surfaceStatuses.length === 1 ? surfaceStatuses[0] : 'mixed', + current_host_platform: hostPlatforms.length === 1 ? hostPlatforms[0] : 'mixed', + packet_ids: packetIds, + packet_labels: packetLabels, + packet_count: group.length, + governing_requirements: dedupeStrings(group.map((entry) => entry.governing_requirement)), + blocked_requirements: dedupeStrings(group.map((entry) => entry.blocked_requirement)), + proof_check_ids: dedupeStrings(group.map((entry) => entry.proof_check_id)), + receipt_paths: dedupeStrings(group.map((entry) => entry.receipt_path)), + source_next_step_paths: dedupeStrings(group.map((entry) => entry.source_next_step_path)), + suggested_loop: dedupeStrings(group.flatMap((entry) => entry.suggested_loop ?? [])), + recommended_commands: dedupeStrings(group.flatMap((entry) => entry.recommended_commands ?? [])), + stop_conditions: dedupeStrings(group.flatMap((entry) => entry.stop_conditions ?? [])) + }; + }) + .sort((left, right) => { + if (right.packet_count !== left.packet_count) return right.packet_count - left.packet_count; + return left.required_surface.localeCompare(right.required_surface); + }); +} + +function buildPostLocalPromotionEscalation(packets) { + const packetIds = packets.map((packet) => packet.id); + const packetLabels = packets.map((packet) => packet.label); + return { + type: 'escalation', + escalation_id: 'post-local-promotion-proof', + status: 'required', + mode: 'promote', + why_now: 'All tracked local proof packets are implemented locally, so the next truthful move is proof on an integration or hosted surface.', + reason: 'Local packet requirements and local proof receipts are green. Further progress now needs integration-level proof of workflow routing, permissions, retained artifacts, and promotion behavior.', + required_surface: 'integration-or-hosted-proof', + current_surface_status: 'local-proof-complete', + current_host_platform: process.platform === 'win32' ? 'Windows' : 'Unix', + packet_count: packets.length, + packet_ids: packetIds, + packet_labels: packetLabels, + governing_requirements: ['REQ-LPAP-003'], + blocked_requirements: [], + proof_check_ids: ['program-post-local-promotion'], + receipt_paths: packets.map((packet) => packet.report_path), + source_next_step_paths: packets.map((packet) => packet.next_step_path), + suggested_loop: [ + 'Prepare a minimal upstream slice from the locally passing worktree.', + 'Push that slice to an integration or proof branch.', + 'Run the relevant hosted or integration proof surface for the affected packets.', + 'If hosted proof exposes a new seam, reopen the owning packet locally and continue from the emitted receipt.' + ], + recommended_commands: [ + 'git status --short', + 'git diff --stat', + 'gh pr status' + ], + stop_conditions: [ + 'Stop when an integration or hosted proof receipt exists for the promoted slice.', + 'Stop when hosted proof reopens a local packet requirement or escalation.' + ] + }; +} + +function selectProgramNextStep(requirements, escalations, fallbackEscalation = null) { + if (requirements.length > 0) return requirements[0]; + if (escalations.length > 0) return escalations[0]; + return fallbackEscalation; +} + +async function validateSchema(schemaPath, payload, label) { + const schema = await readJson(schemaPath); + const ajv = new Ajv2020({ allErrors: true, strict: false }); + addFormats(ajv); + const validate = ajv.compile(schema); + const ok = validate(payload); + if (!ok) { + const details = (validate.errors ?? []).map((entry) => `${entry.instancePath || '/'} ${entry.message}`).join('; '); + throw new Error(`${label} schema validation failed: ${details}`); + } +} + +function buildSummary(report) { + const lines = [ + '# CompareVI Local Program CI', + '', + `- Overall: ${report.overall.status}`, + `- Reason: ${report.overall.reason}`, + '' + ]; + + lines.push('## Packet Status', ''); + for (const packet of report.packets) { + lines.push(`- ${packet.label}: ${packet.overall_status}`); + lines.push(` next step: ${packet.next_step_type ?? 'none'}`); + } + lines.push(''); + + if (report.next_step?.type === 'requirement') { + lines.push('## Next Step', ''); + lines.push(`- Type: requirement`); + lines.push(`- Packet: ${report.next_step.packet_label}`); + lines.push(`- Requirement: ${report.next_step.req_id}`); + lines.push(`- Why now: ${report.next_step.why_now}`); + lines.push(`- Test: ${report.next_step.test_id}`); + lines.push(`- Source packet report: ${report.next_step.source_report_path}`); + lines.push(''); + } else if (report.next_step?.type === 'escalation') { + lines.push('## Next Step', ''); + lines.push(`- Type: escalation`); + lines.push(`- Required surface: ${report.next_step.required_surface}`); + lines.push(`- Packets: ${report.next_step.packet_labels.join(', ')}`); + lines.push(`- Blocked requirements: ${report.next_step.blocked_requirements.join(', ')}`); + lines.push(`- Reason: ${report.next_step.reason}`); + lines.push(''); + } + + return `${lines.join('\n')}\n`; +} + +export { buildRequirementCandidate, rankProgramRequirements, mergeSharedSurfaceEscalations, selectProgramNextStep }; +export { buildPostLocalPromotionEscalation }; + +export async function runCompareviLocalProgramCi({ + repoRoot = repoRootDefault, + resultsDir = DEFAULT_RESULTS_DIR, + outputPath = DEFAULT_REPORT, + summaryPath = DEFAULT_SUMMARY, + nextStepOutputPath = DEFAULT_NEXT_STEP +} = {}) { + const resolved = { + repoRoot, + resultsDir: path.join(repoRoot, resultsDir), + outputPath: path.join(repoRoot, outputPath), + summaryPath: path.join(repoRoot, summaryPath), + nextStepOutputPath: path.join(repoRoot, nextStepOutputPath) + }; + + await ensureDir(resolved.resultsDir); + + const pester = await runPesterServiceModelLocalCi({ repoRoot }); + const viHistory = await runVIHistoryLocalCi({ repoRoot }); + const windowsSurface = await runWindowsDockerSharedSurfaceLocalCi({ repoRoot }); + const packetRuns = [ + { packet: PACKETS[0], result: pester }, + { packet: PACKETS[1], result: viHistory }, + { packet: PACKETS[2], result: windowsSurface } + ]; + + const packets = packetRuns.map(({ packet, result }) => ({ + id: packet.id, + label: packet.label, + report_path: relativeFrom(repoRoot, result.paths.outputPath), + next_step_path: relativeFrom(repoRoot, result.paths.nextStepOutputPath), + overall_status: result.report.overall.status, + overall_reason: result.report.overall.reason, + next_step_type: result.report.next_step?.type ?? null, + next_requirement_id: result.report.next_requirement?.req_id ?? null, + required_surface: result.report.next_step?.type === 'escalation' ? result.report.next_step.required_surface : null + })); + + const requirementCandidates = packetRuns + .filter(({ result }) => result.report.next_step?.type === 'requirement') + .map(({ packet, result }) => buildRequirementCandidate( + packet, + relativeFrom(repoRoot, result.paths.outputPath), + relativeFrom(repoRoot, result.paths.nextStepOutputPath), + result.report.next_step + )); + + const escalationCandidates = packetRuns + .filter(({ result }) => result.report.next_step?.type === 'escalation') + .map(({ packet, result }) => ({ + packet_id: packet.id, + packet_label: packet.label, + source_next_step_path: relativeFrom(repoRoot, result.paths.nextStepOutputPath), + ...result.report.next_step + })); + + const rankedRequirements = rankProgramRequirements(requirementCandidates); + const escalations = mergeSharedSurfaceEscalations(escalationCandidates); + const postLocalPromotionEscalation = buildPostLocalPromotionEscalation(packets); + const nextStep = selectProgramNextStep(rankedRequirements, escalations, postLocalPromotionEscalation); + + let overallStatus = 'pass'; + let overallReason = 'The program selector did not emit a next step.'; + if (nextStep?.type === 'requirement') { + overallStatus = 'pass-with-actions'; + overallReason = `The next local requirement is ${nextStep.req_id} from ${nextStep.packet_label}.`; + } else if (nextStep?.type === 'escalation') { + overallStatus = 'pass-with-escalation'; + overallReason = `All tracked packet-local requirements are implemented, and the next truthful step is the shared '${nextStep.required_surface}' surface for ${nextStep.packet_labels.join(', ')}.`; + } + + const report = { + schema_version: '1.0.0', + generated_at: new Date().toISOString(), + repo_root: repoRoot, + packets, + ranked_requirements: rankedRequirements, + escalations, + next_step: nextStep, + overall: { + status: overallStatus, + reason: overallReason + } + }; + + await validateSchema( + path.join(repoRoot, 'docs', 'schemas', 'comparevi-local-program-ci-report-v1.schema.json'), + report, + 'CompareVI local program CI report' + ); + await validateSchema( + path.join(repoRoot, 'docs', 'schemas', 'comparevi-local-program-next-step-v1.schema.json'), + nextStep, + 'CompareVI local program next step' + ); + + await writeJson(resolved.outputPath, report); + await writeJson(resolved.nextStepOutputPath, nextStep); + await fs.writeFile(resolved.summaryPath, buildSummary(report), 'utf8'); + + return { report, paths: resolved }; +} + +async function main() { + try { + const options = parseArgs(); + if (options.help) { + printHelp(); + return; + } + const { report } = await runCompareviLocalProgramCi(options); + if (options.printNextStep && report.next_step) { + console.log(JSON.stringify(report.next_step, null, 2)); + return; + } + console.log(report.overall.reason); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} + +const entryPath = process.argv[1] ? path.resolve(process.argv[1]) : null; +if (entryPath && fileURLToPath(import.meta.url) === entryPath) { + await main(); +} diff --git a/tools/priority/materialize-pester-service-model-release-evidence.mjs b/tools/priority/materialize-pester-service-model-release-evidence.mjs index 22c034a9c..c83f3a8ef 100644 --- a/tools/priority/materialize-pester-service-model-release-evidence.mjs +++ b/tools/priority/materialize-pester-service-model-release-evidence.mjs @@ -4,15 +4,20 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import process from 'node:process'; import { spawnSync } from 'node:child_process'; - -const repoRoot = process.cwd(); -const defaultBase = path.join(repoRoot, 'tests', 'results', '_agent', 'pester-service-model'); -const defaultOutputDir = path.join(defaultBase, 'release-evidence'); +import { + describeFile, + relativeOrAbsolute, + resolveRunContext, + toPortablePath, + writeJsonFile +} from './pester-service-model-provenance.mjs'; function parseArgs(argv = process.argv.slice(2)) { const options = { + repoRoot: process.cwd(), + baseDir: null, version: 'v0.1.0', - outputDir: defaultOutputDir, + outputDir: null, upstreamIssue: '2069', forkIssue: '2078', forkBasisCommit: '', @@ -27,6 +32,16 @@ function parseArgs(argv = process.argv.slice(2)) { i += 1; continue; } + if (token === '--repo-root') { + options.repoRoot = path.resolve(next); + i += 1; + continue; + } + if (token === '--base-dir') { + options.baseDir = path.resolve(next); + i += 1; + continue; + } if (token === '--output-dir') { options.outputDir = path.resolve(next); i += 1; @@ -59,6 +74,7 @@ function parseArgs(argv = process.argv.slice(2)) { } function runGit(args) { + const repoRoot = optionsCache.repoRoot; const result = spawnSync('git', args, { cwd: repoRoot, encoding: 'utf8' }); if (result.status !== 0) { throw new Error(result.stderr?.trim() || `git ${args.join(' ')} failed`); @@ -66,6 +82,8 @@ function runGit(args) { return result.stdout.trim(); } +const optionsCache = { repoRoot: process.cwd() }; + async function ensureFile(filePath) { await fs.access(filePath); } @@ -78,31 +96,38 @@ async function copyIntoBundle(source, bundleRoot, name = path.basename(source)) async function main() { const options = parseArgs(); - await fs.mkdir(options.outputDir, { recursive: true }); - - const coverageXml = path.join(defaultBase, 'coverage.xml'); - const docsLinkCheck = path.join(defaultBase, 'docs-link-check.json'); - const requirementsPath = path.join(repoRoot, 'docs', 'requirements-pester-service-model-srs.md'); - const rtmPath = path.join(repoRoot, 'docs', 'rtm-pester-service-model.csv'); - const qualityReportPath = path.join(repoRoot, 'docs', 'pester-service-model-quality-report.md'); + optionsCache.repoRoot = options.repoRoot; + const defaultBase = path.join(options.repoRoot, 'tests', 'results', '_agent', 'pester-service-model'); + const baseDir = options.baseDir ?? defaultBase; + const outputDir = options.outputDir ?? path.join(baseDir, 'release-evidence'); + await fs.mkdir(outputDir, { recursive: true }); + + const coverageXml = path.join(baseDir, 'coverage.xml'); + const docsLinkCheck = path.join(baseDir, 'docs-link-check.json'); + const requirementsPath = path.join(options.repoRoot, 'docs', 'requirements-pester-service-model-srs.md'); + const comparisonPath = path.join(options.repoRoot, 'docs', 'pester-service-model-promotion-comparison.json'); + const rtmPath = path.join(options.repoRoot, 'docs', 'rtm-pester-service-model.csv'); + const qualityReportPath = path.join(options.repoRoot, 'docs', 'pester-service-model-quality-report.md'); await Promise.all([ ensureFile(coverageXml), ensureFile(docsLinkCheck), ensureFile(requirementsPath), + ensureFile(comparisonPath), ensureFile(rtmPath), ensureFile(qualityReportPath) ]); await Promise.all([ - copyIntoBundle(coverageXml, options.outputDir), - copyIntoBundle(docsLinkCheck, options.outputDir), - copyIntoBundle(requirementsPath, options.outputDir), - copyIntoBundle(rtmPath, options.outputDir), - copyIntoBundle(qualityReportPath, options.outputDir) + copyIntoBundle(coverageXml, outputDir), + copyIntoBundle(docsLinkCheck, outputDir), + copyIntoBundle(requirementsPath, outputDir), + copyIntoBundle(comparisonPath, outputDir), + copyIntoBundle(rtmPath, outputDir), + copyIntoBundle(qualityReportPath, outputDir) ]); - const recordPath = path.join(options.outputDir, `release-record-${options.version}.md`); + const recordPath = path.join(outputDir, `release-record-${options.version}.md`); const headSha = runGit(['rev-parse', 'HEAD']); const branch = runGit(['rev-parse', '--abbrev-ref', 'HEAD']); @@ -124,13 +149,58 @@ async function main() { '- `coverage.xml`', '- `docs-link-check.json`', '- `requirements-pester-service-model-srs.md`', + '- `pester-service-model-promotion-comparison.json`', '- `rtm-pester-service-model.csv`', '- `pester-service-model-quality-report.md`', + '- `release-evidence-provenance.json`', '' ]; await fs.writeFile(recordPath, `${lines.join('\n')}\n`, 'utf8'); - console.log(`release_evidence_dir=${options.outputDir}`); + const provenancePath = path.join(outputDir, 'release-evidence-provenance.json'); + const sourceInputs = await Promise.all([ + describeFile(options.repoRoot, coverageXml, { kind: 'hosted-quality-input', role: 'coverage-xml', artifactName: 'coverage.xml' }), + describeFile(options.repoRoot, docsLinkCheck, { kind: 'hosted-quality-input', role: 'docs-link-check', artifactName: 'docs-link-check.json' }), + describeFile(options.repoRoot, requirementsPath, { kind: 'packet-document', role: 'requirements-srs' }), + describeFile(options.repoRoot, comparisonPath, { kind: 'packet-document', role: 'promotion-comparison' }), + describeFile(options.repoRoot, rtmPath, { kind: 'packet-document', role: 'rtm' }), + describeFile(options.repoRoot, qualityReportPath, { kind: 'packet-document', role: 'quality-report' }) + ]); + const derivedOutputs = await Promise.all([ + describeFile(options.repoRoot, path.join(outputDir, 'coverage.xml'), { kind: 'release-evidence-output', role: 'coverage-xml' }), + describeFile(options.repoRoot, path.join(outputDir, 'docs-link-check.json'), { kind: 'release-evidence-output', role: 'docs-link-check' }), + describeFile(options.repoRoot, path.join(outputDir, 'requirements-pester-service-model-srs.md'), { kind: 'release-evidence-output', role: 'requirements-srs' }), + describeFile(options.repoRoot, path.join(outputDir, 'pester-service-model-promotion-comparison.json'), { kind: 'release-evidence-output', role: 'promotion-comparison' }), + describeFile(options.repoRoot, path.join(outputDir, 'rtm-pester-service-model.csv'), { kind: 'release-evidence-output', role: 'rtm' }), + describeFile(options.repoRoot, path.join(outputDir, 'pester-service-model-quality-report.md'), { kind: 'release-evidence-output', role: 'quality-report' }), + describeFile(options.repoRoot, recordPath, { kind: 'release-evidence-output', role: 'release-record' }) + ]); + await writeJsonFile(provenancePath, { + schema: 'pester-derived-provenance@v1', + schemaVersion: '1.0.0', + generatedAtUtc: new Date().toISOString(), + provenanceKind: 'release-evidence', + producer: { + id: 'materialize-pester-service-model-release-evidence.mjs', + version: '1.0.0' + }, + subject: { + id: 'pester-service-model-release-evidence', + baselineVersion: options.version, + upstreamIssue: options.upstreamIssue, + forkIssue: options.forkIssue, + forkBasisCommit: options.forkBasisCommit || null, + forkBasisUrl: options.forkBasisUrl || null, + bundleDir: toPortablePath(outputDir), + bundleDirRepoPath: relativeOrAbsolute(options.repoRoot, outputDir) + }, + runContext: resolveRunContext(options.repoRoot, 'Pester service-model release evidence'), + sourceInputs, + derivedOutputs + }); + + console.log(`release_evidence_dir=${outputDir}`); + console.log(`release_evidence_provenance=${provenancePath}`); } main().catch((error) => { diff --git a/tools/priority/pester-service-model-audit-surface.yaml b/tools/priority/pester-service-model-audit-surface.yaml new file mode 100644 index 000000000..fd58b20a6 --- /dev/null +++ b/tools/priority/pester-service-model-audit-surface.yaml @@ -0,0 +1,67 @@ +version: 1 +id: pester-service-model-control-plane +description: > + Dedicated local CI audit surface for the Pester service-model subsystem. This + bundle captures the layered workflows, execution helpers, assurance packet, + and the local autonomy loop that ranks the next requirement to tackle. +include: + - package.json + - tools/priority/pester-service-model-autonomy-policy.json + - .github/workflows/pester-context.yml + - .github/workflows/pester-gate.yml + - .github/workflows/pester-run.yml + - .github/workflows/pester-evidence.yml + - .github/workflows/pester-service-model-on-label.yml + - .github/workflows/pester-selection.yml + - .github/workflows/pester-service-model-quality.yml + - .github/workflows/pester-service-model-release-evidence.yml + - .github/workflows/selfhosted-readiness.yml + - docs/knowledgebase/Pester-Service-Model.md + - docs/knowledgebase/Local-Proof-Autonomy-Program.md + - docs/architecture/pester-service-model-control-plane.md + - docs/pester-service-model-promotion-comparison.json + - docs/requirements-pester-service-model-srs.md + - docs/rtm-pester-service-model.csv + - docs/schemas/pester-derived-provenance-v1.schema.json + - docs/schemas/pester-service-model-local-ci-report-v1.schema.json + - docs/schemas/pester-service-model-next-step-v1.schema.json + - docs/schemas/comparevi-local-program-ci-report-v1.schema.json + - docs/schemas/comparevi-local-program-next-step-v1.schema.json + - docs/schemas/pester-promotion-comparison-v1.schema.json + - docs/testing/pester-service-model-test-plan.md + - docs/pester-service-model-quality-report.md + - tools/priority/pester-service-model-local-ci.mjs + - tools/priority/windows-host-bridge.mjs + - tools/priority/comparevi-local-program-ci.mjs + - tools/priority/pester-service-model-provenance.mjs + - tools/priority/__tests__/comparevi-local-program-ci.test.mjs + - tools/priority/__tests__/pester-service-model-local-ci.test.mjs + - tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs + - tools/priority/__tests__/pester-service-model-release-evidence-provenance.test.mjs + - tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs + - tools/priority/__tests__/windows-host-bridge.test.mjs + - tools/Write-PesterTotals.ps1 + - tools/Invoke-PesterEvidenceClassification.ps1 + - tools/Invoke-PesterEvidenceProvenance.ps1 + - tools/Invoke-PesterWindowsContainerSurfaceProbe.ps1 + - tools/Replay-PesterServiceModelArtifacts.Local.ps1 + - Invoke-PesterTests.ps1 + - tools/PesterPathHygiene.ps1 + - tools/Invoke-PesterExecutionFinalize.ps1 + - tools/Invoke-PesterExecutionPostprocess.ps1 + - tools/Run-PesterExecutionOnly.Local.ps1 + - tests/PesterPathHygiene.Tests.ps1 + - tests/Run-PesterExecutionOnly.Local.PathHygiene.Tests.ps1 + - tests/Invoke-PesterEvidenceClassification.Tests.ps1 + - tests/Invoke-PesterEvidenceProvenance.Tests.ps1 + - tests/Invoke-PesterWindowsContainerSurfaceProbe.Tests.ps1 + - tests/Replay-PesterServiceModelRepresentativeArtifact.Tests.ps1 + - tests/Replay-PesterServiceModelArtifacts.Local.Tests.ps1 + - tests/fixtures/pester-service-model/legacy-results-xml-truncated/pester-run-receipt.json + - tests/fixtures/pester-service-model/legacy-results-xml-truncated/raw/pester-results.xml + - tests/fixtures/pester-service-model/legacy-results-xml-truncated/raw/pester-summary.json + - tests/fixtures/pester-service-model/legacy-results-xml-truncated/raw/dispatcher-events.ndjson + - tests/fixtures/pester-service-model/legacy-results-xml-truncated/raw/pester-failures.json + - scripts/Write-PesterSummaryToStepSummary.ps1 + - tools/Write-PesterTopFailures.ps1 + - tools/Print-PesterTopFailures.ps1 diff --git a/tools/priority/pester-service-model-autonomy-policy.json b/tools/priority/pester-service-model-autonomy-policy.json new file mode 100644 index 000000000..bd3af5e6f --- /dev/null +++ b/tools/priority/pester-service-model-autonomy-policy.json @@ -0,0 +1,80 @@ +{ + "schema_version": "1.0.0", + "phase_guidance": { + "foundation": { + "mode": "local-first", + "preferred_commands": [ + "node --test tools/priority/__tests__/pester-service-model-local-ci.test.mjs tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs", + "npm run tests:execution:local", + "npm run tests:replay:representative", + "npm run priority:pester:local-ci" + ], + "stop_conditions": [ + "planned local coverage exists for the selected requirement", + "local execution harness or packet contract proofs pass for the edited code refs", + "the ranked backlog no longer selects the same requirement at the top without change" + ], + "escalate_when": [ + "the remaining debt is hosted-only proof", + "the local harness can no longer reproduce the active seam" + ] + }, + "execution-governance": { + "mode": "local-first", + "preferred_commands": [ + "node --test tools/priority/__tests__/pester-service-model-local-ci.test.mjs tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs", + "npm run tests:execution:local", + "npm run tests:replay:representative", + "npm run priority:pester:local-ci" + ], + "stop_conditions": [ + "the edited execution boundary is proven locally", + "the next-requirement artifact no longer ranks the edited requirement first" + ], + "escalate_when": [ + "the next proof depends on a retained integration or hosted run" + ] + }, + "promotion-governance": { + "mode": "packet-first", + "preferred_commands": [ + "node --test tools/priority/__tests__/pester-service-model-local-ci.test.mjs tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs", + "npm run priority:pester:local-ci" + ], + "stop_conditions": [ + "the packet retains representative comparison evidence and provenance" + ], + "escalate_when": [ + "the packet is green and the next step is a mounted or hosted comparison proof" + ] + }, + "evidence-governance": { + "mode": "local-first", + "preferred_commands": [ + "node --test tools/priority/__tests__/pester-service-model-local-ci.test.mjs tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs", + "npm run priority:pester:local-ci" + ], + "stop_conditions": [ + "the packet stays coherent across SRS, RTM, test plan, and knowledgebase" + ], + "escalate_when": [ + "the remaining debt is representative hosted proof or promotion evidence" + ] + }, + "autonomy": { + "mode": "local-first", + "preferred_commands": [ + "node --test tools/priority/__tests__/pester-service-model-local-ci.test.mjs tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs", + "npm run tests:windows-surface:probe", + "npm run priority:pester:local-ci" + ], + "stop_conditions": [ + "the autonomy artifacts still select a bounded next requirement", + "the worktree and packet stay aligned" + ], + "escalate_when": [ + "the next action depends on hosted comparison evidence" + ] + } + } +} diff --git a/tools/priority/pester-service-model-local-ci.mjs b/tools/priority/pester-service-model-local-ci.mjs new file mode 100644 index 000000000..09dff7157 --- /dev/null +++ b/tools/priority/pester-service-model-local-ci.mjs @@ -0,0 +1,807 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; +import Ajv2020 from 'ajv/dist/2020.js'; +import addFormats from 'ajv-formats'; +import yaml from 'js-yaml'; + +import { runWindowsSurfaceProof as runSharedWindowsSurfaceProof } from './windows-docker-shared-surface-local-ci.mjs'; + +const repoRootDefault = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); +const defaultSkillRoot = path.join(process.env.HOME ?? '', '.codex', 'skills', 'repo-standards-review'); +const fallbackSkillRoot = '/mnt/c/Users/sveld/.codex/skills/repo-standards-review'; + +const DEFAULT_SURFACE = path.join('tools', 'priority', 'pester-service-model-audit-surface.yaml'); +const DEFAULT_POLICY = path.join('tools', 'priority', 'pester-service-model-autonomy-policy.json'); +const DEFAULT_RESULTS_DIR = path.join('tests', 'results', '_agent', 'pester-service-model', 'local-ci'); +const DEFAULT_REPORT = path.join(DEFAULT_RESULTS_DIR, 'pester-service-model-local-ci-report.json'); +const DEFAULT_SUMMARY = path.join(DEFAULT_RESULTS_DIR, 'pester-service-model-local-ci-summary.md'); +const DEFAULT_NEXT = path.join(DEFAULT_RESULTS_DIR, 'pester-service-model-next-requirement.json'); +const DEFAULT_NEXT_STEP = path.join(DEFAULT_RESULTS_DIR, 'pester-service-model-next-step.json'); + +const HELP = [ + 'Usage: node tools/priority/pester-service-model-local-ci.mjs [options]', + '', + 'Options:', + ` --repo-root (default: ${repoRootDefault})`, + ` --skill-root (default: ${defaultSkillRoot} or ${fallbackSkillRoot})`, + ` --surface (default: ${DEFAULT_SURFACE})`, + ` --policy (default: ${DEFAULT_POLICY})`, + ` --results-dir (default: ${DEFAULT_RESULTS_DIR})`, + ` --output (default: ${DEFAULT_REPORT})`, + ` --summary-output (default: ${DEFAULT_SUMMARY})`, + ` --next-output (default: ${DEFAULT_NEXT})`, + ` --next-step-output (default: ${DEFAULT_NEXT_STEP})`, + ' --print-next print the selected next requirement to stdout', + ' --print-next-step print the selected next step to stdout', + ' --help, -h' +]; + +function parseArgs(argv = process.argv) { + const options = { + repoRoot: repoRootDefault, + skillRoot: null, + surfacePath: DEFAULT_SURFACE, + policyPath: DEFAULT_POLICY, + resultsDir: DEFAULT_RESULTS_DIR, + outputPath: DEFAULT_REPORT, + summaryPath: DEFAULT_SUMMARY, + nextOutputPath: DEFAULT_NEXT, + nextStepOutputPath: DEFAULT_NEXT_STEP, + printNext: false, + printNextStep: false, + help: false + }; + + const args = argv.slice(2); + for (let i = 0; i < args.length; i += 1) { + const token = args[i]; + const next = args[i + 1]; + if (token === '--help' || token === '-h') { + options.help = true; + continue; + } + if (token === '--print-next') { + options.printNext = true; + continue; + } + if (token === '--print-next-step') { + options.printNextStep = true; + continue; + } + if (['--repo-root', '--skill-root', '--surface', '--policy', '--results-dir', '--output', '--summary-output', '--next-output', '--next-step-output'].includes(token)) { + if (!next || next.startsWith('-')) throw new Error(`Missing value for ${token}`); + i += 1; + if (token === '--repo-root') options.repoRoot = path.resolve(next); + if (token === '--skill-root') options.skillRoot = path.resolve(next); + if (token === '--surface') options.surfacePath = next; + if (token === '--policy') options.policyPath = next; + if (token === '--results-dir') options.resultsDir = next; + if (token === '--output') options.outputPath = next; + if (token === '--summary-output') options.summaryPath = next; + if (token === '--next-output') options.nextOutputPath = next; + if (token === '--next-step-output') options.nextStepOutputPath = next; + continue; + } + throw new Error(`Unknown option: ${token}`); + } + + return options; +} + +function printHelp() { + for (const line of HELP) console.log(line); +} + +async function fileExists(filePath) { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } +} + +async function resolveSkillRoot(explicitRoot) { + if (explicitRoot) return explicitRoot; + const candidates = []; + if (process.env.CODEX_HOME) candidates.push(path.join(process.env.CODEX_HOME, 'skills', 'repo-standards-review')); + candidates.push(defaultSkillRoot, fallbackSkillRoot); + for (const candidate of candidates) { + if (await fileExists(candidate)) return candidate; + } + return defaultSkillRoot; +} + +async function readYaml(filePath) { + return yaml.load(await fs.readFile(filePath, 'utf8')); +} + +async function ensureDir(filePath) { + await fs.mkdir(filePath, { recursive: true }); +} + +async function readJson(filePath) { + return JSON.parse(await fs.readFile(filePath, 'utf8')); +} + +async function validateReportSchema(repoRoot, report) { + const schemaPath = path.join(repoRoot, 'docs', 'schemas', 'pester-service-model-local-ci-report-v1.schema.json'); + const schema = await readJson(schemaPath); + const ajv = new Ajv2020({ allErrors: true, strict: false }); + addFormats(ajv); + const validate = ajv.compile(schema); + const ok = validate(report); + if (!ok) { + const details = (validate.errors ?? []).map((entry) => `${entry.instancePath || '/'} ${entry.message}`).join('; '); + throw new Error(`Local CI report schema validation failed: ${details}`); + } +} + +async function validateNextStepSchema(repoRoot, nextStep) { + const schemaPath = path.join(repoRoot, 'docs', 'schemas', 'pester-service-model-next-step-v1.schema.json'); + const schema = await readJson(schemaPath); + const ajv = new Ajv2020({ allErrors: true, strict: false }); + addFormats(ajv); + const validate = ajv.compile(schema); + const ok = validate(nextStep); + if (!ok) { + const details = (validate.errors ?? []).map((entry) => `${entry.instancePath || '/'} ${entry.message}`).join('; '); + throw new Error(`Next-step schema validation failed: ${details}`); + } +} + +function relativeFrom(repoRoot, filePath) { + return path.relative(repoRoot, filePath).split(path.sep).join('/'); +} + +async function materializeAuditSurface(repoRoot, manifest, bundleRoot) { + await ensureDir(bundleRoot); + for (const relativePath of manifest.include) { + const source = path.join(repoRoot, relativePath); + const destination = path.join(bundleRoot, relativePath); + await ensureDir(path.dirname(destination)); + await fs.copyFile(source, destination); + } +} + +function createRunScopedBundleRoot(resultsDir) { + const runId = `run-${Date.now()}-${process.pid}-${Math.random().toString(16).slice(2, 8)}`; + return path.join(resultsDir, 'surface-bundle', runId); +} + +function runCommand(command, args, options = {}) { + const result = spawnSync(command, args, { + cwd: options.cwd, + encoding: 'utf8', + maxBuffer: 20 * 1024 * 1024 + }); + if (result.status !== 0) { + throw new Error([`${command} ${args.join(' ')}`, result.stdout?.trim(), result.stderr?.trim()].filter(Boolean).join('\n')); + } + return result; +} + +function parseCsv(input) { + const rows = []; + let current = ''; + let row = []; + let inQuotes = false; + + for (let i = 0; i < input.length; i += 1) { + const char = input[i]; + const next = input[i + 1]; + if (char === '"') { + if (inQuotes && next === '"') { + current += '"'; + i += 1; + } else { + inQuotes = !inQuotes; + } + continue; + } + if (char === ',' && !inQuotes) { + row.push(current); + current = ''; + continue; + } + if ((char === '\n' || char === '\r') && !inQuotes) { + if (char === '\r' && next === '\n') i += 1; + row.push(current); + current = ''; + if (row.some((value) => value.length > 0)) rows.push(row); + row = []; + continue; + } + current += char; + } + + if (current.length > 0 || row.length > 0) { + row.push(current); + if (row.some((value) => value.length > 0)) rows.push(row); + } + + if (rows.length === 0) return []; + const [header, ...records] = rows; + return records.map((values) => { + const entry = {}; + for (let i = 0; i < header.length; i += 1) { + entry[header[i]] = values[i] ?? ''; + } + return entry; + }); +} + +function parseRequirementNumber(reqId) { + const match = /REQ-PSM-(\d+)/.exec(reqId ?? ''); + return match ? Number.parseInt(match[1], 10) : Number.POSITIVE_INFINITY; +} + +function determinePhase(reqNumber) { + if (reqNumber <= 11) return 'foundation'; + if (reqNumber <= 17) return 'execution-governance'; + if (reqNumber <= 19) return 'promotion-governance'; + if (reqNumber <= 21) return 'evidence-governance'; + return 'autonomy'; +} + +function splitField(value) { + return String(value ?? '') + .split(';') + .map((entry) => entry.trim()) + .filter(Boolean); +} + +function isOperationalRef(ref) { + return !String(ref).startsWith('docs/'); +} + +function deriveWeakAreas(scorePayload) { + return Object.entries(scorePayload.areas ?? {}) + .filter(([, details]) => Number(details.score ?? 0) < 3) + .map(([area]) => area); +} + +function collectWorktreeStatus(repoRoot) { + const result = runCommand('git', ['status', '--short', '--branch'], { cwd: repoRoot }); + const lines = result.stdout.split(/\r?\n/).filter(Boolean); + const branch = (lines.find((line) => line.startsWith('## ')) ?? '## detached').replace(/^##\s+/, ''); + const modifiedPaths = lines + .filter((line) => !line.startsWith('## ')) + .map((line) => line.slice(3).trim()) + .filter(Boolean); + return { branch, modifiedPaths }; +} + +function scoreGap(row) { + const reqNumber = parseRequirementNumber(row.ReqID); + const phase = determinePhase(reqNumber); + const priorityWeight = row.Priority === 'High' ? 200 : row.Priority === 'Medium' ? 100 : 50; + const phaseWeight = { + foundation: 300, + 'execution-governance': 250, + 'promotion-governance': 160, + 'evidence-governance': 150, + autonomy: 180 + }[phase] ?? 0; + const runtimeRefWeight = /(Invoke-PesterTests\.ps1|pester-run\.yml|pester-evidence\.yml|Run-PesterExecutionOnly\.Local\.ps1|Invoke-PesterExecutionFinalize\.ps1|Invoke-PesterExecutionPostprocess\.ps1)/.test(row.CodeRef) ? 40 : 0; + const localFirstWeight = /(local|execution|evidence|failure-detail|path-hygiene|telemetry)/i.test(row.Requirement) ? 25 : 0; + const promotionPenalty = /(promotion|baseline)/i.test(row.Requirement) ? -40 : 0; + return 1000 + priorityWeight + phaseWeight + runtimeRefWeight + localFirstWeight + promotionPenalty - reqNumber; +} + +function deriveSuggestedLoop(row) { + const codeRefs = splitField(row.CodeRef); + const actions = []; + actions.push(`Start with ${row.TestID}: add or tighten local coverage before widening CI.`); + if (codeRefs.length > 0) { + actions.push(`Edit the primary source targets first: ${codeRefs.slice(0, 3).join(', ')}.`); + } + if (/(Invoke-PesterTests\.ps1|Run-PesterExecutionOnly\.Local\.ps1|pester-run\.yml|pester-evidence\.yml)/.test(row.CodeRef)) { + actions.push('Prove the change locally with the execution harness and packet contract tests before another GitHub run.'); + } else { + actions.push('Re-run the local packet CI and contract tests after the change to refresh the ranked backlog.'); + } + return actions; +} + +function buildRequirementEntry(row, overrides = {}) { + const reqNumber = parseRequirementNumber(row.ReqID); + const phase = determinePhase(reqNumber); + return { + req_id: row.ReqID, + priority: row.Priority, + status: overrides.status ?? row.Status, + phase, + score: scoreGap(row) + (overrides.scoreBoost ?? 0), + why_now: overrides.why_now ?? `${row.ReqID} is an unresolved ${row.Priority} ${phase} gap with concrete code refs and planned verification.`, + requirement: row.Requirement, + test_id: row.TestID, + test_artifact: row.TestArtifact, + code_refs: splitField(row.CodeRef), + suggested_loop: overrides.suggested_loop ?? deriveSuggestedLoop(row), + proof_check_id: overrides.proof_check_id ?? null + }; +} + +function rankRequirementGaps(rows) { + return rows + .filter((row) => row.Status !== 'Implemented') + .map((row) => buildRequirementEntry(row)) + .sort((a, b) => { + if (b.score !== a.score) return b.score - a.score; + return parseRequirementNumber(a.req_id) - parseRequirementNumber(b.req_id); + }) + .map((entry, index) => ({ + rank: index + 1, + ...entry + })); +} + +function rankProofRegressions(proofChecks, rows) { + const byReqId = new Map(rows.map((row) => [row.ReqID, row])); + return proofChecks + .filter((check) => check.status === 'fail' && check.owner_requirement && byReqId.has(check.owner_requirement)) + .map((check) => { + const row = byReqId.get(check.owner_requirement); + return buildRequirementEntry(row, { + status: 'Regression', + scoreBoost: 5000, + why_now: `${row.ReqID} regressed under the local proof check '${check.id}': ${check.summary}`, + suggested_loop: [ + `Start with proof check ${check.id}: ${check.summary}`, + ...deriveSuggestedLoop(row) + ], + proof_check_id: check.id + }); + }) + .sort((a, b) => { + if (b.score !== a.score) return b.score - a.score; + return parseRequirementNumber(a.req_id) - parseRequirementNumber(b.req_id); + }); +} + +function deriveEscalations(proofChecks) { + return proofChecks + .filter((check) => check.status === 'advisory' && check.id === 'windows-container-surface') + .map((check) => ({ + type: 'escalation', + escalation_id: 'windows-container-live-proof', + governing_requirement: 'REQ-PSM-027', + blocked_requirement: check.owner_requirement, + proof_check_id: check.id, + status: 'required', + mode: 'escalate', + why_now: `The next truthful local proof surface for ${check.owner_requirement} is unavailable from the current host.`, + reason: check.surface_status === 'not-windows-host' + ? 'Current host is not Windows, so the Docker Desktop + NI Windows image proof surface cannot be exercised here.' + : `Windows-container surrogate reported ${check.surface_status}; the environment must be prepared before another hosted rerun.`, + required_surface: 'windows-docker-desktop-ni-image', + current_surface_status: check.surface_status ?? 'unknown', + current_host_platform: check.host_platform ?? 'unknown', + receipt_path: check.receipt_path ?? null, + suggested_loop: [ + 'Move to a Windows host with Docker Desktop configured for Windows containers.', + 'Bootstrap or verify the pinned NI Windows image before attempting another live proof.', + 'Re-run the bounded Windows-surface probe before choosing a hosted GitHub rerun.' + ], + recommended_commands: check.recommended_commands ?? [], + stop_conditions: [ + 'Stop once the Windows-container surface probe reports status=ready.', + 'Stop if the probe exposes a new blocking defect such as docker-engine-not-windows or ni-image-missing.' + ] + })); +} + +function selectNextStep(nextRequirement, escalations) { + if (nextRequirement) { + return { + type: 'requirement', + ...nextRequirement + }; + } + return escalations[0] ?? null; +} + +function applyAutonomyPolicy(rankedRequirements, policy, modifiedPaths) { + const modifiedSet = new Set(modifiedPaths); + return rankedRequirements + .map((item) => { + const phasePolicy = policy.phase_guidance?.[item.phase] ?? {}; + const activeNow = item.code_refs.some((ref) => isOperationalRef(ref) && modifiedSet.has(ref)); + return { + ...item, + active_now: activeNow, + why_now: activeNow + ? `${item.req_id} is already active in the worktree and remains an unresolved ${item.priority} ${item.phase} gap.` + : item.why_now, + mode: phasePolicy.mode ?? 'local-first', + preferred_commands: phasePolicy.preferred_commands ?? [], + stop_conditions: phasePolicy.stop_conditions ?? [], + escalate_when: phasePolicy.escalate_when ?? [] + }; + }) + .sort((a, b) => { + if (a.active_now !== b.active_now) return a.active_now ? -1 : 1; + if (b.score !== a.score) return b.score - a.score; + return parseRequirementNumber(a.req_id) - parseRequirementNumber(b.req_id); + }) + .map((entry, index) => ({ + ...entry, + rank: index + 1 + })); +} + +function buildSummary(report) { + const lines = [ + '# Pester Service Model Local CI', + '', + `- Overall: ${report.overall.status}`, + `- Reason: ${report.overall.reason}`, + `- Standards audit: ${report.standards_audit.status}`, + `- Requirements: ${report.requirement_summary.implemented}/${report.requirement_summary.total} implemented`, + `- Gaps: ${report.requirement_summary.gaps}`, + `- Proof checks: ${report.proof_checks.blocking_failures} blocking, ${report.proof_checks.advisories} advisory`, + '' + ]; + + if (report.next_step?.type === 'requirement' && report.next_requirement) { + lines.push('## Next Step', ''); + lines.push(`- Type: requirement`); + lines.push(`- ${report.next_requirement.req_id}`); + lines.push(`- Phase: ${report.next_requirement.phase}`); + lines.push(`- Why now: ${report.next_requirement.why_now}`); + lines.push(`- Test: ${report.next_requirement.test_id}`); + lines.push(`- Code refs: ${report.next_requirement.code_refs.join(', ') || '(none)'}`); + lines.push(`- Mode: ${report.next_requirement.mode}`); + lines.push(`- Active in worktree: ${report.next_requirement.active_now ? 'yes' : 'no'}`); + lines.push(''); + lines.push('## Suggested Local Loop', ''); + for (const action of report.next_requirement.suggested_loop) { + lines.push(`- ${action}`); + } + if (report.next_requirement.preferred_commands.length > 0) { + lines.push('', '## Preferred Commands', ''); + for (const command of report.next_requirement.preferred_commands) { + lines.push(`- \`${command}\``); + } + } + if (report.next_requirement.stop_conditions.length > 0) { + lines.push('', '## Stop Conditions', ''); + for (const item of report.next_requirement.stop_conditions) { + lines.push(`- ${item}`); + } + } + if (report.next_requirement.escalate_when.length > 0) { + lines.push('', '## Escalate When', ''); + for (const item of report.next_requirement.escalate_when) { + lines.push(`- ${item}`); + } + } + lines.push(''); + } + + if (report.next_step?.type === 'escalation') { + lines.push('## Next Step', ''); + lines.push(`- Type: escalation`); + lines.push(`- Escalation: ${report.next_step.escalation_id}`); + lines.push(`- Governing requirement: ${report.next_step.governing_requirement}`); + lines.push(`- Blocked requirement: ${report.next_step.blocked_requirement}`); + lines.push(`- Why now: ${report.next_step.why_now}`); + lines.push(`- Reason: ${report.next_step.reason}`); + lines.push(`- Required surface: ${report.next_step.required_surface}`); + lines.push(`- Current surface status: ${report.next_step.current_surface_status}`); + lines.push(`- Current host platform: ${report.next_step.current_host_platform}`); + if (report.next_step.receipt_path) { + lines.push(`- Receipt: ${report.next_step.receipt_path}`); + } + lines.push(''); + lines.push('## Suggested Escalation Loop', ''); + for (const action of report.next_step.suggested_loop) { + lines.push(`- ${action}`); + } + if (report.next_step.recommended_commands.length > 0) { + lines.push('', '## Recommended Commands', ''); + for (const command of report.next_step.recommended_commands) { + lines.push(`- \`${command}\``); + } + } + if (report.next_step.stop_conditions.length > 0) { + lines.push('', '## Stop Conditions', ''); + for (const item of report.next_step.stop_conditions) { + lines.push(`- ${item}`); + } + } + lines.push(''); + } + + lines.push('## Ranked Requirement Backlog', ''); + for (const item of report.ranked_requirements.slice(0, 8)) { + lines.push(`- ${item.rank}. ${item.req_id} [${item.priority}] phase=${item.phase} score=${item.score}`); + lines.push(` ${item.requirement}`); + } + + if (report.proof_checks.checks.length > 0) { + lines.push('', '## Proof Checks', ''); + for (const check of report.proof_checks.checks) { + lines.push(`- ${check.id}: ${check.status}`); + lines.push(` ${check.summary}`); + } + } + + return `${lines.join('\n')}\n`; +} + +async function writeJson(filePath, payload) { + await ensureDir(path.dirname(filePath)); + await fs.writeFile(filePath, JSON.stringify(payload, null, 2), 'utf8'); +} + +async function runRepresentativeReplayProof(repoRoot, resultsDir) { + const workspace = path.join(resultsDir, 'representative-replay'); + await fs.rm(workspace, { recursive: true, force: true }); + await ensureDir(workspace); + + const command = `& 'tools/Replay-PesterServiceModelArtifacts.Local.ps1' -RawArtifactDir 'tests/fixtures/pester-service-model/legacy-results-xml-truncated/raw' -ExecutionReceiptPath 'tests/fixtures/pester-service-model/legacy-results-xml-truncated/pester-run-receipt.json' -WorkspaceResultsDir '${workspace.replace(/'/g, "''")}'`; + const result = spawnSync('pwsh', ['-NoLogo', '-NoProfile', '-Command', command], { + cwd: repoRoot, + encoding: 'utf8', + maxBuffer: 20 * 1024 * 1024 + }); + + const base = { + id: 'representative-replay', + owner_requirement: 'REQ-PSM-024', + workspace: relativeFrom(repoRoot, workspace) + }; + + if (result.status !== 0) { + return { + ...base, + status: 'fail', + blocking: true, + summary: 'Representative retained-artifact replay failed before producing normalized evidence outputs.', + details: [result.stdout?.trim(), result.stderr?.trim()].filter(Boolean) + }; + } + + const classificationPath = path.join(workspace, 'pester-evidence-classification.json'); + const operatorOutcomePath = path.join(workspace, 'pester-operator-outcome.json'); + const summaryPath = path.join(workspace, 'pester-summary.json'); + let classification; + let operatorOutcome; + let summary; + try { + classification = await readJson(classificationPath); + operatorOutcome = await readJson(operatorOutcomePath); + summary = await readJson(summaryPath); + } catch (error) { + return { + ...base, + status: 'fail', + blocking: true, + summary: 'Representative retained-artifact replay completed without emitting the expected normalized evidence files.', + details: [error instanceof Error ? error.message : String(error)] + }; + } + const ok = classification.classification === 'results-xml-truncated' + && operatorOutcome.nextActionId === 'inspect-results-xml-truncation' + && summary.schemaVersion === '1.7.1'; + + return { + ...base, + status: ok ? 'pass' : 'fail', + blocking: !ok, + summary: ok + ? 'Representative retained-artifact replay normalized a schema-lite truncated-XML run into current evidence contracts.' + : 'Representative retained-artifact replay completed but did not preserve the expected truncated-XML operator outcome contract.', + classification: classification.classification, + operator_next_action: operatorOutcome.nextActionId, + summary_schema_version: summary.schemaVersion, + output_paths: [ + relativeFrom(repoRoot, classificationPath), + relativeFrom(repoRoot, operatorOutcomePath), + relativeFrom(repoRoot, summaryPath) + ] + }; +} + +async function runWindowsContainerSurfaceProof(repoRoot, resultsDir) { + const sharedProof = await runSharedWindowsSurfaceProof(repoRoot, resultsDir); + return { + id: 'windows-container-surface', + owner_requirement: 'REQ-PSM-025', + status: sharedProof.status, + blocking: sharedProof.blocking, + summary: sharedProof.status === 'pass' + ? 'Windows-container surface is ready for local Docker Desktop + NI image proof.' + : sharedProof.status === 'advisory' + ? `Windows-container surface is ${sharedProof.current_surface_status}; use the recommended Docker Desktop + NI image commands when a live local proof is required.` + : sharedProof.summary, + surface_status: sharedProof.current_surface_status, + host_platform: sharedProof.current_host_platform, + coordinator_host_platform: sharedProof.coordinator_host_platform, + bridge_mode: sharedProof.bridge_mode, + reason: sharedProof.reason, + receipt_path: sharedProof.receipt_path, + recommended_commands: sharedProof.recommended_commands ?? [], + details: sharedProof.details ?? [] + }; +} + +export { parseCsv, parseRequirementNumber, determinePhase, rankRequirementGaps, rankProofRegressions, deriveEscalations, selectNextStep, applyAutonomyPolicy }; + +export async function runPesterServiceModelLocalCi({ + repoRoot = repoRootDefault, + skillRoot = null, + surfacePath = DEFAULT_SURFACE, + policyPath = DEFAULT_POLICY, + resultsDir = DEFAULT_RESULTS_DIR, + outputPath = DEFAULT_REPORT, + summaryPath = DEFAULT_SUMMARY, + nextOutputPath = DEFAULT_NEXT, + nextStepOutputPath = DEFAULT_NEXT_STEP +} = {}) { + const resolvedSkillRoot = await resolveSkillRoot(skillRoot); + const resolved = { + repoRoot, + skillRoot: resolvedSkillRoot, + surfacePath: path.join(repoRoot, surfacePath), + policyPath: path.join(repoRoot, policyPath), + resultsDir: path.join(repoRoot, resultsDir), + outputPath: path.join(repoRoot, outputPath), + summaryPath: path.join(repoRoot, summaryPath), + nextOutputPath: path.join(repoRoot, nextOutputPath), + nextStepOutputPath: path.join(repoRoot, nextStepOutputPath) + }; + + const manifest = await readYaml(resolved.surfacePath); + const policy = await readJson(resolved.policyPath); + const bundleRoot = createRunScopedBundleRoot(resolved.resultsDir); + await ensureDir(resolved.resultsDir); + await materializeAuditSurface(repoRoot, manifest, bundleRoot); + + const evidencePath = path.join(resolved.resultsDir, 'standards-evidence.json'); + const scorePath = path.join(resolved.resultsDir, 'standards-score.json'); + const rtmPath = path.join(repoRoot, 'docs', 'rtm-pester-service-model.csv'); + const rtmRows = parseCsv(await fs.readFile(rtmPath, 'utf8')); + + const evidence = runCommand('python3', [ + path.join(resolvedSkillRoot, 'scripts', 'repo_evidence_scan.py'), + bundleRoot, + '--format', + 'json', + '--profile', + 'quick-triage', + '--max-examples', + '2', + '--max-evidence-per-rule', + '2' + ], { cwd: repoRoot }).stdout; + await fs.writeFile(evidencePath, evidence, 'utf8'); + + const score = runCommand('python3', [ + path.join(resolvedSkillRoot, 'scripts', 'score_assurance.py'), + evidencePath, + '--format', + 'json' + ], { cwd: repoRoot }).stdout; + await fs.writeFile(scorePath, score, 'utf8'); + + const scorePayload = JSON.parse(score); + const worktreeStatus = collectWorktreeStatus(repoRoot); + const proofChecks = [ + await runRepresentativeReplayProof(repoRoot, resolved.resultsDir), + await runWindowsContainerSurfaceProof(repoRoot, resolved.resultsDir) + ]; + const proofRegressions = rankProofRegressions(proofChecks, rtmRows); + const rankedRequirementGaps = rankRequirementGaps(rtmRows); + const regressionReqIds = new Set(proofRegressions.map((item) => item.req_id)); + const rankedRequirements = applyAutonomyPolicy([ + ...proofRegressions, + ...rankedRequirementGaps.filter((item) => !regressionReqIds.has(item.req_id)) + ], policy, worktreeStatus.modifiedPaths); + const escalations = deriveEscalations(proofChecks); + const nextRequirement = rankedRequirements[0] ?? null; + const nextStep = selectNextStep(nextRequirement, escalations); + const implementedCount = rtmRows.filter((row) => row.Status === 'Implemented').length; + const gapCount = rankedRequirements.length; + const blockingProofFailures = proofChecks.filter((check) => check.blocking).length; + const advisoryProofChecks = proofChecks.filter((check) => check.status === 'advisory').length; + const overallStatus = gapCount === 0 && blockingProofFailures === 0 + ? (advisoryProofChecks > 0 ? 'pass-with-advisories' : 'pass') + : 'pass-with-actions'; + const blockingProof = proofChecks.find((check) => check.blocking) ?? null; + const overallReason = blockingProof + ? `The next recommended requirement is ${nextRequirement.req_id} because the local proof check '${blockingProof.id}' regressed against the current packet.` + : gapCount === 0 + ? (nextStep?.type === 'escalation' + ? `All tracked Pester service-model requirements are implemented locally, and the next step is escalation '${nextStep.escalation_id}' because the current host cannot satisfy the required proof surface.` + : advisoryProofChecks > 0 + ? 'All tracked Pester service-model requirements are implemented, and the remaining local note is the current Windows-container surface advisory.' + : 'All tracked Pester service-model requirements are currently marked implemented.') + : `The next recommended requirement is ${nextRequirement.req_id} because it is the highest-ranked unresolved gap on the local packet under the autonomy policy.`; + + const report = { + schema_version: '1.2.0', + generated_at: new Date().toISOString(), + repo_root: repoRoot, + audit_surface: { + id: manifest.id, + description: manifest.description, + manifest_path: relativeFrom(repoRoot, resolved.surfacePath), + bundle_root: relativeFrom(repoRoot, bundleRoot), + included_paths: manifest.include + }, + worktree_status: { + branch: worktreeStatus.branch, + modified_paths: worktreeStatus.modifiedPaths, + active_requirement_refs: rankedRequirements.filter((item) => item.active_now).map((item) => item.req_id) + }, + standards_audit: { + status: deriveWeakAreas(scorePayload).length === 0 ? 'pass' : 'pass-with-actions', + evidence_path: relativeFrom(repoRoot, evidencePath), + score_path: relativeFrom(repoRoot, scorePath), + weak_areas: deriveWeakAreas(scorePayload) + }, + requirement_summary: { + total: rtmRows.length, + implemented: implementedCount, + gaps: gapCount + }, + proof_checks: { + blocking_failures: blockingProofFailures, + advisories: advisoryProofChecks, + checks: proofChecks + }, + ranked_requirements: rankedRequirements, + escalations, + next_requirement: nextRequirement, + next_step: nextStep, + overall: { + status: overallStatus, + reason: overallReason + } + }; + + await validateReportSchema(repoRoot, report); + await writeJson(resolved.outputPath, report); + await writeJson(resolved.nextOutputPath, nextRequirement); + await validateNextStepSchema(repoRoot, nextStep); + await writeJson(resolved.nextStepOutputPath, nextStep); + await fs.writeFile(resolved.summaryPath, buildSummary(report), 'utf8'); + + return { report, paths: resolved }; +} + +async function main() { + try { + const options = parseArgs(); + if (options.help) { + printHelp(); + return; + } + const { report } = await runPesterServiceModelLocalCi(options); + if (options.printNext && report.next_requirement) { + console.log(JSON.stringify(report.next_requirement, null, 2)); + return; + } + if (options.printNextStep && report.next_step) { + console.log(JSON.stringify(report.next_step, null, 2)); + return; + } + console.log(report.overall.reason); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} + +const entryPath = process.argv[1] ? path.resolve(process.argv[1]) : null; +if (entryPath && fileURLToPath(import.meta.url) === entryPath) { + await main(); +} diff --git a/tools/priority/pester-service-model-provenance.mjs b/tools/priority/pester-service-model-provenance.mjs new file mode 100644 index 000000000..3c4588cac --- /dev/null +++ b/tools/priority/pester-service-model-provenance.mjs @@ -0,0 +1,114 @@ +#!/usr/bin/env node + +import crypto from 'node:crypto'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import process from 'node:process'; +import { spawnSync } from 'node:child_process'; + +export function toPortablePath(value) { + if (!value) { + return null; + } + return String(value).split(path.sep).join('/'); +} + +export function relativeOrAbsolute(repoRoot, filePath) { + const resolvedPath = path.resolve(filePath); + const relative = path.relative(repoRoot, resolvedPath); + if (!relative.startsWith('..') && !path.isAbsolute(relative)) { + return toPortablePath(relative || '.'); + } + return toPortablePath(resolvedPath); +} + +export function runGit(repoRoot, args) { + const result = spawnSync('git', args, { cwd: repoRoot, encoding: 'utf8' }); + if (result.status !== 0) { + return null; + } + return result.stdout.trim() || null; +} + +export function resolveRunContext(repoRoot, workflowFallback) { + const repository = process.env.GITHUB_REPOSITORY || path.basename(repoRoot); + const runId = process.env.GITHUB_RUN_ID || null; + const serverUrl = (process.env.GITHUB_SERVER_URL || 'https://github.com').replace(/\/$/, ''); + const branch = process.env.GITHUB_REF_NAME || runGit(repoRoot, ['rev-parse', '--abbrev-ref', 'HEAD']); + const headSha = process.env.GITHUB_SHA || runGit(repoRoot, ['rev-parse', 'HEAD']); + + return { + source: runId ? 'github-actions' : 'local', + repository, + workflow: process.env.GITHUB_WORKFLOW || workflowFallback, + eventName: process.env.GITHUB_EVENT_NAME || 'local', + runId, + runAttempt: process.env.GITHUB_RUN_ATTEMPT || null, + runUrl: runId ? `${serverUrl}/${repository}/actions/runs/${runId}` : null, + ref: process.env.GITHUB_REF || (branch ? `refs/heads/${branch}` : null), + refName: branch || null, + branch: branch || null, + headRef: process.env.GITHUB_HEAD_REF || null, + baseRef: process.env.GITHUB_BASE_REF || null, + headSha: headSha || null + }; +} + +async function sha256File(filePath) { + const buffer = await fs.readFile(filePath); + return crypto.createHash('sha256').update(buffer).digest('hex'); +} + +async function readOptionalJsonMetadata(filePath) { + try { + const document = JSON.parse(await fs.readFile(filePath, 'utf8')); + return { + schema: typeof document.schema === 'string' ? document.schema : null, + schemaVersion: typeof document.schemaVersion === 'string' ? document.schemaVersion : null, + status: + typeof document.status === 'string' + ? document.status + : typeof document.classification === 'string' + ? document.classification + : null + }; + } catch { + return { schema: null, schemaVersion: null, status: 'invalid-json' }; + } +} + +export async function describeFile(repoRoot, filePath, { kind, role, artifactName = null } = {}) { + const resolvedPath = path.resolve(filePath); + try { + const stat = await fs.stat(resolvedPath); + const metadata = await readOptionalJsonMetadata(resolvedPath); + return { + kind, + role, + artifactName, + path: toPortablePath(resolvedPath), + repoRelativePath: relativeOrAbsolute(repoRoot, resolvedPath), + present: true, + sizeBytes: stat.size, + sha256: await sha256File(resolvedPath), + lastWriteTimeUtc: stat.mtime.toISOString(), + schema: metadata.schema, + schemaVersion: metadata.schemaVersion, + status: metadata.status + }; + } catch { + return { + kind, + role, + artifactName, + path: toPortablePath(resolvedPath), + repoRelativePath: relativeOrAbsolute(repoRoot, resolvedPath), + present: false + }; + } +} + +export async function writeJsonFile(outputPath, payload) { + await fs.mkdir(path.dirname(outputPath), { recursive: true }); + await fs.writeFile(outputPath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8'); +} diff --git a/tools/priority/render-pester-service-model-promotion-dossier.mjs b/tools/priority/render-pester-service-model-promotion-dossier.mjs index 56d8c8208..23ec79f76 100644 --- a/tools/priority/render-pester-service-model-promotion-dossier.mjs +++ b/tools/priority/render-pester-service-model-promotion-dossier.mjs @@ -4,14 +4,19 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import process from 'node:process'; import { spawnSync } from 'node:child_process'; - -const repoRoot = process.cwd(); -const baseDir = path.join(repoRoot, 'tests', 'results', '_agent', 'pester-service-model'); -const releaseEvidenceDir = path.join(baseDir, 'release-evidence'); -const outputPath = path.join(releaseEvidenceDir, 'promotion-dossier.md'); +import { + describeFile, + relativeOrAbsolute, + resolveRunContext, + toPortablePath, + writeJsonFile +} from './pester-service-model-provenance.mjs'; function parseArgs(argv = process.argv.slice(2)) { const options = { + repoRoot: process.cwd(), + releaseEvidenceDir: null, + outputPath: null, upstreamIssue: '2069', forkIssue: '2078', forkBasisCommit: '', @@ -26,6 +31,21 @@ function parseArgs(argv = process.argv.slice(2)) { i += 1; continue; } + if (token === '--repo-root') { + options.repoRoot = path.resolve(next); + i += 1; + continue; + } + if (token === '--release-evidence-dir') { + options.releaseEvidenceDir = path.resolve(next); + i += 1; + continue; + } + if (token === '--output') { + options.outputPath = path.resolve(next); + i += 1; + continue; + } if (token === '--fork-issue') { options.forkIssue = next; i += 1; @@ -48,6 +68,7 @@ function parseArgs(argv = process.argv.slice(2)) { } function runGit(args) { + const repoRoot = optionsCache.repoRoot; const result = spawnSync('git', args, { cwd: repoRoot, encoding: 'utf8' }); if (result.status !== 0) { throw new Error(result.stderr?.trim() || `git ${args.join(' ')} failed`); @@ -55,18 +76,50 @@ function runGit(args) { return result.stdout.trim(); } +const optionsCache = { repoRoot: process.cwd() }; + async function ensureFile(filePath) { await fs.access(filePath); } async function main() { const options = parseArgs(); + optionsCache.repoRoot = options.repoRoot; + const baseDir = path.join(options.repoRoot, 'tests', 'results', '_agent', 'pester-service-model'); + const releaseEvidenceDir = options.releaseEvidenceDir ?? path.join(baseDir, 'release-evidence'); + const outputPath = options.outputPath ?? path.join(releaseEvidenceDir, 'promotion-dossier.md'); const coveragePath = path.join(releaseEvidenceDir, 'coverage.xml'); const docsPath = path.join(releaseEvidenceDir, 'docs-link-check.json'); - const rtm = await fs.readFile(path.join(repoRoot, 'docs', 'rtm-pester-service-model.csv'), 'utf8'); + const comparisonBundlePath = path.join(releaseEvidenceDir, 'pester-service-model-promotion-comparison.json'); + const requirementsPath = path.join(options.repoRoot, 'docs', 'requirements-pester-service-model-srs.md'); + const rtmPath = path.join(options.repoRoot, 'docs', 'rtm-pester-service-model.csv'); + const qualityReportPath = path.join(options.repoRoot, 'docs', 'pester-service-model-quality-report.md'); + const releaseProvenancePath = path.join(releaseEvidenceDir, 'release-evidence-provenance.json'); + const rtm = await fs.readFile(rtmPath, 'utf8'); + const comparison = JSON.parse(await fs.readFile(comparisonBundlePath, 'utf8')); await fs.mkdir(releaseEvidenceDir, { recursive: true }); - await Promise.all([ensureFile(coveragePath), ensureFile(docsPath)]); + await Promise.all([ensureFile(coveragePath), ensureFile(docsPath), ensureFile(comparisonBundlePath), ensureFile(releaseProvenancePath)]); + + const comparisonLines = []; + for (const item of comparison.comparisons ?? []) { + comparisonLines.push(`### ${item.packId}`); + comparisonLines.push(''); + comparisonLines.push(`- Comparison: \`${item.comparisonId}\``); + comparisonLines.push(`- Representativeness: ${item.representativeness}`); + comparisonLines.push(`- Requirement coverage: ${(item.requirementCoverage ?? []).join(', ')}`); + comparisonLines.push(`- Baseline: [run ${item.baseline.runId}](${item.baseline.runUrl}) on \`${item.baseline.ref}\` (${item.baseline.conclusion}; pack=\`${item.baseline.packIdentity}\`)`); + comparisonLines.push(`- Candidate: [run ${item.candidate.runId}](${item.candidate.runUrl}) on \`${item.candidate.ref}\` (${item.candidate.conclusion}; pack=\`${item.candidate.packIdentity}\`)`); + comparisonLines.push(`- Decision: ${item.decision}`); + comparisonLines.push(`- Next action: ${item.nextAction}`); + if ((item.observedDeltas ?? []).length > 0) { + comparisonLines.push('- Observed deltas:'); + for (const delta of item.observedDeltas) { + comparisonLines.push(` - ${delta}`); + } + } + comparisonLines.push(''); + } const lines = [ '# Pester Service Model Promotion Dossier', @@ -96,6 +149,15 @@ async function main() { '- hosted packet quality retains `coverage.xml`', '- hosted docs integrity retains `docs-link-check.json`', '- the release-evidence bundle retains the upstream quality report and RTM alongside the evidence artifacts', + '- the release-evidence bundle retains `pester-service-model-promotion-comparison.json` for requirement-to-run comparison evidence', + '- provenance is retained in `release-evidence-provenance.json` and `promotion-dossier-provenance.json`', + '', + '## Representative Pack Comparisons', + '', + `- Decision state: ${comparison.decisionState}`, + `- Summary: ${comparison.summary}`, + '', + ...comparisonLines, '', '## Minimal Upstream Slice', '', @@ -106,7 +168,48 @@ async function main() { ]; await fs.writeFile(outputPath, `${lines.join('\n')}\n`, 'utf8'); + const releaseRecordPaths = (await fs.readdir(releaseEvidenceDir)) + .filter((entry) => /^release-record-.*\.md$/.test(entry)) + .sort() + .map((entry) => path.join(releaseEvidenceDir, entry)); + const provenanceOutputPath = path.join(releaseEvidenceDir, 'promotion-dossier-provenance.json'); + const sourceInputs = await Promise.all([ + describeFile(options.repoRoot, coveragePath, { kind: 'release-evidence-input', role: 'coverage-xml' }), + describeFile(options.repoRoot, docsPath, { kind: 'release-evidence-input', role: 'docs-link-check' }), + describeFile(options.repoRoot, requirementsPath, { kind: 'release-evidence-input', role: 'requirements-srs' }), + describeFile(options.repoRoot, comparisonBundlePath, { kind: 'release-evidence-input', role: 'promotion-comparison' }), + describeFile(options.repoRoot, rtmPath, { kind: 'release-evidence-input', role: 'rtm' }), + describeFile(options.repoRoot, qualityReportPath, { kind: 'release-evidence-input', role: 'quality-report' }), + describeFile(options.repoRoot, releaseProvenancePath, { kind: 'release-evidence-input', role: 'release-evidence-provenance' }), + ...releaseRecordPaths.map((filePath, index) => describeFile(options.repoRoot, filePath, { kind: 'release-evidence-input', role: `release-record-${index + 1}` })) + ]); + const derivedOutputs = await Promise.all([ + describeFile(options.repoRoot, outputPath, { kind: 'promotion-output', role: 'promotion-dossier' }) + ]); + await writeJsonFile(provenanceOutputPath, { + schema: 'pester-derived-provenance@v1', + schemaVersion: '1.0.0', + generatedAtUtc: new Date().toISOString(), + provenanceKind: 'promotion-dossier', + producer: { + id: 'render-pester-service-model-promotion-dossier.mjs', + version: '1.0.0' + }, + subject: { + id: 'pester-service-model-promotion-dossier', + upstreamIssue: options.upstreamIssue, + forkIssue: options.forkIssue, + forkBasisCommit: options.forkBasisCommit || null, + forkBasisUrl: options.forkBasisUrl || null, + releaseEvidenceDir: toPortablePath(releaseEvidenceDir), + releaseEvidenceDirRepoPath: relativeOrAbsolute(options.repoRoot, releaseEvidenceDir) + }, + runContext: resolveRunContext(options.repoRoot, 'Pester service-model release evidence'), + sourceInputs, + derivedOutputs + }); console.log(`promotion_dossier=${outputPath}`); + console.log(`promotion_dossier_provenance=${provenanceOutputPath}`); } main().catch((error) => { diff --git a/tools/priority/vi-history-live-candidate.json b/tools/priority/vi-history-live-candidate.json new file mode 100644 index 000000000..6143cf52d --- /dev/null +++ b/tools/priority/vi-history-live-candidate.json @@ -0,0 +1,21 @@ +{ + "$schema": "../../docs/schemas/vi-history-live-candidate-v1.schema.json", + "schemaVersion": "1.0.0", + "id": "ni-icon-editor-vip-preuninstall-live-history", + "repoSlug": "ni/labview-icon-editor", + "repoUrl": "https://github.com/ni/labview-icon-editor", + "defaultBranch": "develop", + "cloneRootEnvVar": "COMPAREVI_VI_HISTORY_CANDIDATE_ROOT", + "preferredLocalCloneRoots": [ + "/tmp/labview-icon-editor" + ], + "targetViPath": "Tooling/deployment/VIP_Pre-Uninstall Custom Action.vi", + "historyExpectation": { + "minCommits": 2 + }, + "iterationRationale": "Use a real downstream repo clone with live commit history for VI History iteration instead of relying only on retained fixtures or public proof seeds.", + "notes": [ + "This manifest governs the local clone-backed iteration candidate only.", + "Public accepted corpus targets may differ until target-specific public proof is promoted." + ] +} diff --git a/tools/priority/vi-history-local-ci.mjs b/tools/priority/vi-history-local-ci.mjs new file mode 100644 index 000000000..fb5c9163b --- /dev/null +++ b/tools/priority/vi-history-local-ci.mjs @@ -0,0 +1,1017 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; +import Ajv2020 from 'ajv/dist/2020.js'; +import addFormats from 'ajv-formats'; +import yaml from 'js-yaml'; + +import { + buildWindowsNodeBridgeSpec, + detectWindowsHostBridge, + runBridgeSpec, +} from './windows-host-bridge.mjs'; +import { runWindowsSurfaceProof as runSharedWindowsDockerSurfaceProof } from './windows-docker-shared-surface-local-ci.mjs'; + +const repoRootDefault = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); +const defaultSkillRoot = path.join(process.env.HOME ?? '', '.codex', 'skills', 'repo-standards-review'); +const fallbackSkillRoot = '/mnt/c/Users/sveld/.codex/skills/repo-standards-review'; + +const DEFAULT_SURFACE = path.join('tools', 'priority', 'vi-history-local-proof-audit-surface.yaml'); +const DEFAULT_POLICY = path.join('tools', 'priority', 'vi-history-local-proof-autonomy-policy.json'); +const DEFAULT_LIVE_CANDIDATE = path.join('tools', 'priority', 'vi-history-live-candidate.json'); +const DEFAULT_RESULTS_DIR = path.join('tests', 'results', '_agent', 'vi-history-local-proof', 'local-ci'); +const DEFAULT_REPORT = path.join(DEFAULT_RESULTS_DIR, 'vi-history-local-ci-report.json'); +const DEFAULT_SUMMARY = path.join(DEFAULT_RESULTS_DIR, 'vi-history-local-ci-summary.md'); +const DEFAULT_NEXT = path.join(DEFAULT_RESULTS_DIR, 'vi-history-local-next-requirement.json'); +const DEFAULT_NEXT_STEP = path.join(DEFAULT_RESULTS_DIR, 'vi-history-local-next-step.json'); + +const HELP = [ + 'Usage: node tools/priority/vi-history-local-ci.mjs [options]', + '', + 'Options:', + ` --repo-root (default: ${repoRootDefault})`, + ` --skill-root (default: ${defaultSkillRoot} or ${fallbackSkillRoot})`, + ` --surface (default: ${DEFAULT_SURFACE})`, + ` --policy (default: ${DEFAULT_POLICY})`, + ` --results-dir (default: ${DEFAULT_RESULTS_DIR})`, + ` --output (default: ${DEFAULT_REPORT})`, + ` --summary-output (default: ${DEFAULT_SUMMARY})`, + ` --next-output (default: ${DEFAULT_NEXT})`, + ` --next-step-output (default: ${DEFAULT_NEXT_STEP})`, + ' --print-next print the selected next requirement to stdout', + ' --print-next-step print the selected next step to stdout', + ' --help, -h' +]; + +function parseArgs(argv = process.argv) { + const options = { + repoRoot: repoRootDefault, + skillRoot: null, + surfacePath: DEFAULT_SURFACE, + policyPath: DEFAULT_POLICY, + resultsDir: DEFAULT_RESULTS_DIR, + outputPath: DEFAULT_REPORT, + summaryPath: DEFAULT_SUMMARY, + nextOutputPath: DEFAULT_NEXT, + nextStepOutputPath: DEFAULT_NEXT_STEP, + printNext: false, + printNextStep: false, + help: false + }; + + const args = argv.slice(2); + for (let i = 0; i < args.length; i += 1) { + const token = args[i]; + const next = args[i + 1]; + if (token === '--help' || token === '-h') { + options.help = true; + continue; + } + if (token === '--print-next') { + options.printNext = true; + continue; + } + if (token === '--print-next-step') { + options.printNextStep = true; + continue; + } + if (['--repo-root', '--skill-root', '--surface', '--policy', '--results-dir', '--output', '--summary-output', '--next-output', '--next-step-output'].includes(token)) { + if (!next || next.startsWith('-')) { + throw new Error(`Missing value for ${token}`); + } + i += 1; + if (token === '--repo-root') options.repoRoot = path.resolve(next); + if (token === '--skill-root') options.skillRoot = path.resolve(next); + if (token === '--surface') options.surfacePath = next; + if (token === '--policy') options.policyPath = next; + if (token === '--results-dir') options.resultsDir = next; + if (token === '--output') options.outputPath = next; + if (token === '--summary-output') options.summaryPath = next; + if (token === '--next-output') options.nextOutputPath = next; + if (token === '--next-step-output') options.nextStepOutputPath = next; + continue; + } + throw new Error(`Unknown option: ${token}`); + } + + return options; +} + +function printHelp() { + for (const line of HELP) console.log(line); +} + +async function fileExists(filePath) { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } +} + +async function resolveSkillRoot(explicitRoot) { + if (explicitRoot) return explicitRoot; + const candidates = []; + if (process.env.CODEX_HOME) { + candidates.push(path.join(process.env.CODEX_HOME, 'skills', 'repo-standards-review')); + } + candidates.push(defaultSkillRoot, fallbackSkillRoot); + for (const candidate of candidates) { + if (await fileExists(candidate)) return candidate; + } + return defaultSkillRoot; +} + +async function readYaml(filePath) { + return yaml.load(await fs.readFile(filePath, 'utf8')); +} + +async function ensureDir(filePath) { + await fs.mkdir(filePath, { recursive: true }); +} + +async function readJson(filePath) { + return JSON.parse(await fs.readFile(filePath, 'utf8')); +} + +function relativeFrom(repoRoot, filePath) { + return path.relative(repoRoot, filePath).split(path.sep).join('/'); +} + +function dedupeOrdered(values) { + const ordered = []; + const seen = new Set(); + for (const value of values) { + if (!value || seen.has(value)) continue; + seen.add(value); + ordered.push(value); + } + return ordered; +} + +async function materializeAuditSurface(repoRoot, manifest, bundleRoot) { + await ensureDir(bundleRoot); + for (const relativePath of manifest.include) { + const source = path.join(repoRoot, relativePath); + const destination = path.join(bundleRoot, relativePath); + await ensureDir(path.dirname(destination)); + await fs.copyFile(source, destination); + } +} + +function createRunScopedBundleRoot(resultsDir) { + const runId = `run-${Date.now()}-${process.pid}-${Math.random().toString(16).slice(2, 8)}`; + return path.join(resultsDir, 'surface-bundle', runId); +} + +function runCommand(command, args, options = {}) { + const result = spawnSync(command, args, { + cwd: options.cwd, + encoding: 'utf8', + maxBuffer: 20 * 1024 * 1024 + }); + if (result.status !== 0) { + throw new Error([`${command} ${args.join(' ')}`, result.stdout?.trim(), result.stderr?.trim()].filter(Boolean).join('\n')); + } + return result; +} + +function parseCsv(input) { + const rows = []; + let current = ''; + let row = []; + let inQuotes = false; + + for (let i = 0; i < input.length; i += 1) { + const char = input[i]; + const next = input[i + 1]; + if (char === '"') { + if (inQuotes && next === '"') { + current += '"'; + i += 1; + } else { + inQuotes = !inQuotes; + } + continue; + } + if (char === ',' && !inQuotes) { + row.push(current); + current = ''; + continue; + } + if ((char === '\n' || char === '\r') && !inQuotes) { + if (char === '\r' && next === '\n') i += 1; + row.push(current); + current = ''; + if (row.some((value) => value.length > 0)) rows.push(row); + row = []; + continue; + } + current += char; + } + + if (current.length > 0 || row.length > 0) { + row.push(current); + if (row.some((value) => value.length > 0)) rows.push(row); + } + + if (rows.length === 0) return []; + const [header, ...records] = rows; + return records.map((values) => { + const entry = {}; + for (let i = 0; i < header.length; i += 1) { + entry[header[i]] = values[i] ?? ''; + } + return entry; + }); +} + +function parseRequirementNumber(reqId) { + const match = /REQ-VHLP-(\d+)/.exec(reqId ?? ''); + return match ? Number.parseInt(match[1], 10) : Number.POSITIVE_INFINITY; +} + +function determinePhase(reqNumber) { + if (reqNumber <= 4) return 'foundation'; + return 'autonomy'; +} + +function splitField(value) { + return String(value ?? '') + .split(';') + .map((entry) => entry.trim()) + .filter(Boolean); +} + +function isOperationalRef(ref) { + return !String(ref).startsWith('docs/'); +} + +function collectWorktreeStatus(repoRoot) { + const result = runCommand('git', ['status', '--short', '--branch'], { cwd: repoRoot }); + const lines = result.stdout.split(/\r?\n/).filter(Boolean); + const branch = (lines.find((line) => line.startsWith('## ')) ?? '## detached').replace(/^##\s+/, ''); + const modifiedPaths = lines + .filter((line) => !line.startsWith('## ')) + .map((line) => line.slice(3).trim()) + .filter(Boolean); + return { branch, modifiedPaths }; +} + +function deriveWeakAreas(scorePayload) { + return Object.entries(scorePayload.areas ?? {}) + .filter(([, details]) => Number(details.score ?? 0) < 3) + .map(([area]) => area); +} + +function scoreGap(row) { + const reqNumber = parseRequirementNumber(row.ReqID); + const phase = determinePhase(reqNumber); + const priorityWeight = row.Priority === 'High' ? 200 : row.Priority === 'Medium' ? 100 : 50; + const phaseWeight = phase === 'foundation' ? 300 : 180; + const runtimeWeight = /(windows-workflow-replay-lane|Invoke-VIHistoryLocalRefinement|Invoke-VIHistoryLocalOperatorSession|Write-VIHistoryWorkflowReadiness)/.test(row.CodeRef) ? 40 : 0; + return 1000 + priorityWeight + phaseWeight + runtimeWeight - reqNumber; +} + +function deriveSuggestedLoop(row) { + const codeRefs = splitField(row.CodeRef); + const actions = []; + actions.push(`Start with ${row.TestID}: tighten local VI History proof coverage before spending GitHub CI.`); + if (codeRefs.length > 0) { + actions.push(`Edit the primary source targets first: ${codeRefs.slice(0, 3).join(', ')}.`); + } + actions.push('Re-run the local VI History CI and next-step entrypoints after the change.'); + return actions; +} + +function buildRequirementEntry(row, overrides = {}) { + const reqNumber = parseRequirementNumber(row.ReqID); + const phase = determinePhase(reqNumber); + return { + req_id: row.ReqID, + priority: row.Priority, + status: overrides.status ?? row.Status, + phase, + score: scoreGap(row) + (overrides.scoreBoost ?? 0), + why_now: overrides.why_now ?? `${row.ReqID} is an unresolved ${row.Priority} ${phase} VI History local-proof gap.`, + requirement: row.Requirement, + test_id: row.TestID, + test_artifact: row.TestArtifact, + code_refs: splitField(row.CodeRef), + suggested_loop: overrides.suggested_loop ?? deriveSuggestedLoop(row), + proof_check_id: overrides.proof_check_id ?? null + }; +} + +function rankRequirementGaps(rows) { + return rows + .filter((row) => row.Status !== 'Implemented') + .map((row) => buildRequirementEntry(row)) + .sort((a, b) => { + if (b.score !== a.score) return b.score - a.score; + return parseRequirementNumber(a.req_id) - parseRequirementNumber(b.req_id); + }) + .map((entry, index) => ({ rank: index + 1, ...entry })); +} + +function rankProofRegressions(proofChecks, rows) { + const byReqId = new Map(rows.map((row) => [row.ReqID, row])); + return proofChecks + .filter((check) => check.status === 'fail' && check.owner_requirement && byReqId.has(check.owner_requirement)) + .map((check) => { + const row = byReqId.get(check.owner_requirement); + return buildRequirementEntry(row, { + status: 'Regression', + scoreBoost: 5000, + why_now: `${row.ReqID} regressed under the VI History proof check '${check.id}': ${check.summary}`, + suggested_loop: [ + `Start with proof check ${check.id}: ${check.summary}`, + ...deriveSuggestedLoop(row) + ], + proof_check_id: check.id + }); + }) + .sort((a, b) => { + if (b.score !== a.score) return b.score - a.score; + return parseRequirementNumber(a.req_id) - parseRequirementNumber(b.req_id); + }); +} + +function deriveEscalations(proofChecks) { + return proofChecks + .filter((check) => check.status === 'advisory' && (check.id === 'windows-workflow-replay' || check.id === 'live-history-candidate')) + .map((check) => { + if (check.id === 'live-history-candidate') { + return { + type: 'escalation', + escalation_id: 'clone-backed-live-history-candidate', + governing_requirement: 'REQ-VHLP-009', + blocked_requirement: 'REQ-VHLP-008', + proof_check_id: check.id, + status: 'required', + mode: 'escalate', + why_now: 'The governed clone-backed VI History candidate is not ready for truthful local iteration yet.', + reason: check.reason, + required_surface: 'clone-backed-live-history-candidate', + current_surface_status: check.current_surface_status ?? 'unknown', + current_host_platform: check.current_host_platform ?? 'unknown', + receipt_path: check.receipt_path ?? null, + suggested_loop: [ + 'Prepare the governed repository clone at the declared root or set the clone-root override environment variable.', + 'Verify the target VI path and its git history before choosing Windows replay or GitHub CI.', + 'Re-run the VI History local CI entrypoint once the candidate receipt reaches status=ready.' + ], + recommended_commands: check.recommended_commands ?? [], + stop_conditions: [ + 'Stop once vi-history-live-candidate-readiness.json reaches status=ready.', + 'Stop if the candidate repo exposes a different target path or history shape than the governed packet expects.' + ] + }; + } + + return { + type: 'escalation', + escalation_id: 'windows-docker-desktop-ni-image', + governing_requirement: 'REQ-VHLP-006', + blocked_requirement: check.owner_requirement, + proof_check_id: check.id, + status: 'required', + mode: 'escalate', + why_now: `The next truthful VI History proof surface for ${check.owner_requirement} is unavailable from the current host.`, + reason: check.current_host_platform === 'Unix' + ? 'Current host is not Windows, so the VI History Windows workflow replay lane cannot be exercised here.' + : `VI History Windows workflow replay reported ${check.current_surface_status}; prepare the Windows Docker Desktop + NI image surface before another hosted rerun.`, + required_surface: 'windows-docker-desktop-ni-image', + current_surface_status: check.current_surface_status ?? 'unknown', + current_host_platform: check.current_host_platform ?? 'unknown', + receipt_path: check.receipt_path ?? null, + suggested_loop: [ + 'Move to a Windows host with Docker Desktop configured for Windows containers.', + 'Bootstrap or verify the pinned NI Windows image before running the VI History replay lane.', + 'Re-run the governed Windows workflow replay lane before choosing a hosted GitHub rerun.' + ], + recommended_commands: check.recommended_commands ?? [], + stop_conditions: [ + 'Stop once the VI History Windows workflow replay lane reaches status=passed.', + 'Stop if the replay lane exposes a new blocking Windows-host or image defect.' + ] + }; + }); +} + +function selectNextStep(nextRequirement, escalations) { + if (nextRequirement) { + return { type: 'requirement', ...nextRequirement }; + } + return escalations[0] ?? null; +} + +function applyAutonomyPolicy(rankedRequirements, policy, modifiedPaths) { + const modifiedSet = new Set(modifiedPaths); + return rankedRequirements + .map((item) => { + const phasePolicy = policy.phase_guidance?.[item.phase] ?? {}; + const activeNow = item.code_refs.some((ref) => isOperationalRef(ref) && modifiedSet.has(ref)); + return { + ...item, + active_now: activeNow, + mode: phasePolicy.mode ?? 'local-first', + preferred_commands: phasePolicy.preferred_commands ?? [], + stop_conditions: phasePolicy.stop_conditions ?? [], + escalate_when: phasePolicy.escalate_when ?? [] + }; + }) + .sort((a, b) => { + if (a.active_now !== b.active_now) return a.active_now ? -1 : 1; + if (b.score !== a.score) return b.score - a.score; + return parseRequirementNumber(a.req_id) - parseRequirementNumber(b.req_id); + }) + .map((entry, index) => ({ ...entry, rank: index + 1 })); +} + +function buildSummary(report) { + const lines = [ + '# VI History Local CI', + '', + `- Overall: ${report.overall.status}`, + `- Reason: ${report.overall.reason}`, + `- Standards audit: ${report.standards_audit.status}`, + `- Requirements: ${report.requirement_summary.implemented}/${report.requirement_summary.total} implemented`, + `- Gaps: ${report.requirement_summary.gaps}`, + `- Proof checks: ${report.proof_checks.blocking_failures} blocking, ${report.proof_checks.advisories} advisory`, + '' + ]; + + if (report.next_step?.type === 'requirement' && report.next_requirement) { + lines.push('## Next Step', ''); + lines.push(`- Type: requirement`); + lines.push(`- ${report.next_requirement.req_id}`); + lines.push(`- Phase: ${report.next_requirement.phase}`); + lines.push(`- Why now: ${report.next_requirement.why_now}`); + lines.push(`- Test: ${report.next_requirement.test_id}`); + lines.push(`- Code refs: ${report.next_requirement.code_refs.join(', ') || '(none)'}`); + lines.push(''); + } + + if (report.next_step?.type === 'escalation') { + lines.push('## Next Step', ''); + lines.push(`- Type: escalation`); + lines.push(`- Escalation: ${report.next_step.escalation_id}`); + lines.push(`- Governing requirement: ${report.next_step.governing_requirement}`); + lines.push(`- Blocked requirement: ${report.next_step.blocked_requirement}`); + lines.push(`- Reason: ${report.next_step.reason}`); + lines.push(`- Required surface: ${report.next_step.required_surface}`); + lines.push(`- Current host platform: ${report.next_step.current_host_platform}`); + lines.push(''); + } + + if (report.proof_checks.checks.length > 0) { + lines.push('## Proof Checks', ''); + for (const check of report.proof_checks.checks) { + lines.push(`- ${check.id}: ${check.status}`); + lines.push(` ${check.summary}`); + } + lines.push(''); + } + + return `${lines.join('\n')}\n`; +} + +async function writeJson(filePath, payload) { + await ensureDir(path.dirname(filePath)); + await fs.writeFile(filePath, JSON.stringify(payload, null, 2), 'utf8'); +} + +async function validateJsonAgainstSchema(repoRoot, schemaRelativePath, payload, label) { + const schemaPath = path.join(repoRoot, schemaRelativePath); + const schema = await readJson(schemaPath); + const ajv = new Ajv2020({ allErrors: true, strict: false }); + addFormats(ajv); + const validate = ajv.compile(schema); + const ok = validate(payload); + if (!ok) { + const details = (validate.errors ?? []).map((entry) => `${entry.instancePath || '/'} ${entry.message}`).join('; '); + throw new Error(`${label} schema validation failed: ${details}`); + } +} + +async function validateReportSchema(repoRoot, report) { + await validateJsonAgainstSchema(repoRoot, path.join('docs', 'schemas', 'vi-history-local-ci-report-v1.schema.json'), report, 'VI History local CI report'); +} + +async function validateNextStepSchema(repoRoot, nextStep) { + await validateJsonAgainstSchema(repoRoot, path.join('docs', 'schemas', 'vi-history-local-next-step-v1.schema.json'), nextStep, 'VI History next-step'); +} + +async function loadLiveCandidate(repoRoot, candidatePath = DEFAULT_LIVE_CANDIDATE) { + const resolvedPath = path.join(repoRoot, candidatePath); + const candidate = await readJson(resolvedPath); + await validateJsonAgainstSchema(repoRoot, path.join('docs', 'schemas', 'vi-history-live-candidate-v1.schema.json'), candidate, 'VI History live candidate'); + return { candidate, resolvedPath }; +} + +async function runLiveHistoryCandidateProof(repoRoot, resultsDir, candidatePath = DEFAULT_LIVE_CANDIDATE) { + const { candidate } = await loadLiveCandidate(repoRoot, candidatePath); + const candidateDir = path.join(resultsDir, 'live-candidate'); + const receiptPath = path.join(candidateDir, 'vi-history-live-candidate-readiness.json'); + await fs.rm(candidateDir, { recursive: true, force: true }); + await ensureDir(candidateDir); + + const envCloneRoot = process.env[candidate.cloneRootEnvVar] ?? ''; + const cloneRootCandidates = dedupeOrdered([envCloneRoot, ...(candidate.preferredLocalCloneRoots ?? [])]); + const resolvedCloneRoot = cloneRootCandidates.find((entry) => entry && spawnSync('git', ['-C', entry, 'rev-parse', '--show-toplevel'], { encoding: 'utf8' }).status === 0) ?? null; + + const baseReceipt = { + schema: 'vi-history/live-candidate-readiness@v1', + generatedAt: new Date().toISOString(), + candidateId: candidate.id, + repoSlug: candidate.repoSlug, + repoUrl: candidate.repoUrl, + defaultBranch: candidate.defaultBranch, + cloneRootEnvVar: candidate.cloneRootEnvVar, + cloneRootCandidates, + resolvedCloneRoot, + targetViPath: candidate.targetViPath, + historyExpectation: candidate.historyExpectation, + recommendedCommands: [] + }; + + if (!resolvedCloneRoot) { + const preferredCloneRoot = candidate.preferredLocalCloneRoots?.[0] ?? '/tmp/labview-icon-editor'; + const receipt = { + ...baseReceipt, + status: 'missing-clone', + reason: `No local clone for ${candidate.repoSlug} was found at the governed candidate roots.`, + recommendedCommands: [ + `git clone ${candidate.repoUrl}.git ${preferredCloneRoot}`, + `git -C ${preferredCloneRoot} switch ${candidate.defaultBranch}`, + 'npm run priority:vi-history:local-ci' + ] + }; + await validateJsonAgainstSchema(repoRoot, path.join('docs', 'schemas', 'vi-history-live-candidate-readiness-v1.schema.json'), receipt, 'VI History live candidate readiness'); + await writeJson(receiptPath, receipt); + return { + id: 'live-history-candidate', + owner_requirement: 'REQ-VHLP-009', + status: 'advisory', + blocking: false, + summary: 'The governed clone-backed VI History candidate is not cloned locally yet.', + current_surface_status: receipt.status, + current_host_platform: process.platform === 'win32' ? 'Windows' : 'Unix', + reason: receipt.reason, + receipt_path: relativeFrom(repoRoot, receiptPath), + recommended_commands: receipt.recommendedCommands + }; + } + + const targetAbsolutePath = path.join(resolvedCloneRoot, candidate.targetViPath); + if (!(await fileExists(targetAbsolutePath))) { + const receipt = { + ...baseReceipt, + status: 'missing-target', + reason: `The governed target VI does not exist at ${candidate.targetViPath} inside ${resolvedCloneRoot}.`, + recommendedCommands: [ + `git -C ${resolvedCloneRoot} switch ${candidate.defaultBranch}`, + `find ${resolvedCloneRoot} -type f | rg "VIP_(Pre|Post)-.*Custom Action\\\\.vi$"`, + 'npm run priority:vi-history:local-ci' + ] + }; + await validateJsonAgainstSchema(repoRoot, path.join('docs', 'schemas', 'vi-history-live-candidate-readiness-v1.schema.json'), receipt, 'VI History live candidate readiness'); + await writeJson(receiptPath, receipt); + return { + id: 'live-history-candidate', + owner_requirement: 'REQ-VHLP-009', + status: 'advisory', + blocking: false, + summary: 'The governed clone-backed VI History candidate is cloned, but the target VI path is missing.', + current_surface_status: receipt.status, + current_host_platform: process.platform === 'win32' ? 'Windows' : 'Unix', + reason: receipt.reason, + receipt_path: relativeFrom(repoRoot, receiptPath), + recommended_commands: receipt.recommendedCommands + }; + } + + const historyResult = spawnSync('git', [ + '-C', + resolvedCloneRoot, + 'log', + '--follow', + '--format=%H', + '--', + candidate.targetViPath + ], { + encoding: 'utf8', + maxBuffer: 20 * 1024 * 1024 + }); + + if (historyResult.status !== 0) { + const receipt = { + ...baseReceipt, + status: 'git-failed', + reason: `git history lookup failed for ${candidate.targetViPath}.`, + recommendedCommands: [ + `git -C ${resolvedCloneRoot} status --short --branch`, + `git -C ${resolvedCloneRoot} log --follow --oneline -- "${candidate.targetViPath}"`, + 'npm run priority:vi-history:local-ci' + ], + details: [historyResult.stdout?.trim(), historyResult.stderr?.trim()].filter(Boolean) + }; + await validateJsonAgainstSchema(repoRoot, path.join('docs', 'schemas', 'vi-history-live-candidate-readiness-v1.schema.json'), receipt, 'VI History live candidate readiness'); + await writeJson(receiptPath, receipt); + return { + id: 'live-history-candidate', + owner_requirement: 'REQ-VHLP-009', + status: 'fail', + blocking: true, + summary: 'git history lookup for the governed VI History candidate failed.', + current_surface_status: receipt.status, + current_host_platform: process.platform === 'win32' ? 'Windows' : 'Unix', + reason: receipt.reason, + receipt_path: relativeFrom(repoRoot, receiptPath), + recommended_commands: receipt.recommendedCommands, + details: receipt.details + }; + } + + const commitLines = historyResult.stdout.split(/\r?\n/).map((line) => line.trim()).filter(Boolean); + if (commitLines.length < Number(candidate.historyExpectation?.minCommits ?? 1)) { + const receipt = { + ...baseReceipt, + status: 'missing-history', + reason: `The governed target VI does not expose the minimum required git history depth (${candidate.historyExpectation.minCommits} commits).`, + history: { + commitCount: commitLines.length, + latestCommit: commitLines[0] ?? null + }, + recommendedCommands: [ + `git -C ${resolvedCloneRoot} log --follow --oneline -- "${candidate.targetViPath}"`, + 'npm run priority:vi-history:local-ci' + ] + }; + await validateJsonAgainstSchema(repoRoot, path.join('docs', 'schemas', 'vi-history-live-candidate-readiness-v1.schema.json'), receipt, 'VI History live candidate readiness'); + await writeJson(receiptPath, receipt); + return { + id: 'live-history-candidate', + owner_requirement: 'REQ-VHLP-009', + status: 'advisory', + blocking: false, + summary: 'The governed clone-backed VI History candidate does not expose enough git history yet.', + current_surface_status: receipt.status, + current_host_platform: process.platform === 'win32' ? 'Windows' : 'Unix', + reason: receipt.reason, + receipt_path: relativeFrom(repoRoot, receiptPath), + recommended_commands: receipt.recommendedCommands + }; + } + + const receipt = { + ...baseReceipt, + status: 'ready', + reason: 'The governed live-history candidate is cloned, the target VI exists, and git history is available.', + history: { + commitCount: commitLines.length, + latestCommit: commitLines[0] ?? null + }, + recommendedCommands: [ + `git -C ${resolvedCloneRoot} log --follow --oneline -- "${candidate.targetViPath}"`, + 'npm run priority:workflow:replay:windows:vi-history' + ] + }; + await validateJsonAgainstSchema(repoRoot, path.join('docs', 'schemas', 'vi-history-live-candidate-readiness-v1.schema.json'), receipt, 'VI History live candidate readiness'); + await writeJson(receiptPath, receipt); + return { + id: 'live-history-candidate', + owner_requirement: 'REQ-VHLP-009', + status: 'pass', + blocking: false, + summary: 'The governed clone-backed VI History candidate is ready for local iteration.', + current_surface_status: receipt.status, + current_host_platform: process.platform === 'win32' ? 'Windows' : 'Unix', + reason: receipt.reason, + receipt_path: relativeFrom(repoRoot, receiptPath), + recommended_commands: receipt.recommendedCommands + }; +} + +async function runSharedWindowsSurfaceProof(repoRoot, resultsDir) { + const surfaceDir = path.join(resultsDir, 'windows-surface'); + const normalizedReceiptPath = path.join(surfaceDir, 'vi-history-windows-surface.json'); + await fs.rm(surfaceDir, { recursive: true, force: true }); + await ensureDir(surfaceDir); + const surfaceProof = await runSharedWindowsDockerSurfaceProof(repoRoot, resultsDir); + const normalizedReceipt = { + schema: 'vi-history/windows-surface@v1', + generatedAt: new Date().toISOString(), + status: surfaceProof.current_surface_status, + hostPlatform: surfaceProof.current_host_platform, + coordinatorHostPlatform: surfaceProof.coordinator_host_platform ?? (process.platform === 'win32' ? 'Windows' : 'Unix'), + bridgeMode: surfaceProof.bridge_mode ?? (process.platform === 'win32' ? 'native-windows' : 'none'), + reason: surfaceProof.reason, + recommendedCommands: surfaceProof.recommended_commands ?? [], + sourceProbeReceiptPath: surfaceProof.receipt_path ?? null + }; + await writeJson(normalizedReceiptPath, normalizedReceipt); + + return { + id: 'windows-surface', + owner_requirement: 'REQ-VHLP-006', + status: surfaceProof.status, + blocking: surfaceProof.blocking, + summary: surfaceProof.status === 'advisory' + ? `Shared Windows Docker Desktop + NI image surface is ${surfaceProof.current_surface_status}; use the recommended Windows commands before running VI History replay.` + : surfaceProof.status === 'pass' + ? 'Shared Windows Docker Desktop + NI image surface is ready for VI History workflow replay.' + : surfaceProof.summary, + current_surface_status: surfaceProof.current_surface_status, + current_host_platform: surfaceProof.current_host_platform, + coordinator_host_platform: surfaceProof.coordinator_host_platform, + bridge_mode: surfaceProof.bridge_mode, + reason: surfaceProof.reason, + receipt_path: relativeFrom(repoRoot, normalizedReceiptPath), + recommended_commands: normalizedReceipt.recommendedCommands, + details: surfaceProof.details ?? [] + }; +} + +async function runWindowsWorkflowReplayProof(repoRoot, resultsDir) { + const surfaceProof = await runSharedWindowsSurfaceProof(repoRoot, resultsDir); + if (surfaceProof.status !== 'pass') { + return { + id: 'windows-workflow-replay', + owner_requirement: 'REQ-VHLP-001', + status: surfaceProof.status, + blocking: surfaceProof.blocking, + summary: surfaceProof.status === 'advisory' + ? 'VI History Windows workflow replay is gated behind the shared Windows Docker Desktop + NI image surface.' + : 'VI History Windows workflow replay cannot start because the shared Windows surface probe failed.', + current_surface_status: surfaceProof.current_surface_status, + current_host_platform: surfaceProof.current_host_platform, + receipt_path: surfaceProof.receipt_path, + reason: surfaceProof.reason, + recommended_commands: surfaceProof.recommended_commands ?? [] + }; + } + + const receiptPath = path.join('tests', 'results', 'docker-tools-parity', 'workflow-replay', 'vi-history-scenarios-windows-receipt.json'); + const result = process.platform === 'win32' + ? spawnSync('node', [ + 'tools/priority/windows-workflow-replay-lane.mjs', + '--mode', + 'vi-history-scenarios-windows', + '--allow-unavailable' + ], { + cwd: repoRoot, + encoding: 'utf8', + maxBuffer: 20 * 1024 * 1024 + }) + : (() => { + const bridge = detectWindowsHostBridge(repoRoot); + if (bridge.status !== 'reachable') { + throw new Error(bridge.reason); + } + const bridgeSpec = buildWindowsNodeBridgeSpec({ + bridge, + scriptRelativePath: path.join('tools', 'priority', 'windows-workflow-replay-lane.mjs'), + scriptArgs: ['--mode', 'vi-history-scenarios-windows', '--allow-unavailable'] + }); + return runBridgeSpec(bridgeSpec, { cwd: repoRoot, maxBuffer: 64 * 1024 * 1024 }); + })(); + + const base = { + id: 'windows-workflow-replay', + owner_requirement: 'REQ-VHLP-001', + receipt_path: receiptPath + }; + + let receipt; + try { + receipt = await readJson(path.join(repoRoot, receiptPath)); + } catch (error) { + return { + ...base, + status: 'fail', + blocking: true, + summary: 'VI History Windows workflow replay did not emit its governed receipt.', + details: [result.stdout?.trim(), result.stderr?.trim(), error instanceof Error ? error.message : String(error)].filter(Boolean) + }; + } + + const receiptStatus = receipt?.result?.status ?? 'unknown'; + if (receiptStatus === 'passed') { + return { + ...base, + status: 'pass', + blocking: false, + summary: 'VI History Windows workflow replay passed and emitted a workflow-grade receipt.', + current_surface_status: receiptStatus, + current_host_platform: 'Windows', + coordinator_host_platform: process.platform === 'win32' ? 'Windows' : 'Unix', + bridge_mode: process.platform === 'win32' ? 'native-windows' : 'wsl-windows', + recommended_commands: [ + 'npm run priority:workflow:replay:windows:vi-history' + ] + }; + } + if (receiptStatus === 'unavailable') { + return { + ...base, + status: 'advisory', + blocking: false, + summary: 'VI History Windows workflow replay is unavailable from the current host; use the shared Windows Docker Desktop + NI image surface.', + current_surface_status: receiptStatus, + current_host_platform: 'Windows', + coordinator_host_platform: process.platform === 'win32' ? 'Windows' : 'Unix', + bridge_mode: process.platform === 'win32' ? 'native-windows' : 'wsl-windows', + recommended_commands: [ + 'npm run docker:ni:windows:bootstrap', + 'npm run compare:docker:ni:windows:probe', + 'npm run priority:workflow:replay:windows:vi-history' + ] + }; + } + return { + ...base, + status: 'fail', + blocking: true, + summary: 'VI History Windows workflow replay failed on the current packet.', + current_surface_status: receiptStatus, + current_host_platform: 'Windows', + coordinator_host_platform: process.platform === 'win32' ? 'Windows' : 'Unix', + bridge_mode: process.platform === 'win32' ? 'native-windows' : 'wsl-windows', + details: [result.stdout?.trim(), result.stderr?.trim()].filter(Boolean) + }; +} + +export { parseCsv, parseRequirementNumber, determinePhase, rankRequirementGaps, rankProofRegressions, deriveEscalations, selectNextStep, applyAutonomyPolicy, runLiveHistoryCandidateProof }; + +export async function runVIHistoryLocalCi({ + repoRoot = repoRootDefault, + skillRoot = null, + surfacePath = DEFAULT_SURFACE, + policyPath = DEFAULT_POLICY, + resultsDir = DEFAULT_RESULTS_DIR, + outputPath = DEFAULT_REPORT, + summaryPath = DEFAULT_SUMMARY, + nextOutputPath = DEFAULT_NEXT, + nextStepOutputPath = DEFAULT_NEXT_STEP +} = {}) { + const resolvedSkillRoot = await resolveSkillRoot(skillRoot); + const resolved = { + repoRoot, + skillRoot: resolvedSkillRoot, + surfacePath: path.join(repoRoot, surfacePath), + policyPath: path.join(repoRoot, policyPath), + resultsDir: path.join(repoRoot, resultsDir), + outputPath: path.join(repoRoot, outputPath), + summaryPath: path.join(repoRoot, summaryPath), + nextOutputPath: path.join(repoRoot, nextOutputPath), + nextStepOutputPath: path.join(repoRoot, nextStepOutputPath) + }; + + const manifest = await readYaml(resolved.surfacePath); + const policy = await readJson(resolved.policyPath); + const bundleRoot = createRunScopedBundleRoot(resolved.resultsDir); + await ensureDir(resolved.resultsDir); + await materializeAuditSurface(repoRoot, manifest, bundleRoot); + + const evidencePath = path.join(resolved.resultsDir, 'standards-evidence.json'); + const scorePath = path.join(resolved.resultsDir, 'standards-score.json'); + const rtmPath = path.join(repoRoot, 'docs', 'rtm-vi-history-local-proof.csv'); + const rtmRows = parseCsv(await fs.readFile(rtmPath, 'utf8')); + + const evidence = runCommand('python3', [ + path.join(resolvedSkillRoot, 'scripts', 'repo_evidence_scan.py'), + bundleRoot, + '--format', + 'json', + '--profile', + 'quick-triage', + '--max-examples', + '2', + '--max-evidence-per-rule', + '2' + ], { cwd: repoRoot }).stdout; + await fs.writeFile(evidencePath, evidence, 'utf8'); + + const score = runCommand('python3', [ + path.join(resolvedSkillRoot, 'scripts', 'score_assurance.py'), + evidencePath, + '--format', + 'json' + ], { cwd: repoRoot }).stdout; + await fs.writeFile(scorePath, score, 'utf8'); + + const scorePayload = JSON.parse(score); + const worktreeStatus = collectWorktreeStatus(repoRoot); + const proofChecks = [ + await runLiveHistoryCandidateProof(repoRoot, resolved.resultsDir), + await runWindowsWorkflowReplayProof(repoRoot, resolved.resultsDir) + ]; + const proofRegressions = rankProofRegressions(proofChecks, rtmRows); + const rankedRequirementGaps = rankRequirementGaps(rtmRows); + const regressionReqIds = new Set(proofRegressions.map((item) => item.req_id)); + const rankedRequirements = applyAutonomyPolicy([ + ...proofRegressions, + ...rankedRequirementGaps.filter((item) => !regressionReqIds.has(item.req_id)) + ], policy, worktreeStatus.modifiedPaths); + const escalations = deriveEscalations(proofChecks); + const nextRequirement = rankedRequirements[0] ?? null; + const nextStep = selectNextStep(nextRequirement, escalations); + const implementedCount = rtmRows.filter((row) => row.Status === 'Implemented').length; + const gapCount = rankedRequirements.length; + const blockingProofFailures = proofChecks.filter((check) => check.blocking).length; + const advisoryProofChecks = proofChecks.filter((check) => check.status === 'advisory').length; + const overallStatus = gapCount === 0 && blockingProofFailures === 0 + ? (advisoryProofChecks > 0 ? 'pass-with-advisories' : 'pass') + : 'pass-with-actions'; + const overallReason = nextStep?.type === 'escalation' + ? `All tracked VI History local-proof requirements are implemented locally, and the next step is escalation '${nextStep.escalation_id}' because additional preparation is required before the next truthful proof.` + : gapCount === 0 + ? 'All tracked VI History local-proof requirements are currently marked implemented.' + : `The next recommended VI History requirement is ${nextRequirement.req_id} because it is the highest-ranked unresolved local-proof gap.`; + + const report = { + schema_version: '1.0.0', + generated_at: new Date().toISOString(), + repo_root: repoRoot, + audit_surface: { + id: manifest.id, + description: manifest.description, + manifest_path: relativeFrom(repoRoot, resolved.surfacePath), + bundle_root: relativeFrom(repoRoot, bundleRoot), + included_paths: manifest.include + }, + worktree_status: { + branch: worktreeStatus.branch, + modified_paths: worktreeStatus.modifiedPaths, + active_requirement_refs: rankedRequirements.filter((item) => item.active_now).map((item) => item.req_id) + }, + standards_audit: { + status: deriveWeakAreas(scorePayload).length === 0 ? 'pass' : 'pass-with-actions', + evidence_path: relativeFrom(repoRoot, evidencePath), + score_path: relativeFrom(repoRoot, scorePath), + weak_areas: deriveWeakAreas(scorePayload) + }, + requirement_summary: { + total: rtmRows.length, + implemented: implementedCount, + gaps: gapCount + }, + proof_checks: { + blocking_failures: blockingProofFailures, + advisories: advisoryProofChecks, + checks: proofChecks + }, + ranked_requirements: rankedRequirements, + escalations, + next_requirement: nextRequirement, + next_step: nextStep, + overall: { + status: overallStatus, + reason: overallReason + } + }; + + await validateReportSchema(repoRoot, report); + await writeJson(resolved.outputPath, report); + await writeJson(resolved.nextOutputPath, nextRequirement); + await validateNextStepSchema(repoRoot, nextStep); + await writeJson(resolved.nextStepOutputPath, nextStep); + await fs.writeFile(resolved.summaryPath, buildSummary(report), 'utf8'); + + return { report, paths: resolved }; +} + +async function main() { + try { + const options = parseArgs(); + if (options.help) { + printHelp(); + return; + } + const { report } = await runVIHistoryLocalCi(options); + if (options.printNext && report.next_requirement) { + console.log(JSON.stringify(report.next_requirement, null, 2)); + return; + } + if (options.printNextStep && report.next_step) { + console.log(JSON.stringify(report.next_step, null, 2)); + return; + } + console.log(report.overall.reason); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} + +const entryPath = process.argv[1] ? path.resolve(process.argv[1]) : null; +if (entryPath && fileURLToPath(import.meta.url) === entryPath) { + await main(); +} diff --git a/tools/priority/vi-history-local-proof-audit-surface.yaml b/tools/priority/vi-history-local-proof-audit-surface.yaml new file mode 100644 index 000000000..fb3715a15 --- /dev/null +++ b/tools/priority/vi-history-local-proof-audit-surface.yaml @@ -0,0 +1,42 @@ +version: 1 +id: vi-history-local-proof +description: > + Dedicated local CI audit surface for the VI History local proof subsystem. + This bundle captures the workflow replay lane, local refinement and operator + surfaces, workflow-readiness envelope, and the VI History autonomy loop. +include: + - package.json + - docs/knowledgebase/Local-Proof-Autonomy-Program.md + - tools/priority/vi-history-local-ci.mjs + - tools/priority/windows-host-bridge.mjs + - tools/priority/vi-history-live-candidate.json + - tools/priority/comparevi-local-program-ci.mjs + - tools/priority/vi-history-local-proof-audit-surface.yaml + - tools/priority/vi-history-local-proof-autonomy-policy.json + - tools/priority/__tests__/comparevi-local-program-ci.test.mjs + - tools/priority/__tests__/vi-history-local-ci.test.mjs + - tools/priority/__tests__/vi-history-local-proof-contract.test.mjs + - tools/priority/__tests__/windows-host-bridge.test.mjs + - tools/priority/__tests__/windows-workflow-replay-lane.test.mjs + - tools/priority/windows-workflow-replay-lane.mjs + - tools/Invoke-VIHistoryLocalRefinement.ps1 + - tools/Invoke-VIHistoryLocalOperatorSession.ps1 + - tools/Write-VIHistoryWorkflowReadiness.ps1 + - tools/Test-WindowsNI2026q1HostPreflight.ps1 + - tools/Run-NIWindowsContainerCompare.ps1 + - tools/docker/Dockerfile.vi-history-dev + - tests/VIHistoryLocalAcceleration.Tests.ps1 + - tests/VIHistoryLocalOperatorSession.Tests.ps1 + - tests/Write-VIHistoryWorkflowReadiness.Tests.ps1 + - docs/requirements-vi-history-local-proof-srs.md + - docs/rtm-vi-history-local-proof.csv + - docs/testing/vi-history-local-proof-test-plan.md + - docs/knowledgebase/VI-History-Local-Proof.md + - docs/architecture/vi-history-local-proof-control-plane.md + - docs/schemas/windows-workflow-replay-lane-v1.schema.json + - docs/schemas/comparevi-local-program-ci-report-v1.schema.json + - docs/schemas/comparevi-local-program-next-step-v1.schema.json + - docs/schemas/vi-history-live-candidate-v1.schema.json + - docs/schemas/vi-history-live-candidate-readiness-v1.schema.json + - docs/schemas/vi-history-local-ci-report-v1.schema.json + - docs/schemas/vi-history-local-next-step-v1.schema.json diff --git a/tools/priority/vi-history-local-proof-autonomy-policy.json b/tools/priority/vi-history-local-proof-autonomy-policy.json new file mode 100644 index 000000000..30726fe91 --- /dev/null +++ b/tools/priority/vi-history-local-proof-autonomy-policy.json @@ -0,0 +1,37 @@ +{ + "schema_version": "1.0.0", + "phase_guidance": { + "foundation": { + "mode": "local-first", + "preferred_commands": [ + "npm run priority:workflow:replay:windows:vi-history", + "npm run history:local:proof", + "npm run history:local:refine" + ], + "stop_conditions": [ + "stop when the required VI History packet tests pass and the next-step artifact no longer ranks the edited requirement first", + "stop when the next truthful proof surface becomes unavailable from the current host and the loop emits an escalation step" + ], + "escalate_when": [ + "the current host cannot satisfy the Windows replay surface", + "the next truthful proof needs the shared windows-docker-desktop-ni-image surface" + ] + }, + "autonomy": { + "mode": "local-first", + "preferred_commands": [ + "npm run priority:vi-history:local-ci", + "npm run priority:vi-history:next-step" + ], + "stop_conditions": [ + "stop when the next-step artifact becomes an escalation or null", + "stop when the packet says all tracked requirements are implemented and only a host-bound proof surface remains" + ], + "escalate_when": [ + "the governed downstream live-history clone is absent, the target VI is missing, or git history is unavailable", + "the next-step artifact names windows-docker-desktop-ni-image as the required surface", + "another host or agent is needed to satisfy the current proof surface truthfully" + ] + } + } +} diff --git a/tools/priority/windows-docker-shared-surface-audit-surface.yaml b/tools/priority/windows-docker-shared-surface-audit-surface.yaml new file mode 100644 index 000000000..6417d8181 --- /dev/null +++ b/tools/priority/windows-docker-shared-surface-audit-surface.yaml @@ -0,0 +1,29 @@ +version: 1 +id: windows-docker-shared-surface +description: > + Dedicated local CI audit surface for the shared Windows Docker Desktop + NI + image proof packet. This bundle captures the bounded probe, bootstrap, + path-hygiene checks, and the shared-surface autonomy loop. +include: + - package.json + - docs/knowledgebase/Local-Proof-Autonomy-Program.md + - docs/knowledgebase/Windows-Docker-Shared-Surface.md + - docs/requirements-windows-docker-shared-surface-srs.md + - docs/rtm-windows-docker-shared-surface.csv + - docs/testing/windows-docker-shared-surface-test-plan.md + - docs/architecture/windows-docker-shared-surface-control-plane.md + - docs/schemas/windows-docker-shared-surface-local-ci-report-v1.schema.json + - docs/schemas/windows-docker-shared-surface-next-step-v1.schema.json + - tools/Invoke-PesterWindowsContainerSurfaceProbe.ps1 + - tools/Test-WindowsNI2026q1HostPreflight.ps1 + - tools/Run-NIWindowsContainerCompare.ps1 + - tools/priority/windows-host-bridge.mjs + - tools/priority/windows-docker-shared-surface-audit-surface.yaml + - tools/priority/windows-docker-shared-surface-autonomy-policy.json + - tools/priority/windows-docker-shared-surface-local-ci.mjs + - tools/priority/comparevi-local-program-ci.mjs + - tools/priority/__tests__/windows-host-bridge.test.mjs + - tools/priority/__tests__/windows-docker-shared-surface-local-ci.test.mjs + - tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs + - tools/priority/__tests__/comparevi-local-program-ci.test.mjs + - tests/Invoke-PesterWindowsContainerSurfaceProbe.Tests.ps1 diff --git a/tools/priority/windows-docker-shared-surface-autonomy-policy.json b/tools/priority/windows-docker-shared-surface-autonomy-policy.json new file mode 100644 index 000000000..7cb7ed146 --- /dev/null +++ b/tools/priority/windows-docker-shared-surface-autonomy-policy.json @@ -0,0 +1,36 @@ +{ + "schema_version": "1.0.0", + "phase_guidance": { + "foundation": { + "mode": "local-first", + "preferred_commands": [ + "npm run tests:windows-surface:probe", + "npm run docker:ni:windows:bootstrap", + "npm run compare:docker:ni:windows:probe" + ], + "stop_conditions": [ + "stop when the shared-surface packet tests pass and the next-step artifact no longer ranks the edited requirement first", + "stop when the current host and any reachable Windows host bridge still cannot satisfy the shared Windows surface and the loop emits an escalation step" + ], + "escalate_when": [ + "the current host is not Windows and no reachable Windows host bridge is available", + "the shared Windows Docker Desktop + NI image surface must still be prepared on another host after bridge-backed probe or preflight" + ] + }, + "autonomy": { + "mode": "local-first", + "preferred_commands": [ + "npm run priority:windows-surface:local-ci", + "npm run priority:windows-surface:next-step" + ], + "stop_conditions": [ + "stop when the next-step artifact becomes an escalation or null", + "stop when all tracked shared-surface requirements are implemented and only a host-bound proof remains" + ], + "escalate_when": [ + "the next-step artifact names windows-docker-desktop-ni-image as the required surface", + "unsafe synced roots require relocation before Windows proof is run" + ] + } + } +} diff --git a/tools/priority/windows-docker-shared-surface-local-ci.mjs b/tools/priority/windows-docker-shared-surface-local-ci.mjs new file mode 100644 index 000000000..26db789b0 --- /dev/null +++ b/tools/priority/windows-docker-shared-surface-local-ci.mjs @@ -0,0 +1,968 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; +import Ajv2020 from 'ajv/dist/2020.js'; +import addFormats from 'ajv-formats'; +import yaml from 'js-yaml'; + +import { + buildWindowsPowerShellFileBridgeSpec, + detectWindowsHostBridge, + resolveRepoWindowsPath, + runBridgeSpec, +} from './windows-host-bridge.mjs'; + +const repoRootDefault = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); +const defaultSkillRoot = path.join(process.env.HOME ?? '', '.codex', 'skills', 'repo-standards-review'); +const fallbackSkillRoot = '/mnt/c/Users/sveld/.codex/skills/repo-standards-review'; + +const DEFAULT_SURFACE = path.join('tools', 'priority', 'windows-docker-shared-surface-audit-surface.yaml'); +const DEFAULT_POLICY = path.join('tools', 'priority', 'windows-docker-shared-surface-autonomy-policy.json'); +const DEFAULT_RESULTS_DIR = path.join('tests', 'results', '_agent', 'windows-docker-shared-surface', 'local-ci'); +const DEFAULT_REPORT = path.join(DEFAULT_RESULTS_DIR, 'windows-docker-shared-surface-local-ci-report.json'); +const DEFAULT_SUMMARY = path.join(DEFAULT_RESULTS_DIR, 'windows-docker-shared-surface-local-ci-summary.md'); +const DEFAULT_NEXT = path.join(DEFAULT_RESULTS_DIR, 'windows-docker-shared-surface-next-requirement.json'); +const DEFAULT_NEXT_STEP = path.join(DEFAULT_RESULTS_DIR, 'windows-docker-shared-surface-next-step.json'); + +const HELP = [ + 'Usage: node tools/priority/windows-docker-shared-surface-local-ci.mjs [options]', + '', + 'Options:', + ` --repo-root (default: ${repoRootDefault})`, + ` --skill-root (default: ${defaultSkillRoot} or ${fallbackSkillRoot})`, + ` --surface (default: ${DEFAULT_SURFACE})`, + ` --policy (default: ${DEFAULT_POLICY})`, + ` --results-dir (default: ${DEFAULT_RESULTS_DIR})`, + ` --output (default: ${DEFAULT_REPORT})`, + ` --summary-output (default: ${DEFAULT_SUMMARY})`, + ` --next-output (default: ${DEFAULT_NEXT})`, + ` --next-step-output (default: ${DEFAULT_NEXT_STEP})`, + ' --print-next print the selected next requirement to stdout', + ' --print-next-step print the selected next step to stdout', + ' --help, -h' +]; + +function parseArgs(argv = process.argv) { + const options = { + repoRoot: repoRootDefault, + skillRoot: null, + surfacePath: DEFAULT_SURFACE, + policyPath: DEFAULT_POLICY, + resultsDir: DEFAULT_RESULTS_DIR, + outputPath: DEFAULT_REPORT, + summaryPath: DEFAULT_SUMMARY, + nextOutputPath: DEFAULT_NEXT, + nextStepOutputPath: DEFAULT_NEXT_STEP, + printNext: false, + printNextStep: false, + help: false + }; + + const args = argv.slice(2); + for (let i = 0; i < args.length; i += 1) { + const token = args[i]; + const next = args[i + 1]; + if (token === '--help' || token === '-h') { + options.help = true; + continue; + } + if (token === '--print-next') { + options.printNext = true; + continue; + } + if (token === '--print-next-step') { + options.printNextStep = true; + continue; + } + if (['--repo-root', '--skill-root', '--surface', '--policy', '--results-dir', '--output', '--summary-output', '--next-output', '--next-step-output'].includes(token)) { + if (!next || next.startsWith('-')) { + throw new Error(`Missing value for ${token}`); + } + i += 1; + if (token === '--repo-root') options.repoRoot = path.resolve(next); + if (token === '--skill-root') options.skillRoot = path.resolve(next); + if (token === '--surface') options.surfacePath = next; + if (token === '--policy') options.policyPath = next; + if (token === '--results-dir') options.resultsDir = next; + if (token === '--output') options.outputPath = next; + if (token === '--summary-output') options.summaryPath = next; + if (token === '--next-output') options.nextOutputPath = next; + if (token === '--next-step-output') options.nextStepOutputPath = next; + continue; + } + throw new Error(`Unknown option: ${token}`); + } + + return options; +} + +function printHelp() { + for (const line of HELP) console.log(line); +} + +async function fileExists(filePath) { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } +} + +async function resolveSkillRoot(explicitRoot) { + if (explicitRoot) return explicitRoot; + const candidates = []; + if (process.env.CODEX_HOME) { + candidates.push(path.join(process.env.CODEX_HOME, 'skills', 'repo-standards-review')); + } + candidates.push(defaultSkillRoot, fallbackSkillRoot); + for (const candidate of candidates) { + if (await fileExists(candidate)) return candidate; + } + return defaultSkillRoot; +} + +async function readYaml(filePath) { + return yaml.load(await fs.readFile(filePath, 'utf8')); +} + +async function ensureDir(filePath) { + await fs.mkdir(filePath, { recursive: true }); +} + +async function readJson(filePath) { + return JSON.parse(await fs.readFile(filePath, 'utf8')); +} + +function relativeFrom(repoRoot, filePath) { + return path.relative(repoRoot, filePath).split(path.sep).join('/'); +} + +async function materializeAuditSurface(repoRoot, manifest, bundleRoot) { + await ensureDir(bundleRoot); + for (const relativePath of manifest.include) { + const source = path.join(repoRoot, relativePath); + const destination = path.join(bundleRoot, relativePath); + await ensureDir(path.dirname(destination)); + await fs.copyFile(source, destination); + } +} + +function createRunScopedBundleRoot(resultsDir) { + const runId = `run-${Date.now()}-${process.pid}-${Math.random().toString(16).slice(2, 8)}`; + return path.join(resultsDir, 'surface-bundle', runId); +} + +function runCommand(command, args, options = {}) { + const result = spawnSync(command, args, { + cwd: options.cwd, + encoding: 'utf8', + maxBuffer: 20 * 1024 * 1024 + }); + if (result.status !== 0) { + throw new Error([`${command} ${args.join(' ')}`, result.stdout?.trim(), result.stderr?.trim()].filter(Boolean).join('\n')); + } + return result; +} + +function parseCsv(input) { + const rows = []; + let current = ''; + let row = []; + let inQuotes = false; + + for (let i = 0; i < input.length; i += 1) { + const char = input[i]; + const next = input[i + 1]; + if (char === '"') { + if (inQuotes && next === '"') { + current += '"'; + i += 1; + } else { + inQuotes = !inQuotes; + } + continue; + } + if (char === ',' && !inQuotes) { + row.push(current); + current = ''; + continue; + } + if ((char === '\n' || char === '\r') && !inQuotes) { + if (char === '\r' && next === '\n') i += 1; + row.push(current); + current = ''; + if (row.some((value) => value.length > 0)) rows.push(row); + row = []; + continue; + } + current += char; + } + + if (current.length > 0 || row.length > 0) { + row.push(current); + if (row.some((value) => value.length > 0)) rows.push(row); + } + + if (rows.length === 0) return []; + const [header, ...records] = rows; + return records.map((values) => { + const entry = {}; + for (let i = 0; i < header.length; i += 1) { + entry[header[i]] = values[i] ?? ''; + } + return entry; + }); +} + +function parseRequirementNumber(reqId) { + const match = /REQ-WDSS-(\d+)/.exec(reqId ?? ''); + return match ? Number.parseInt(match[1], 10) : Number.POSITIVE_INFINITY; +} + +function determinePhase(reqNumber) { + if (reqNumber <= 3) return 'foundation'; + return 'autonomy'; +} + +function splitField(value) { + return String(value ?? '') + .split(';') + .map((entry) => entry.trim()) + .filter(Boolean); +} + +function isOperationalRef(ref) { + return !String(ref).startsWith('docs/'); +} + +function collectWorktreeStatus(repoRoot) { + const result = runCommand('git', ['status', '--short', '--branch'], { cwd: repoRoot }); + const lines = result.stdout.split(/\r?\n/).filter(Boolean); + const branch = (lines.find((line) => line.startsWith('## ')) ?? '## detached').replace(/^##\s+/, ''); + const modifiedPaths = lines + .filter((line) => !line.startsWith('## ')) + .map((line) => line.slice(3).trim()) + .filter(Boolean); + return { branch, modifiedPaths }; +} + +function deriveWeakAreas(scorePayload) { + return Object.entries(scorePayload.areas ?? {}) + .filter(([, details]) => Number(details.score ?? 0) < 3) + .map(([area]) => area); +} + +function scoreGap(row) { + const reqNumber = parseRequirementNumber(row.ReqID); + const phase = determinePhase(reqNumber); + const priorityWeight = row.Priority === 'High' ? 200 : row.Priority === 'Medium' ? 100 : 50; + const phaseWeight = phase === 'foundation' ? 300 : 180; + const runtimeWeight = /(Invoke-PesterWindowsContainerSurfaceProbe|Test-WindowsNI2026q1HostPreflight|Run-NIWindowsContainerCompare|windows-docker-shared-surface-local-ci)/.test(row.CodeRef) ? 40 : 0; + return 1000 + priorityWeight + phaseWeight + runtimeWeight - reqNumber; +} + +function deriveSuggestedLoop(row) { + const codeRefs = splitField(row.CodeRef); + const actions = []; + actions.push(`Start with ${row.TestID}: tighten shared Windows-surface proof coverage before spending GitHub CI.`); + if (codeRefs.length > 0) { + actions.push(`Edit the primary source targets first: ${codeRefs.slice(0, 3).join(', ')}.`); + } + actions.push('Re-run the shared-surface local CI and next-step entrypoints after the change.'); + return actions; +} + +function buildRequirementEntry(row, overrides = {}) { + const reqNumber = parseRequirementNumber(row.ReqID); + const phase = determinePhase(reqNumber); + return { + req_id: row.ReqID, + priority: row.Priority, + status: overrides.status ?? row.Status, + phase, + score: scoreGap(row) + (overrides.scoreBoost ?? 0), + why_now: overrides.why_now ?? `${row.ReqID} is an unresolved ${row.Priority} ${phase} shared-surface gap.`, + requirement: row.Requirement, + test_id: row.TestID, + test_artifact: row.TestArtifact, + code_refs: splitField(row.CodeRef), + suggested_loop: overrides.suggested_loop ?? deriveSuggestedLoop(row), + proof_check_id: overrides.proof_check_id ?? null + }; +} + +function rankRequirementGaps(rows) { + return rows + .filter((row) => row.Status !== 'Implemented') + .map((row) => buildRequirementEntry(row)) + .sort((a, b) => { + if (b.score !== a.score) return b.score - a.score; + return parseRequirementNumber(a.req_id) - parseRequirementNumber(b.req_id); + }) + .map((entry, index) => ({ rank: index + 1, ...entry })); +} + +function rankProofRegressions(proofChecks, rows) { + const byReqId = new Map(rows.map((row) => [row.ReqID, row])); + return proofChecks + .filter((check) => check.status === 'fail' && check.owner_requirement && byReqId.has(check.owner_requirement)) + .map((check) => { + const row = byReqId.get(check.owner_requirement); + return buildRequirementEntry(row, { + status: 'Regression', + scoreBoost: 5000, + why_now: `${row.ReqID} regressed under the shared-surface proof check '${check.id}': ${check.summary}`, + suggested_loop: [ + `Start with proof check ${check.id}: ${check.summary}`, + ...deriveSuggestedLoop(row) + ], + proof_check_id: check.id + }); + }) + .sort((a, b) => { + if (b.score !== a.score) return b.score - a.score; + return parseRequirementNumber(a.req_id) - parseRequirementNumber(b.req_id); + }); +} + +function applyAutonomyPolicy(rankedRequirements, policy, modifiedPaths) { + const modifiedSet = new Set(modifiedPaths); + return rankedRequirements + .map((item) => { + const phasePolicy = policy.phase_guidance?.[item.phase] ?? {}; + const activeNow = item.code_refs.some((ref) => isOperationalRef(ref) && modifiedSet.has(ref)); + return { + ...item, + active_now: activeNow, + mode: phasePolicy.mode ?? 'local-first', + preferred_commands: phasePolicy.preferred_commands ?? [], + stop_conditions: phasePolicy.stop_conditions ?? [], + escalate_when: phasePolicy.escalate_when ?? [] + }; + }) + .sort((a, b) => { + if (a.active_now !== b.active_now) return a.active_now ? -1 : 1; + if (b.score !== a.score) return b.score - a.score; + return parseRequirementNumber(a.req_id) - parseRequirementNumber(b.req_id); + }) + .map((entry, index) => ({ ...entry, rank: index + 1 })); +} + +function portablePath(pathValue) { + return String(pathValue ?? '').replace(/\\/g, '/'); +} + +function getManagedRootRisks(pathValue) { + const normalized = portablePath(pathValue); + const risks = []; + if (/(^|\/)OneDrive(?:\/|$|[\s-])/i.test(normalized)) { + risks.push({ + id: 'onedrive-managed-root', + message: 'Path appears to live under a OneDrive-managed root.' + }); + } + return risks; +} + +async function writeJson(filePath, payload) { + await ensureDir(path.dirname(filePath)); + await fs.writeFile(filePath, JSON.stringify(payload, null, 2), 'utf8'); +} + +async function validateJsonAgainstSchema(repoRoot, schemaRelativePath, payload, label) { + const schemaPath = path.join(repoRoot, schemaRelativePath); + const schema = await readJson(schemaPath); + const ajv = new Ajv2020({ allErrors: true, strict: false }); + addFormats(ajv); + const validate = ajv.compile(schema); + const ok = validate(payload); + if (!ok) { + const details = (validate.errors ?? []).map((entry) => `${entry.instancePath || '/'} ${entry.message}`).join('; '); + throw new Error(`${label} schema validation failed: ${details}`); + } +} + +async function validateReportSchema(repoRoot, report) { + await validateJsonAgainstSchema(repoRoot, path.join('docs', 'schemas', 'windows-docker-shared-surface-local-ci-report-v1.schema.json'), report, 'Windows Docker shared surface local CI report'); +} + +async function validateNextStepSchema(repoRoot, nextStep) { + await validateJsonAgainstSchema(repoRoot, path.join('docs', 'schemas', 'windows-docker-shared-surface-next-step-v1.schema.json'), nextStep, 'Windows Docker shared surface next-step'); +} + +async function runPathHygieneProof(repoRoot, resultsDir) { + const proofDir = path.join(resultsDir, 'path-hygiene'); + const receiptPath = path.join(proofDir, 'windows-docker-shared-surface-path-hygiene.json'); + await fs.rm(proofDir, { recursive: true, force: true }); + await ensureDir(proofDir); + + const evaluatedPaths = [ + { id: 'repo-root', path: repoRoot }, + { id: 'results-root', path: resultsDir } + ]; + const findings = evaluatedPaths.flatMap((entry) => + getManagedRootRisks(entry.path).map((risk) => ({ + scope: entry.id, + path: portablePath(entry.path), + ...risk + })) + ); + + const receipt = { + schema: 'comparevi/windows-docker-shared-surface-path-hygiene@v1', + generatedAt: new Date().toISOString(), + status: findings.length > 0 ? 'unsafe-synced-root' : 'safe', + findings, + evaluatedPaths, + recommendedCommands: findings.length > 0 + ? [ + 'Move or clone the repo to a non-OneDrive local path before running Windows Docker proof.', + 'Set a safe local results root before running the shared Windows surface loop.', + 'npm run priority:windows-surface:local-ci' + ] + : [] + }; + await writeJson(receiptPath, receipt); + + if (findings.length > 0) { + return { + id: 'path-hygiene', + owner_requirement: 'REQ-WDSS-003', + status: 'advisory', + blocking: false, + summary: 'The shared Windows surface is currently rooted in a synchronized or externally managed path.', + current_surface_status: receipt.status, + current_host_platform: process.platform === 'win32' ? 'Windows' : 'Unix', + reason: 'OneDrive-like managed roots can mutate artifacts during live Windows proof.', + receipt_path: relativeFrom(repoRoot, receiptPath), + recommended_commands: receipt.recommendedCommands + }; + } + + return { + id: 'path-hygiene', + owner_requirement: 'REQ-WDSS-003', + status: 'pass', + blocking: false, + summary: 'The shared Windows surface packet is rooted in paths that do not appear OneDrive-managed.', + current_surface_status: receipt.status, + current_host_platform: process.platform === 'win32' ? 'Windows' : 'Unix', + reason: 'No OneDrive-like managed roots were detected for the shared-surface loop.', + receipt_path: relativeFrom(repoRoot, receiptPath), + recommended_commands: [] + }; +} + +async function runWindowsSurfaceProof(repoRoot, resultsDir) { + const proofDir = path.join(resultsDir, 'windows-surface'); + const receiptPath = path.join(proofDir, 'pester-windows-container-surface.json'); + await fs.rm(proofDir, { recursive: true, force: true }); + await ensureDir(proofDir); + const recommendedCommands = [ + 'npm run docker:ni:windows:bootstrap', + 'npm run compare:docker:ni:windows:probe', + 'npm run compare:docker:ni:windows' + ]; + const bridge = detectWindowsHostBridge(repoRoot); + + let probeResult; + if (process.platform === 'win32') { + probeResult = spawnSync('pwsh', [ + '-NoLogo', + '-NoProfile', + '-File', + 'tools/Invoke-PesterWindowsContainerSurfaceProbe.ps1', + '-ResultsDir', + proofDir + ], { + cwd: repoRoot, + encoding: 'utf8', + maxBuffer: 20 * 1024 * 1024 + }); + } else if (bridge.status === 'reachable') { + const proofDirWindows = resolveRepoWindowsPath(proofDir); + const bridgeSpec = buildWindowsPowerShellFileBridgeSpec({ + bridge, + scriptRelativePath: path.join('tools', 'Invoke-PesterWindowsContainerSurfaceProbe.ps1'), + scriptArgs: ['-ResultsDir', proofDirWindows] + }); + probeResult = runBridgeSpec(bridgeSpec, { cwd: repoRoot }); + } else { + return { + id: 'windows-surface', + owner_requirement: 'REQ-WDSS-001', + status: 'advisory', + blocking: false, + summary: 'The shared Windows Docker surface is unavailable because no reachable Windows host bridge is available from the current coordinator.', + current_surface_status: 'windows-host-bridge-unavailable', + current_host_platform: bridge.current_host_platform ?? (process.platform === 'win32' ? 'Windows' : 'Unix'), + coordinator_host_platform: bridge.coordinator_host_platform ?? (process.platform === 'win32' ? 'Windows' : 'Unix'), + bridge_mode: bridge.bridge_mode ?? 'none', + reason: bridge.reason, + receipt_path: relativeFrom(repoRoot, receiptPath), + recommended_commands: recommendedCommands + }; + } + + if (probeResult.status !== 0) { + return { + id: 'windows-surface', + owner_requirement: 'REQ-WDSS-001', + status: 'fail', + blocking: true, + summary: 'The shared Windows surface probe failed before it could emit a bounded receipt.', + current_surface_status: 'probe-failed', + current_host_platform: bridge.current_host_platform ?? (process.platform === 'win32' ? 'Windows' : 'Unix'), + coordinator_host_platform: bridge.coordinator_host_platform ?? (process.platform === 'win32' ? 'Windows' : 'Unix'), + bridge_mode: bridge.bridge_mode ?? (process.platform === 'win32' ? 'native-windows' : 'none'), + reason: 'Invoke-PesterWindowsContainerSurfaceProbe.ps1 failed.', + receipt_path: relativeFrom(repoRoot, receiptPath), + recommended_commands: recommendedCommands, + details: [probeResult.stdout?.trim(), probeResult.stderr?.trim()].filter(Boolean) + }; + } + + let receipt; + try { + receipt = await readJson(receiptPath); + } catch (error) { + return { + id: 'windows-surface', + owner_requirement: 'REQ-WDSS-001', + status: 'fail', + blocking: true, + summary: 'The shared Windows surface probe completed without writing its receipt.', + current_surface_status: 'missing-receipt', + current_host_platform: bridge.current_host_platform ?? (process.platform === 'win32' ? 'Windows' : 'Unix'), + coordinator_host_platform: bridge.coordinator_host_platform ?? (process.platform === 'win32' ? 'Windows' : 'Unix'), + bridge_mode: bridge.bridge_mode ?? (process.platform === 'win32' ? 'native-windows' : 'none'), + reason: error instanceof Error ? error.message : String(error), + receipt_path: relativeFrom(repoRoot, receiptPath), + recommended_commands: ['npm run tests:windows-surface:probe'] + }; + } + + const advisory = receipt.status !== 'ready'; + return { + id: 'windows-surface', + owner_requirement: 'REQ-WDSS-001', + status: advisory ? 'advisory' : 'pass', + blocking: false, + summary: advisory + ? `The shared Windows Docker surface is ${receipt.status}; additional host preparation is required before live proof.` + : 'The shared Windows Docker surface is ready for live proof.', + current_surface_status: receipt.status, + current_host_platform: receipt.hostPlatform ?? (process.platform === 'win32' ? 'Windows' : 'Unix'), + coordinator_host_platform: bridge.coordinator_host_platform ?? (process.platform === 'win32' ? 'Windows' : 'Unix'), + bridge_mode: bridge.bridge_mode ?? (process.platform === 'win32' ? 'native-windows' : 'none'), + reason: receipt.reason ?? 'unknown', + receipt_path: relativeFrom(repoRoot, receiptPath), + recommended_commands: receipt.recommendedCommands ?? recommendedCommands + }; +} + +async function runWindowsHostPreflightProof(repoRoot, resultsDir) { + const proofDir = path.join(resultsDir, 'host-preflight'); + const receiptPath = path.join(proofDir, 'windows-ni-2026q1-host-preflight.json'); + await fs.rm(proofDir, { recursive: true, force: true }); + await ensureDir(proofDir); + const recommendedCommands = [ + 'npm run docker:ni:windows:bootstrap', + 'npm run compare:docker:ni:windows:probe' + ]; + const bridge = detectWindowsHostBridge(repoRoot); + + let preflightResult; + if (process.platform === 'win32') { + preflightResult = spawnSync('pwsh', [ + '-NoLogo', + '-NoProfile', + '-File', + 'tools/Test-WindowsNI2026q1HostPreflight.ps1', + '-ResultsDir', + proofDir, + '-AllowUnavailable' + ], { + cwd: repoRoot, + encoding: 'utf8', + maxBuffer: 20 * 1024 * 1024 + }); + } else if (bridge.status === 'reachable') { + const proofDirWindows = resolveRepoWindowsPath(proofDir); + const bridgeSpec = buildWindowsPowerShellFileBridgeSpec({ + bridge, + scriptRelativePath: path.join('tools', 'Test-WindowsNI2026q1HostPreflight.ps1'), + scriptArgs: ['-ResultsDir', proofDirWindows, '-AllowUnavailable'] + }); + preflightResult = runBridgeSpec(bridgeSpec, { cwd: repoRoot }); + } else { + return { + id: 'windows-host-preflight', + owner_requirement: 'REQ-WDSS-002', + status: 'advisory', + blocking: false, + summary: 'Deterministic Windows host preflight is unavailable because no reachable Windows host bridge is available from the current coordinator.', + current_surface_status: 'windows-host-bridge-unavailable', + current_host_platform: bridge.current_host_platform ?? (process.platform === 'win32' ? 'Windows' : 'Unix'), + coordinator_host_platform: bridge.coordinator_host_platform ?? (process.platform === 'win32' ? 'Windows' : 'Unix'), + bridge_mode: bridge.bridge_mode ?? 'none', + reason: bridge.reason, + receipt_path: relativeFrom(repoRoot, receiptPath), + recommended_commands: recommendedCommands + }; + } + + if (preflightResult.status !== 0) { + return { + id: 'windows-host-preflight', + owner_requirement: 'REQ-WDSS-002', + status: 'fail', + blocking: true, + summary: 'The deterministic Windows host preflight failed before it could emit its bounded receipt.', + current_surface_status: 'preflight-failed', + current_host_platform: bridge.current_host_platform ?? (process.platform === 'win32' ? 'Windows' : 'Unix'), + coordinator_host_platform: bridge.coordinator_host_platform ?? (process.platform === 'win32' ? 'Windows' : 'Unix'), + bridge_mode: bridge.bridge_mode ?? (process.platform === 'win32' ? 'native-windows' : 'none'), + reason: 'Test-WindowsNI2026q1HostPreflight.ps1 failed.', + receipt_path: relativeFrom(repoRoot, receiptPath), + recommended_commands: recommendedCommands, + details: [preflightResult.stdout?.trim(), preflightResult.stderr?.trim()].filter(Boolean) + }; + } + + let receipt; + try { + receipt = await readJson(receiptPath); + } catch (error) { + return { + id: 'windows-host-preflight', + owner_requirement: 'REQ-WDSS-002', + status: 'fail', + blocking: true, + summary: 'Deterministic Windows host preflight completed without emitting its receipt.', + current_surface_status: 'missing-receipt', + current_host_platform: bridge.current_host_platform ?? (process.platform === 'win32' ? 'Windows' : 'Unix'), + coordinator_host_platform: bridge.coordinator_host_platform ?? (process.platform === 'win32' ? 'Windows' : 'Unix'), + bridge_mode: bridge.bridge_mode ?? (process.platform === 'win32' ? 'native-windows' : 'none'), + reason: error instanceof Error ? error.message : String(error), + receipt_path: relativeFrom(repoRoot, receiptPath), + recommended_commands: recommendedCommands + }; + } + + const advisory = receipt.status !== 'ready'; + return { + id: 'windows-host-preflight', + owner_requirement: 'REQ-WDSS-002', + status: advisory ? 'advisory' : 'pass', + blocking: false, + summary: advisory + ? `Deterministic Windows host preflight is ${receipt.status}; additional host preparation is required before live proof.` + : 'Deterministic Windows host preflight is ready for shared local proof.', + current_surface_status: receipt.status, + current_host_platform: 'Windows', + coordinator_host_platform: bridge.coordinator_host_platform ?? (process.platform === 'win32' ? 'Windows' : 'Unix'), + bridge_mode: bridge.bridge_mode ?? (process.platform === 'win32' ? 'native-windows' : 'none'), + reason: receipt.failureMessage || receipt.failureClass || 'ready', + receipt_path: relativeFrom(repoRoot, receiptPath), + recommended_commands: recommendedCommands + }; +} + +function deriveEscalations(proofChecks) { + return proofChecks + .filter((check) => check.status === 'advisory') + .map((check) => { + if (check.id === 'path-hygiene') { + return { + type: 'escalation', + escalation_id: 'local-safe-root', + governing_requirement: 'REQ-WDSS-003', + blocked_requirement: 'REQ-WDSS-003', + proof_check_id: check.id, + status: 'required', + mode: 'escalate', + why_now: 'The shared Windows surface should not run from a OneDrive-like or externally managed root.', + reason: check.reason, + required_surface: 'local-safe-root', + current_surface_status: check.current_surface_status ?? 'unknown', + current_host_platform: check.current_host_platform ?? 'unknown', + receipt_path: check.receipt_path ?? null, + suggested_loop: [ + 'Relocate the repo and results roots to a non-synchronized local path.', + 'Re-run the shared-surface local CI before choosing Windows live proof.', + 'Do not spend GitHub CI on a Windows surface rooted in a managed sync path.' + ], + recommended_commands: check.recommended_commands ?? [], + stop_conditions: [ + 'Stop once the path-hygiene receipt reaches status=safe.', + 'Stop if a different synchronized-root risk appears that requires broader path governance.' + ] + }; + } + + return { + type: 'escalation', + escalation_id: 'windows-docker-desktop-ni-image', + governing_requirement: 'REQ-WDSS-005', + blocked_requirement: 'REQ-WDSS-001', + proof_check_id: check.id, + status: 'required', + mode: 'escalate', + why_now: 'The next truthful shared-surface proof is unavailable from the current host.', + reason: check.current_host_platform === 'Unix' + ? 'Current host is not Windows, so the shared Windows Docker Desktop + NI image surface cannot be exercised here.' + : `Shared Windows surface reported ${check.current_surface_status}; prepare the host before another live proof.`, + required_surface: 'windows-docker-desktop-ni-image', + current_surface_status: check.current_surface_status ?? 'unknown', + current_host_platform: check.current_host_platform ?? 'unknown', + receipt_path: check.receipt_path ?? null, + suggested_loop: [ + 'Move to a Windows host with Docker Desktop configured for Windows containers.', + 'Bootstrap or verify the pinned NI Windows image before running a live proof.', + 'Re-run the shared Windows-surface loop before choosing a hosted rerun.' + ], + recommended_commands: check.recommended_commands ?? [], + stop_conditions: [ + 'Stop once the shared Windows surface probe reaches status=ready.', + 'Stop if the probe exposes a new blocking host or image defect.' + ] + }; + }); +} + +function selectNextStep(nextRequirement, escalations) { + if (nextRequirement) return { type: 'requirement', ...nextRequirement }; + return escalations[0] ?? null; +} + +function buildSummary(report) { + const lines = [ + '# Windows Docker Shared Surface Local CI', + '', + `- Overall: ${report.overall.status}`, + `- Reason: ${report.overall.reason}`, + `- Standards audit: ${report.standards_audit.status}`, + `- Requirements: ${report.requirement_summary.implemented}/${report.requirement_summary.total} implemented`, + `- Gaps: ${report.requirement_summary.gaps}`, + `- Proof checks: ${report.proof_checks.blocking_failures} blocking, ${report.proof_checks.advisories} advisory`, + '' + ]; + + if (report.next_step?.type === 'requirement' && report.next_requirement) { + lines.push('## Next Step', ''); + lines.push('- Type: requirement'); + lines.push(`- ${report.next_requirement.req_id}`); + lines.push(`- Why now: ${report.next_requirement.why_now}`); + lines.push(`- Test: ${report.next_requirement.test_id}`); + lines.push(''); + } + + if (report.next_step?.type === 'escalation') { + lines.push('## Next Step', ''); + lines.push('- Type: escalation'); + lines.push(`- Escalation: ${report.next_step.escalation_id}`); + lines.push(`- Governing requirement: ${report.next_step.governing_requirement}`); + lines.push(`- Blocked requirement: ${report.next_step.blocked_requirement}`); + lines.push(`- Required surface: ${report.next_step.required_surface}`); + lines.push(`- Reason: ${report.next_step.reason}`); + lines.push(''); + } + + if (report.proof_checks.checks.length > 0) { + lines.push('## Proof Checks', ''); + for (const check of report.proof_checks.checks) { + lines.push(`- ${check.id}: ${check.status}`); + lines.push(` ${check.summary}`); + } + lines.push(''); + } + + return `${lines.join('\n')}\n`; +} + +export { + applyAutonomyPolicy, + deriveEscalations, + determinePhase, + parseCsv, + parseRequirementNumber, + rankProofRegressions, + rankRequirementGaps, + runPathHygieneProof, + runWindowsHostPreflightProof, + runWindowsSurfaceProof, + selectNextStep +}; + +export async function runWindowsDockerSharedSurfaceLocalCi({ + repoRoot = repoRootDefault, + skillRoot = null, + surfacePath = DEFAULT_SURFACE, + policyPath = DEFAULT_POLICY, + resultsDir = DEFAULT_RESULTS_DIR, + outputPath = DEFAULT_REPORT, + summaryPath = DEFAULT_SUMMARY, + nextOutputPath = DEFAULT_NEXT, + nextStepOutputPath = DEFAULT_NEXT_STEP +} = {}) { + const resolvedSkillRoot = await resolveSkillRoot(skillRoot); + const resolved = { + repoRoot, + skillRoot: resolvedSkillRoot, + surfacePath: path.join(repoRoot, surfacePath), + policyPath: path.join(repoRoot, policyPath), + resultsDir: path.join(repoRoot, resultsDir), + outputPath: path.join(repoRoot, outputPath), + summaryPath: path.join(repoRoot, summaryPath), + nextOutputPath: path.join(repoRoot, nextOutputPath), + nextStepOutputPath: path.join(repoRoot, nextStepOutputPath) + }; + + const manifest = await readYaml(resolved.surfacePath); + const policy = await readJson(resolved.policyPath); + const bundleRoot = createRunScopedBundleRoot(resolved.resultsDir); + await ensureDir(resolved.resultsDir); + await materializeAuditSurface(repoRoot, manifest, bundleRoot); + + const evidencePath = path.join(resolved.resultsDir, 'standards-evidence.json'); + const scorePath = path.join(resolved.resultsDir, 'standards-score.json'); + const rtmPath = path.join(repoRoot, 'docs', 'rtm-windows-docker-shared-surface.csv'); + const rtmRows = parseCsv(await fs.readFile(rtmPath, 'utf8')); + + const evidence = runCommand('python3', [ + path.join(resolvedSkillRoot, 'scripts', 'repo_evidence_scan.py'), + bundleRoot, + '--format', + 'json', + '--profile', + 'quick-triage', + '--max-examples', + '2', + '--max-evidence-per-rule', + '2' + ], { cwd: repoRoot }).stdout; + await fs.writeFile(evidencePath, evidence, 'utf8'); + + const score = runCommand('python3', [ + path.join(resolvedSkillRoot, 'scripts', 'score_assurance.py'), + evidencePath, + '--format', + 'json' + ], { cwd: repoRoot }).stdout; + await fs.writeFile(scorePath, score, 'utf8'); + + const scorePayload = JSON.parse(score); + const worktreeStatus = collectWorktreeStatus(repoRoot); + const proofChecks = [ + await runPathHygieneProof(repoRoot, resolved.resultsDir), + await runWindowsSurfaceProof(repoRoot, resolved.resultsDir), + await runWindowsHostPreflightProof(repoRoot, resolved.resultsDir) + ]; + const proofRegressions = rankProofRegressions(proofChecks, rtmRows); + const rankedRequirementGaps = rankRequirementGaps(rtmRows); + const regressionReqIds = new Set(proofRegressions.map((item) => item.req_id)); + const rankedRequirements = applyAutonomyPolicy([ + ...proofRegressions, + ...rankedRequirementGaps.filter((item) => !regressionReqIds.has(item.req_id)) + ], policy, worktreeStatus.modifiedPaths); + const escalations = deriveEscalations(proofChecks); + const nextRequirement = rankedRequirements[0] ?? null; + const nextStep = selectNextStep(nextRequirement, escalations); + const implementedCount = rtmRows.filter((row) => row.Status === 'Implemented').length; + const gapCount = rankedRequirements.length; + const blockingProofFailures = proofChecks.filter((check) => check.blocking).length; + const advisoryProofChecks = proofChecks.filter((check) => check.status === 'advisory').length; + const overallStatus = gapCount === 0 && blockingProofFailures === 0 + ? (advisoryProofChecks > 0 ? 'pass-with-advisories' : 'pass') + : 'pass-with-actions'; + const overallReason = nextStep?.type === 'escalation' + ? `All tracked Windows shared-surface requirements are implemented locally, and the next step is escalation '${nextStep.escalation_id}' because additional preparation is required before the next truthful proof.` + : gapCount === 0 + ? 'All tracked Windows shared-surface requirements are currently marked implemented.' + : `The next recommended Windows shared-surface requirement is ${nextRequirement.req_id} because it is the highest-ranked unresolved shared-surface gap.`; + + const report = { + schema_version: '1.0.0', + generated_at: new Date().toISOString(), + repo_root: repoRoot, + audit_surface: { + id: manifest.id, + description: manifest.description, + manifest_path: relativeFrom(repoRoot, resolved.surfacePath), + bundle_root: relativeFrom(repoRoot, bundleRoot), + included_paths: manifest.include + }, + worktree_status: { + branch: worktreeStatus.branch, + modified_paths: worktreeStatus.modifiedPaths, + active_requirement_refs: rankedRequirements.filter((item) => item.active_now).map((item) => item.req_id) + }, + standards_audit: { + status: deriveWeakAreas(scorePayload).length === 0 ? 'pass' : 'pass-with-actions', + evidence_path: relativeFrom(repoRoot, evidencePath), + score_path: relativeFrom(repoRoot, scorePath), + weak_areas: deriveWeakAreas(scorePayload) + }, + requirement_summary: { + total: rtmRows.length, + implemented: implementedCount, + gaps: gapCount + }, + proof_checks: { + blocking_failures: blockingProofFailures, + advisories: advisoryProofChecks, + checks: proofChecks + }, + ranked_requirements: rankedRequirements, + escalations, + next_requirement: nextRequirement, + next_step: nextStep, + overall: { + status: overallStatus, + reason: overallReason + } + }; + + await validateReportSchema(repoRoot, report); + await writeJson(resolved.outputPath, report); + await writeJson(resolved.nextOutputPath, nextRequirement); + await validateNextStepSchema(repoRoot, nextStep); + await writeJson(resolved.nextStepOutputPath, nextStep); + await fs.writeFile(resolved.summaryPath, buildSummary(report), 'utf8'); + + return { report, paths: resolved }; +} + +async function main() { + try { + const options = parseArgs(); + if (options.help) { + printHelp(); + return; + } + const { report } = await runWindowsDockerSharedSurfaceLocalCi(options); + if (options.printNext && report.next_requirement) { + console.log(JSON.stringify(report.next_requirement, null, 2)); + return; + } + if (options.printNextStep && report.next_step) { + console.log(JSON.stringify(report.next_step, null, 2)); + return; + } + console.log(report.overall.reason); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} + +const entryPath = process.argv[1] ? path.resolve(process.argv[1]) : null; +if (entryPath && fileURLToPath(import.meta.url) === entryPath) { + await main(); +} diff --git a/tools/priority/windows-host-bridge.mjs b/tools/priority/windows-host-bridge.mjs new file mode 100644 index 000000000..eb196f4aa --- /dev/null +++ b/tools/priority/windows-host-bridge.mjs @@ -0,0 +1,240 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; + +export const DEFAULT_WINDOWS_PWSH_CANDIDATES = Object.freeze([ + '/mnt/c/Program Files/PowerShell/7/pwsh.exe', + '/mnt/c/Windows/System32/WindowsPowerShell/v1.0/powershell.exe', +]); + +const DEFAULT_MAX_BUFFER_BYTES = 64 * 1024 * 1024; + +function runProcess(command, args, options = {}) { + const result = spawnSync(command, args, { + cwd: options.cwd ?? process.cwd(), + encoding: 'utf8', + env: options.env ?? process.env, + maxBuffer: options.maxBuffer ?? DEFAULT_MAX_BUFFER_BYTES, + }); + return { + status: typeof result.status === 'number' ? result.status : null, + stdout: result.stdout ?? '', + stderr: result.stderr ?? '', + error: result.error ?? null, + }; +} + +function trimText(value) { + return String(value ?? '').trim(); +} + +function quotePowerShellLiteral(value) { + return `'${String(value ?? '').replace(/'/g, "''")}'`; +} + +function splitRelativeWindowsPath(value) { + return String(value ?? '') + .split(/[\\/]+/) + .filter(Boolean); +} + +export function resolveRepoWindowsPath(repoRoot, runProcessFn = runProcess) { + const result = runProcessFn('wslpath', ['-w', repoRoot], { cwd: repoRoot }); + if (result.status !== 0) { + throw new Error( + trimText(result.stderr) || + trimText(result.stdout) || + 'wslpath failed to translate the repository root to a Windows path.', + ); + } + const translated = trimText(result.stdout); + if (!translated) { + throw new Error('wslpath did not return a Windows path for the repository root.'); + } + return translated; +} + +export function resolveWindowsPwshPath( + runProcessFn = runProcess, + pathExists = existsSync, + candidates = DEFAULT_WINDOWS_PWSH_CANDIDATES, +) { + for (const candidate of candidates) { + if (!pathExists(candidate)) continue; + const probe = runProcessFn(candidate, ['-NoLogo', '-NoProfile', '-Command', '$PSVersionTable.PSVersion.ToString()']); + if (probe.status === 0) { + return candidate; + } + } + + const fallback = runProcessFn('pwsh.exe', ['-NoLogo', '-NoProfile', '-Command', '$PSVersionTable.PSVersion.ToString()']); + if (fallback.status === 0) { + return 'pwsh.exe'; + } + + return null; +} + +export function resolveWindowsNodePath(windowsPwshPath, runProcessFn = runProcess) { + const result = runProcessFn(windowsPwshPath, [ + '-NoLogo', + '-NoProfile', + '-ExecutionPolicy', + 'Bypass', + '-Command', + '(Get-Command node.exe -ErrorAction Stop).Source', + ]); + if (result.status !== 0) { + throw new Error( + trimText(result.stderr) || + trimText(result.stdout) || + 'Unable to resolve node.exe on the reachable Windows host.', + ); + } + const nodePath = trimText(result.stdout); + if (!nodePath) { + throw new Error('The reachable Windows host did not return a node.exe path.'); + } + return nodePath; +} + +export function detectWindowsHostBridge( + repoRoot, + { + platform = process.platform, + runProcessFn = runProcess, + pathExists = existsSync, + } = {}, +) { + const coordinatorHostPlatform = platform === 'win32' ? 'Windows' : 'Unix'; + if (platform === 'win32') { + return { + status: 'native', + bridge_mode: 'native-windows', + coordinator_host_platform: coordinatorHostPlatform, + current_host_platform: 'Windows', + repo_root_windows: repoRoot, + windows_pwsh_path: 'pwsh', + windows_node_path: process.execPath, + reason: 'Current coordinator is already running on Windows.', + }; + } + + let repoRootWindows; + try { + repoRootWindows = resolveRepoWindowsPath(repoRoot, runProcessFn); + } catch (error) { + return { + status: 'unavailable', + bridge_mode: 'wsl-windows', + coordinator_host_platform: coordinatorHostPlatform, + current_host_platform: coordinatorHostPlatform, + reason: error instanceof Error ? error.message : String(error), + }; + } + + const windowsPwshPath = resolveWindowsPwshPath(runProcessFn, pathExists); + if (!windowsPwshPath) { + return { + status: 'unavailable', + bridge_mode: 'wsl-windows', + coordinator_host_platform: coordinatorHostPlatform, + current_host_platform: coordinatorHostPlatform, + repo_root_windows: repoRootWindows, + reason: 'No reachable Windows PowerShell executable is available from the current coordinator.', + }; + } + + let windowsNodePath = null; + try { + windowsNodePath = resolveWindowsNodePath(windowsPwshPath, runProcessFn); + } catch { + windowsNodePath = null; + } + + return { + status: 'reachable', + bridge_mode: 'wsl-windows', + coordinator_host_platform: coordinatorHostPlatform, + current_host_platform: 'Windows', + repo_root_windows: repoRootWindows, + windows_pwsh_path: windowsPwshPath, + windows_node_path: windowsNodePath, + reason: 'A reachable Windows host is available through a local bridge from the current coordinator.', + }; +} + +export function buildWindowsPath(rootPathWindows, relativePath) { + return path.win32.join(rootPathWindows, ...splitRelativeWindowsPath(relativePath)); +} + +function serializePowerShellArgs(args) { + return args.map((value) => quotePowerShellLiteral(value)).join(' '); +} + +function isPowerShellParameterToken(value) { + return /^-[A-Za-z][A-Za-z0-9-]*$/.test(String(value ?? '')); +} + +function serializePowerShellFileArgs(args) { + return args + .map((value) => (isPowerShellParameterToken(value) ? String(value) : quotePowerShellLiteral(value))) + .join(' '); +} + +export function buildWindowsPowerShellFileBridgeSpec({ + bridge, + scriptRelativePath, + scriptArgs = [], +}) { + const scriptPathWindows = buildWindowsPath(bridge.repo_root_windows, scriptRelativePath); + const commandText = [ + `Set-Location -LiteralPath ${quotePowerShellLiteral(bridge.repo_root_windows)}`, + `& ${quotePowerShellLiteral(scriptPathWindows)}${scriptArgs.length > 0 ? ` ${serializePowerShellFileArgs(scriptArgs)}` : ''}`, + ].join('; '); + return { + command: bridge.windows_pwsh_path, + args: [ + '-NoLogo', + '-NoProfile', + '-ExecutionPolicy', + 'Bypass', + '-Command', + commandText, + ], + script_path_windows: scriptPathWindows, + }; +} + +export function buildWindowsNodeBridgeSpec({ + bridge, + scriptRelativePath, + scriptArgs = [], +}) { + if (!bridge.windows_node_path) { + throw new Error('A reachable Windows node.exe path is required before building a Windows Node bridge spec.'); + } + const scriptPathWindows = buildWindowsPath(bridge.repo_root_windows, scriptRelativePath); + const commandText = [ + `Set-Location -LiteralPath ${quotePowerShellLiteral(bridge.repo_root_windows)}`, + `& ${quotePowerShellLiteral(bridge.windows_node_path)} ${quotePowerShellLiteral(scriptPathWindows)}${scriptArgs.length > 0 ? ` ${serializePowerShellArgs(scriptArgs)}` : ''}`, + ].join('; '); + return { + command: bridge.windows_pwsh_path, + args: [ + '-NoLogo', + '-NoProfile', + '-ExecutionPolicy', + 'Bypass', + '-Command', + commandText, + ], + script_path_windows: scriptPathWindows, + node_path_windows: bridge.windows_node_path, + }; +} + +export function runBridgeSpec(spec, options = {}) { + return runProcess(spec.command, spec.args, options); +} From 4d1aa1c8f802470147a364a17e887d39cdef4e4b Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Tue, 31 Mar 2026 23:17:02 -0700 Subject: [PATCH 35/44] ci: promote Windows NI proof authority and local proof autonomy (#2088) * Promote Windows NI proof authority and local proof autonomy * ci(windows): fail closed on hosted NI proof timeouts --------- Co-authored-by: svelderrainruiz --- .github/workflows/vi-binary-gate.yml | 18 +- .github/workflows/windows-hosted-parity.yml | 155 +----------- .../workflows/windows-ni-proof-reusable.yml | 196 +++++++++++++++ .../vi-history-local-proof-control-plane.md | 10 + ...ows-docker-shared-surface-control-plane.md | 9 +- docs/knowledgebase/FEATURE_BRANCH_POLICY.md | 6 +- docs/knowledgebase/Pester-Service-Model.md | 8 + docs/knowledgebase/VI-History-Local-Proof.md | 15 ++ .../Windows-Docker-Shared-Surface.md | 18 ++ docs/requirements-pester-service-model-srs.md | 7 +- ...requirements-vi-history-local-proof-srs.md | 5 +- ...ments-windows-docker-shared-surface-srs.md | 9 +- docs/rtm-pester-service-model.csv | 1 + docs/rtm-vi-history-local-proof.csv | 3 + docs/rtm-windows-docker-shared-surface.csv | 2 + .../testing/pester-service-model-test-plan.md | 4 +- .../vi-history-local-proof-test-plan.md | 8 +- ...windows-docker-shared-surface-test-plan.md | 6 +- tests/Invoke-DockerRuntimeManager.Tests.ps1 | 88 +++++++ ...est-WindowsNI2026q1HostPreflight.Tests.ps1 | 82 +++++++ tests/ViBinaryHandling.Tests.ps1 | 40 +-- tools/Assert-DockerRuntimeDeterminism.ps1 | 12 +- tools/Invoke-DockerRuntimeManager.ps1 | 227 ++++++++++++++++-- tools/Test-VIBinaryHandlingInvariants.ps1 | 169 +++++++++++++ tools/Test-WindowsNI2026q1HostPreflight.ps1 | 200 ++++++++++++++- ...vice-model-local-harness-contract.test.mjs | 9 + ...r-service-model-workflow-contract.test.mjs | 6 + .../__tests__/vi-history-local-ci.test.mjs | 55 ++++- .../vi-history-local-proof-contract.test.mjs | 32 ++- ...ws-docker-shared-surface-contract.test.mjs | 16 ++ ...indows-ni-proof-workflow-contract.test.mjs | 69 ++++++ .../windows-workflow-replay-lane.test.mjs | 33 +++ tools/priority/vi-history-local-ci.mjs | 165 +++++++------ .../priority/windows-workflow-replay-lane.mjs | 26 ++ 34 files changed, 1408 insertions(+), 301 deletions(-) create mode 100644 .github/workflows/windows-ni-proof-reusable.yml create mode 100644 tools/Test-VIBinaryHandlingInvariants.ps1 create mode 100644 tools/priority/__tests__/windows-ni-proof-workflow-contract.test.mjs diff --git a/.github/workflows/vi-binary-gate.yml b/.github/workflows/vi-binary-gate.yml index de212b31d..808688633 100644 --- a/.github/workflows/vi-binary-gate.yml +++ b/.github/workflows/vi-binary-gate.yml @@ -4,12 +4,20 @@ on: paths: - '.gitattributes' - 'scripts/**' + - 'tools/**' + - 'fixtures/vi-stage/control-rename/**' + - '.github/workflows/vi-binary-gate.yml' + - '.github/workflows/windows-ni-proof-reusable.yml' - 'tests/ViBinaryHandling.Tests.ps1' push: branches: [ main ] paths: - '.gitattributes' - 'scripts/**' + - 'tools/**' + - 'fixtures/vi-stage/control-rename/**' + - '.github/workflows/vi-binary-gate.yml' + - '.github/workflows/windows-ni-proof-reusable.yml' - 'tests/ViBinaryHandling.Tests.ps1' merge_group: branches: [ main ] @@ -22,9 +30,11 @@ on: jobs: vi-binary-check: - uses: ./.github/workflows/pester-reusable.yml + uses: ./.github/workflows/windows-ni-proof-reusable.yml with: - include_integration: ${{ 'false' }} - install_pester: ${{ 'true' }} - include_patterns: ${{ 'tests/ViBinaryHandling.Tests.ps1' }} sample_id: ${{ github.event.inputs.sample_id || '' }} + base_vi: fixtures/vi-stage/control-rename/Base.vi + head_vi: fixtures/vi-stage/control-rename/Head.vi + results_root: tests/results/vi-binary-gate + artifact_name: vi-binary-gate + run_binary_invariants: true diff --git a/.github/workflows/windows-hosted-parity.yml b/.github/workflows/windows-hosted-parity.yml index 662976e80..3e50a7aec 100644 --- a/.github/workflows/windows-hosted-parity.yml +++ b/.github/workflows/windows-hosted-parity.yml @@ -8,153 +8,12 @@ on: required: false type: string -concurrency: - group: ${{ github.workflow }}-${{ github.event.inputs.sample_id || github.ref }} - cancel-in-progress: true - jobs: hosted-ni-proof: - runs-on: windows-2022 - timeout-minutes: 50 - permissions: - contents: read - env: - NI_WINDOWS_IMAGE: nationalinstruments/labview:2026q1-windows - NI_WINDOWS_LABVIEW_PATH: C:\Program Files\National Instruments\LabVIEW 2026\LabVIEW.exe - defaults: - run: - shell: pwsh - steps: - - uses: actions/checkout@v5 - - - uses: actions/setup-node@v6 - with: - node-version: '20' - - - name: Install dependencies (no scripts) - run: node tools/npm/cli.mjs ci --ignore-scripts - - - name: Runner health (notice-only) - if: ${{ vars.RUNNER_HEALTH != '0' }} - run: pwsh -File tools/Collect-RunnerHealth.ps1 -AppendSummary -EmitJson - - - name: Hooks preflight parity - run: | - node tools/npm/run-script.mjs hooks:plane - node tools/npm/run-script.mjs hooks:preflight - - - name: Prepare NI Windows image and hosted runtime - run: | - $resultsRoot = 'tests/results/windows-hosted-parity' - New-Item -ItemType Directory -Path $resultsRoot -Force | Out-Null - $summaryPath = Join-Path $resultsRoot 'windows-ni-2026q1-host-preflight.json' - pwsh -NoLogo -NoProfile -File tools/Test-WindowsNI2026q1HostPreflight.ps1 ` - -Image $env:NI_WINDOWS_IMAGE ` - -ResultsDir $resultsRoot ` - -ExecutionSurface 'github-hosted-windows' ` - -OutputJsonPath $summaryPath ` - -GitHubOutputPath $env:GITHUB_OUTPUT ` - -StepSummaryPath $env:GITHUB_STEP_SUMMARY - - - name: Run NI Windows create comparison report - id: windows-compare - run: | - $resultsRoot = 'tests/results/windows-hosted-parity' - New-Item -ItemType Directory -Path $resultsRoot -Force | Out-Null - $reportPath = Join-Path $resultsRoot 'windows-compare-report.html' - "windows_compare_report=$reportPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - $runtimeSnapshot = Join-Path $resultsRoot 'runtime-manager-compare-windows.json' - pwsh -NoLogo -NoProfile -File tools/Run-NIWindowsContainerCompare.ps1 ` - -BaseVi 'fixtures/vi-stage/control-rename/Base.vi' ` - -HeadVi 'fixtures/vi-stage/control-rename/Head.vi' ` - -Image $env:NI_WINDOWS_IMAGE ` - -LabVIEWPath $env:NI_WINDOWS_LABVIEW_PATH ` - -ReportPath $reportPath ` - -TimeoutSeconds 600 ` - -RuntimeEngineReadyTimeoutSeconds 180 ` - -RuntimeEngineReadyPollSeconds 5 ` - -RuntimeSnapshotPath $runtimeSnapshot - $compareExit = $LASTEXITCODE - - $capturePath = Join-Path $resultsRoot 'ni-windows-container-capture.json' - if (Test-Path -LiteralPath $capturePath -PathType Leaf) { - "windows_compare_capture=$capturePath" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append - $capture = Get-Content -LiteralPath $capturePath -Raw | ConvertFrom-Json -Depth 12 - $gateOutcome = if ($capture.PSObject.Properties['gateOutcome']) { [string]$capture.gateOutcome } else { '' } - $resultClass = if ($capture.PSObject.Properties['resultClass']) { [string]$capture.resultClass } else { '' } - Write-Host ("[windows-hosted-compare] exit={0} gateOutcome={1} resultClass={2}" -f $compareExit, $gateOutcome, $resultClass) - if ($compareExit -ne 0 -and $gateOutcome -ne 'pass') { - exit $compareExit - } - } elseif ($compareExit -ne 0) { - throw ("Windows compare failed (exit={0}) and capture file was not found at {1}" -f $compareExit, $capturePath) - } - - - name: Validate Windows comparison report artifact contract - if: always() - run: | - $resultsRoot = 'tests/results/windows-hosted-parity' - $reportPath = '${{ steps.windows-compare.outputs.windows_compare_report }}' - if ([string]::IsNullOrWhiteSpace($reportPath)) { - $reportPath = Join-Path $resultsRoot 'windows-compare-report.html' - } - $capturePath = '${{ steps.windows-compare.outputs.windows_compare_capture }}' - if ([string]::IsNullOrWhiteSpace($capturePath)) { - $capturePath = Join-Path $resultsRoot 'ni-windows-container-capture.json' - } - $summaryPath = Join-Path $resultsRoot 'windows-compare-artifact-summary.json' - $captureResultClass = '' - $captureGateOutcome = '' - $captureExists = Test-Path -LiteralPath $capturePath -PathType Leaf - if ($captureExists) { - $capture = Get-Content -LiteralPath $capturePath -Raw | ConvertFrom-Json -Depth 12 - if ($capture.PSObject.Properties['resultClass']) { - $captureResultClass = [string]$capture.resultClass - } - if ($capture.PSObject.Properties['gateOutcome']) { - $captureGateOutcome = [string]$capture.gateOutcome - } - } - - $reportExists = Test-Path -LiteralPath $reportPath -PathType Leaf - $compareSucceeded = '${{ steps.windows-compare.conclusion }}' -eq 'success' - - $payload = [ordered]@{ - schema = 'vi-history/windows-compare-artifact-summary@v1' - generatedAt = (Get-Date).ToUniversalTime().ToString('o') - compareStepConclusion = '${{ steps.windows-compare.conclusion }}' - reportPath = $reportPath - reportExists = [bool]$reportExists - capturePath = $capturePath - captureExists = [bool]$captureExists - captureResultClass = $captureResultClass - captureGateOutcome = $captureGateOutcome - } - $payload | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $summaryPath -Encoding utf8 - - $requireReport = $compareSucceeded -and [string]::Equals($captureResultClass, 'success-diff', [System.StringComparison]::OrdinalIgnoreCase) - if ($requireReport -and -not $reportExists) { - throw ("Windows compare classified success-diff but report file was not found: {0}" -f $reportPath) - } - if ($compareSucceeded -and -not $reportExists -and -not $requireReport) { - Write-Host ("::warning::Windows compare succeeded without diff classification ({0}); no report file was generated at {1}." -f $captureResultClass, $reportPath) - } - if (-not $reportExists) { - Write-Host ("::warning::Windows comparison report not found at {0}" -f $reportPath) - } - - - name: Upload Windows hosted proof artifacts - if: always() - uses: actions/upload-artifact@v7 - with: - name: windows-hosted-ni-proof - path: | - tests/results/windows-hosted-parity/windows-compare-report.html - tests/results/windows-hosted-parity/windows-compare-artifact-summary.json - tests/results/windows-hosted-parity/windows-ni-2026q1-host-preflight.json - tests/results/windows-hosted-parity/ni-windows-container-capture.json - tests/results/windows-hosted-parity/ni-windows-container-stdout.txt - tests/results/windows-hosted-parity/ni-windows-container-stderr.txt - tests/results/windows-hosted-parity/container-export/** - tests/results/windows-hosted-parity/runtime-manager-compare-windows.json - if-no-files-found: warn + uses: ./.github/workflows/windows-ni-proof-reusable.yml + with: + sample_id: ${{ github.event.inputs.sample_id || '' }} + base_vi: fixtures/vi-stage/control-rename/Base.vi + head_vi: fixtures/vi-stage/control-rename/Head.vi + results_root: tests/results/windows-hosted-parity + artifact_name: windows-hosted-ni-proof diff --git a/.github/workflows/windows-ni-proof-reusable.yml b/.github/workflows/windows-ni-proof-reusable.yml new file mode 100644 index 000000000..b00354a8c --- /dev/null +++ b/.github/workflows/windows-ni-proof-reusable.yml @@ -0,0 +1,196 @@ +name: Windows NI proof (reusable) + +on: + workflow_call: + inputs: + base_vi: + required: true + type: string + head_vi: + required: true + type: string + sample_id: + required: false + type: string + default: '' + results_root: + required: false + type: string + default: tests/results/windows-ni-proof + artifact_name: + required: false + type: string + default: windows-ni-proof + image: + required: false + type: string + default: nationalinstruments/labview:2026q1-windows + labview_path: + required: false + type: string + default: C:\Program Files\National Instruments\LabVIEW 2026\LabVIEW.exe + report_type: + required: false + type: string + default: html + timeout_seconds: + required: false + type: number + default: 600 + run_binary_invariants: + required: false + type: boolean + default: false + +concurrency: + group: ${{ github.workflow }}-windows-ni-proof-${{ inputs.sample_id || github.ref }} + cancel-in-progress: true + +jobs: + windows-ni-proof: + runs-on: windows-2022 + timeout-minutes: 50 + permissions: + contents: read + defaults: + run: + shell: pwsh + env: + NI_WINDOWS_IMAGE: ${{ inputs.image }} + NI_WINDOWS_LABVIEW_PATH: ${{ inputs.labview_path }} + steps: + - uses: actions/checkout@v5 + with: + repository: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + + - name: Validate VI binary-handling invariants + if: ${{ inputs.run_binary_invariants }} + run: | + $resultsRoot = '${{ inputs.results_root }}' + New-Item -ItemType Directory -Path $resultsRoot -Force | Out-Null + $reportPath = Join-Path $resultsRoot 'vi-binary-handling-invariants.json' + pwsh -NoLogo -NoProfile -File tools/Test-VIBinaryHandlingInvariants.ps1 ` + -OutputJsonPath $reportPath ` + -StepSummaryPath $env:GITHUB_STEP_SUMMARY + + - name: Prepare NI Windows image and hosted runtime + id: windows-preflight + timeout-minutes: 20 + run: | + $resultsRoot = '${{ inputs.results_root }}' + New-Item -ItemType Directory -Path $resultsRoot -Force | Out-Null + $summaryPath = Join-Path $resultsRoot 'windows-ni-2026q1-host-preflight.json' + pwsh -NoLogo -NoProfile -File tools/Test-WindowsNI2026q1HostPreflight.ps1 ` + -Image $env:NI_WINDOWS_IMAGE ` + -ResultsDir $resultsRoot ` + -ExecutionSurface 'github-hosted-windows' ` + -OutputJsonPath $summaryPath ` + -GitHubOutputPath $env:GITHUB_OUTPUT ` + -StepSummaryPath $env:GITHUB_STEP_SUMMARY + + - name: Run NI Windows create comparison report + id: windows-compare + if: steps.windows-preflight.outputs.windows_host_preflight_status == 'ready' + run: | + $resultsRoot = '${{ inputs.results_root }}' + New-Item -ItemType Directory -Path $resultsRoot -Force | Out-Null + $reportPath = Join-Path $resultsRoot ('windows-compare-report.{0}' -f '${{ inputs.report_type }}') + "windows_compare_report=$reportPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + $runtimeSnapshot = Join-Path $resultsRoot 'runtime-manager-compare-windows.json' + pwsh -NoLogo -NoProfile -File tools/Run-NIWindowsContainerCompare.ps1 ` + -BaseVi '${{ inputs.base_vi }}' ` + -HeadVi '${{ inputs.head_vi }}' ` + -Image $env:NI_WINDOWS_IMAGE ` + -LabVIEWPath $env:NI_WINDOWS_LABVIEW_PATH ` + -ReportType '${{ inputs.report_type }}' ` + -ReportPath $reportPath ` + -TimeoutSeconds ${{ inputs.timeout_seconds }} ` + -RuntimeEngineReadyTimeoutSeconds 180 ` + -RuntimeEngineReadyPollSeconds 5 ` + -RuntimeSnapshotPath $runtimeSnapshot + $compareExit = $LASTEXITCODE + + $capturePath = Join-Path $resultsRoot 'ni-windows-container-capture.json' + if (Test-Path -LiteralPath $capturePath -PathType Leaf) { + "windows_compare_capture=$capturePath" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + $capture = Get-Content -LiteralPath $capturePath -Raw | ConvertFrom-Json -Depth 12 + $gateOutcome = if ($capture.PSObject.Properties['gateOutcome']) { [string]$capture.gateOutcome } else { '' } + $resultClass = if ($capture.PSObject.Properties['resultClass']) { [string]$capture.resultClass } else { '' } + Write-Host ("[windows-ni-proof] exit={0} gateOutcome={1} resultClass={2}" -f $compareExit, $gateOutcome, $resultClass) + if ($compareExit -ne 0 -and $gateOutcome -ne 'pass') { + exit $compareExit + } + } elseif ($compareExit -ne 0) { + throw ("Windows compare failed (exit={0}) and capture file was not found at {1}" -f $compareExit, $capturePath) + } + + - name: Validate Windows comparison report artifact contract + if: always() + run: | + $resultsRoot = '${{ inputs.results_root }}' + $reportPath = '${{ steps.windows-compare.outputs.windows_compare_report }}' + if ([string]::IsNullOrWhiteSpace($reportPath)) { + $reportPath = Join-Path $resultsRoot ('windows-compare-report.{0}' -f '${{ inputs.report_type }}') + } + $capturePath = '${{ steps.windows-compare.outputs.windows_compare_capture }}' + if ([string]::IsNullOrWhiteSpace($capturePath)) { + $capturePath = Join-Path $resultsRoot 'ni-windows-container-capture.json' + } + $summaryPath = Join-Path $resultsRoot 'windows-compare-artifact-summary.json' + $captureResultClass = '' + $captureGateOutcome = '' + $captureExists = Test-Path -LiteralPath $capturePath -PathType Leaf + if ($captureExists) { + $capture = Get-Content -LiteralPath $capturePath -Raw | ConvertFrom-Json -Depth 12 + if ($capture.PSObject.Properties['resultClass']) { + $captureResultClass = [string]$capture.resultClass + } + if ($capture.PSObject.Properties['gateOutcome']) { + $captureGateOutcome = [string]$capture.gateOutcome + } + } + + $reportExists = Test-Path -LiteralPath $reportPath -PathType Leaf + $compareSucceeded = '${{ steps.windows-compare.conclusion }}' -eq 'success' + + $payload = [ordered]@{ + schema = 'vi-history/windows-compare-artifact-summary@v1' + generatedAt = (Get-Date).ToUniversalTime().ToString('o') + compareStepConclusion = '${{ steps.windows-compare.conclusion }}' + reportPath = $reportPath + reportExists = [bool]$reportExists + capturePath = $capturePath + captureExists = [bool]$captureExists + captureResultClass = $captureResultClass + captureGateOutcome = $captureGateOutcome + } + $payload | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $summaryPath -Encoding utf8 + + $requireReport = $compareSucceeded -and [string]::Equals($captureResultClass, 'success-diff', [System.StringComparison]::OrdinalIgnoreCase) + if ($requireReport -and -not $reportExists) { + throw ("Windows compare classified success-diff but report file was not found: {0}" -f $reportPath) + } + if ($compareSucceeded -and -not $reportExists -and -not $requireReport) { + Write-Host ("::warning::Windows compare succeeded without diff classification ({0}); no report file was generated at {1}." -f $captureResultClass, $reportPath) + } + if (-not $reportExists) { + Write-Host ("::warning::Windows comparison report not found at {0}" -f $reportPath) + } + + - name: Upload Windows NI proof artifacts + if: always() + uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.artifact_name }} + path: | + ${{ inputs.results_root }}/vi-binary-handling-invariants.json + ${{ inputs.results_root }}/windows-compare-report.${{ inputs.report_type }} + ${{ inputs.results_root }}/windows-compare-artifact-summary.json + ${{ inputs.results_root }}/windows-ni-2026q1-host-preflight.json + ${{ inputs.results_root }}/ni-windows-container-capture.json + ${{ inputs.results_root }}/ni-windows-container-stdout.txt + ${{ inputs.results_root }}/ni-windows-container-stderr.txt + ${{ inputs.results_root }}/container-export/** + ${{ inputs.results_root }}/runtime-manager-compare-windows.json + if-no-files-found: warn diff --git a/docs/architecture/vi-history-local-proof-control-plane.md b/docs/architecture/vi-history-local-proof-control-plane.md index 5fa480a1e..4578d4ab6 100644 --- a/docs/architecture/vi-history-local-proof-control-plane.md +++ b/docs/architecture/vi-history-local-proof-control-plane.md @@ -36,5 +36,15 @@ be exercised before another hosted run is chosen. - The local VI History packet should reuse the shared `windows-docker-desktop-ni-image` proof surface when the current host cannot satisfy the Windows replay lane. +- The local VI History packet should keep live Windows workflow replay as an + explicit next-step escalation once the shared Windows surface and clone-backed + candidate are ready, rather than invoking the replay lane implicitly during + packet selection. +- The governed Windows workflow replay lane should own bounded helper-process + timeouts so the autonomy loop cannot be left hanging after a live replay + attempt. +- A passing governed Windows workflow replay receipt should be consumable by + the local autonomy surface so the loop advances instead of requesting the + same replay step repeatedly. - Local proof should prefer declared profiles and wrappers over free-form helper invocation. diff --git a/docs/architecture/windows-docker-shared-surface-control-plane.md b/docs/architecture/windows-docker-shared-surface-control-plane.md index fc2d744d9..77b377a99 100644 --- a/docs/architecture/windows-docker-shared-surface-control-plane.md +++ b/docs/architecture/windows-docker-shared-surface-control-plane.md @@ -4,7 +4,8 @@ This control plane covers the shared Windows Docker Desktop + pinned NI Windows image surface that adjacent local proof packets should use before another -hosted rerun is chosen. +hosted rerun is chosen, and the hosted proof workflow that owns blocking CI +authority for Windows image-backed binary-handling lanes. ## Surface View @@ -15,6 +16,7 @@ hosted rerun is chosen. | Bridge surface | Reach Windows-local PowerShell and Node execution from a Unix or WSL coordinator | Node.js + PowerShell | | Path-hygiene surface | Detect synced or externally managed roots such as OneDrive-managed paths before live proof | Node.js | | Windows-local staging surface | Stage UNC-backed or otherwise non-bindable Windows inputs and outputs into a local mount root for Docker consumption | PowerShell + Docker | +| Hosted CI authority surface | Route blocking Windows image-backed gates through the shared Windows NI proof workflow instead of the generic Pester reusable workflow | GitHub Actions + PowerShell | | Local autonomy surface | Emit ranked local guidance, proof checks, and next-step handoffs for the shared surface | Node.js + assurance packet | ## Component View @@ -25,6 +27,8 @@ hosted rerun is chosen. | `Test-WindowsNI2026q1HostPreflight.ps1` | Bootstrap and preflight surface | Emit deterministic Windows host preflight contracts | | `Run-NIWindowsContainerCompare.ps1` | Bootstrap and preflight surface | Provide the bounded compare probe on the shared NI Windows image surface | | `Run-NIWindowsContainerCompare.ps1` | Windows-local staging surface | Stage UNC-backed WSL container-bound paths into a Windows-local mount root and synchronize report artifacts back | +| `windows-ni-proof-reusable.yml` | Hosted CI authority surface | Reuse the hosted Windows NI proof contract across manual parity and blocking CI gates | +| `Test-VIBinaryHandlingInvariants.ps1` | Hosted CI authority surface | Emit the supporting static invariant contract without routing the gate through Pester | | `windows-host-bridge.mjs` | Bridge surface | Resolve reachable Windows PowerShell/Node entrypoints and run governed Windows-local work from Unix or WSL | | `windows-docker-shared-surface-local-ci.mjs` | Path-hygiene + local autonomy surfaces | Detect OneDrive-like risks, synthesize the packet, and emit next-step handoffs | @@ -36,6 +40,9 @@ hosted rerun is chosen. shared surface instead of an incidental environment concern. - The shared surface should remain the first-class owner of `windows-docker-desktop-ni-image` escalation semantics. +- Blocking CI lanes for Windows image-backed binary-handling proof should route + through `windows-ni-proof-reusable.yml` and not through + `.github/workflows/pester-reusable.yml`. - When a reachable Windows Desktop exists behind Unix or WSL, the bridge surface should execute Windows-local probe and preflight work before the packet emits a host-unavailable escalation. diff --git a/docs/knowledgebase/FEATURE_BRANCH_POLICY.md b/docs/knowledgebase/FEATURE_BRANCH_POLICY.md index 5b9366cf3..b271d20e2 100644 --- a/docs/knowledgebase/FEATURE_BRANCH_POLICY.md +++ b/docs/knowledgebase/FEATURE_BRANCH_POLICY.md @@ -85,7 +85,7 @@ promotion behavior, not the branch-class source of truth. | Manifest identity | Scope | Highlights | |-------------------|----------------------|----------------------------------------------------------------------------------------------| | `develop` (live id may drift) | `refs/heads/develop` | Merge queue enabled (`merge_method=SQUASH`, `grouping=ALLGREEN`, build queue <=20 entries, 1-minute quiet window). Required checks: `lint`, `fixtures`, `Policy Guard (Upstream) / policy-guard`, `vi-history-scenarios-linux`, `commit-integrity`. Non-required hosted proof lanes may run alongside the queue contract, including `session-index`, `issue-snapshot`, `semver`, `agent-review-policy`, `hook-parity`, and `vi-history-scenarios-windows` on GitHub-hosted `windows-2022`. Copilot review settings are no longer enforced through policy; draft/ready review semantics are repo-owned and validated by `agent-review-policy`. | -| `main` (`tools/priority/policy.json` key `8614140`) | `refs/heads/main` | Merge queue enabled (`merge_method=SQUASH`, `grouping=ALLGREEN`, build queue <=5 entries, 1-minute quiet window). Required checks: `lint`, `pester`, `vi-binary-check`, `vi-compare`, `Policy Guard (Upstream) / policy-guard`, `commit-integrity`. Required approving reviews: `0`. | +| `main` (`tools/priority/policy.json` key `8614140`) | `refs/heads/main` | Merge queue enabled (`merge_method=SQUASH`, `grouping=ALLGREEN`, build queue <=5 entries, 1-minute quiet window). Required checks: `lint`, `pester`, `vi-binary-check`, `vi-compare`, `Policy Guard (Upstream) / policy-guard`, `commit-integrity`. `vi-binary-check` is the Windows NI proof gate; `pester` remains a separate harness/evidence gate. Required approving reviews: `0`. | | `release` (`tools/priority/policy.json` key `8614172`) | `refs/heads/release/*` | No merge queue; protects against force-push/deletion. Required checks: `lint`, `pester / normalize`, `smoke-gate`, `Policy Guard (Upstream) / policy-guard`, `commit-integrity`. Required approving reviews: `0`. | `node tools/npm/run-script.mjs priority:policy` queries these rulesets and fails if the live configuration drifts from @@ -285,6 +285,10 @@ checked into `tools/priority/policy.json` so `priority:policy` stays authoritati - `min_entries_to_merge_wait_minutes=1` - `check_response_timeout_minutes=60` - **Required checks**: `lint`, `pester`, `vi-binary-check`, `vi-compare`, `Policy Guard (Upstream) / policy-guard`, `commit-integrity`. +- **Windows NI proof authority**: `vi-binary-check` now routes through the + shared Windows NI proof workflow and is the authoritative blocking proof for + Windows image-backed binary handling on `main`; it is no longer a thin + `pester-reusable` wrapper. - **Workflow triggers**: Ensure those required checks run on both `pull_request` and `merge_group` so queued entries can merge. - **Approval policy**: 0 required reviews; stale review dismissal and thread resolution are not enforced. - **Quick verification**: diff --git a/docs/knowledgebase/Pester-Service-Model.md b/docs/knowledgebase/Pester-Service-Model.md index 9e64a302d..90bad793e 100644 --- a/docs/knowledgebase/Pester-Service-Model.md +++ b/docs/knowledgebase/Pester-Service-Model.md @@ -118,6 +118,14 @@ The additive pilot introduces seven workflow surfaces: - When the next truthful proof surface is unavailable from the current host, local autonomy should emit a machine-readable escalation step instead of a human-only advisory. The canonical handoff artifact is `pester-service-model-next-step.json`. - When more than one local proof packet exists, the shared program selector should reconcile them into one next step. Requirement work still outranks escalations, and shared `windows-docker-desktop-ni-image` escalations should merge into one `comparevi-local-program-next-step.json` handoff instead of competing packet advisories. - The existing required gate remains in place until the pilot proves equivalent or better behavior. +- For Windows image-backed binary-handling CI surfaces, the authoritative + blocking proof should stay on the shared Windows NI image path. On that + surface, Pester is secondary harness and evidence truth rather than the + primary execution gate. +- The current implementation of that authority split is + `.github/workflows/vi-binary-gate.yml`, which now routes through + `.github/workflows/windows-ni-proof-reusable.yml` instead of + `.github/workflows/pester-reusable.yml`. - Trusted PR proving must stay on `pull_request_target` with same-owner gating. Cross-owner fork heads are not allowed to drive self-hosted execution. ## Promotion Rule diff --git a/docs/knowledgebase/VI-History-Local-Proof.md b/docs/knowledgebase/VI-History-Local-Proof.md index f0b8a07fd..4dd72cb4c 100644 --- a/docs/knowledgebase/VI-History-Local-Proof.md +++ b/docs/knowledgebase/VI-History-Local-Proof.md @@ -31,6 +31,9 @@ Pester service model, not as an incidental side effect of Pester work. prose-only advisory. - The VI History packet should reuse the shared `windows-docker-desktop-ni-image` proof surface with Pester rather than inventing a second unmanaged Windows proof lane. - When a reachable Windows Desktop is available behind WSL or another Unix coordinator, the VI History packet should consume that bridge before emitting a `windows-docker-desktop-ni-image` escalation. +- Local VI History CI should not launch the live Windows workflow replay lane merely to decide what comes next. +- Once the shared Windows surface and clone-backed live-history candidate are ready, the next step should be an explicit `vi-history-windows-workflow-replay` handoff that points at `priority:workflow:replay:windows:vi-history`. +- The governed Windows workflow replay lane should terminate or fail closed within bounded helper-process timeouts, and it should still emit a workflow receipt when a timeout occurs. - When the replay lane runs from a UNC-backed WSL checkout through the shared Windows Docker surface, container-bound inputs and output targets should be staged into a Windows-local mount root and synchronized back to the @@ -86,6 +89,18 @@ When more than one local proof packet exists, the shared selector should emit `windows-docker-desktop-ni-image` handoff across VI History and Pester instead of leaving two independent advisories for a human to reconcile. +When the shared Windows surface and governed live-history candidate are already +ready, the VI History packet should instead emit +`vi-history-local-next-step.json` with required surface +`vi-history-windows-workflow-replay` and command: + +- `npm run priority:workflow:replay:windows:vi-history` + +Once that governed replay lane emits a passing +`vi-history-scenarios-windows-receipt.json`, local VI History CI should consume +that receipt as satisfied replay proof instead of continuing to re-select the +same replay step. + ## Governed Live-History Candidate The current clone-backed iteration target is governed explicitly in diff --git a/docs/knowledgebase/Windows-Docker-Shared-Surface.md b/docs/knowledgebase/Windows-Docker-Shared-Surface.md index ca439c682..3e18dc325 100644 --- a/docs/knowledgebase/Windows-Docker-Shared-Surface.md +++ b/docs/knowledgebase/Windows-Docker-Shared-Surface.md @@ -4,6 +4,11 @@ The shared Windows Docker Desktop + pinned NI Windows image surface should be treated as its own local proof packet, not as an incidental advisory merged from sibling packets. +For Windows image-backed CI gates, this surface is also the authoritative +execution-truth plane. Supporting static invariants may run beside it, but the +blocking gate should route through the Windows NI proof path instead of through +the generic Pester reusable workflow. + ## Local Proof Surfaces - `tests:windows-surface:probe` @@ -15,6 +20,9 @@ from sibling packets. - bounded container compare probe on the shared Windows surface - `priority:windows-surface:local-ci` - machine-readable local assurance loop for the shared Windows surface +- `.github/workflows/windows-ni-proof-reusable.yml` + - reusable hosted Windows NI proof lane for CI gates such as + `.github/workflows/vi-binary-gate.yml` ## Design Rules @@ -24,6 +32,10 @@ from sibling packets. recommended. OneDrive-like managed roots are risk until a safe local root is used. - Bootstrap, probe, and packet-local proof should remain separate contracts. +- Windows image-backed CI gates should use `windows-ni-proof-reusable.yml` as + the reusable hosted proof contract, with `Run-NIWindowsContainerCompare.ps1` + as the authoritative execution helper and `Test-VIBinaryHandlingInvariants.ps1` + as an optional supporting invariant surface. - The shared Windows surface should participate in the same local proof program selector as Pester and VI History. - When the coordinator is running under WSL or another Unix host but a @@ -45,6 +57,7 @@ from sibling packets. - `windows-docker-shared-surface-local-ci-report.json` - `windows-docker-shared-surface-next-step.json` - `ni-windows-container-capture.json` +- `comparevi/vi-binary-handling-invariants@v1` ## Local Commands @@ -63,3 +76,8 @@ This packet is the authoritative home for the shared escalate to that surface, but the surface itself should also be governable as a packet with its own requirements, proof checks, bounded next-step handoff, and reachable Windows host bridge rules. + +The current CI authority slice that implements this rule is: + +- `.github/workflows/vi-binary-gate.yml` +- `.github/workflows/windows-ni-proof-reusable.yml` diff --git a/docs/requirements-pester-service-model-srs.md b/docs/requirements-pester-service-model-srs.md index 971140e25..59dc89099 100644 --- a/docs/requirements-pester-service-model-srs.md +++ b/docs/requirements-pester-service-model-srs.md @@ -3,7 +3,7 @@ ## Document Control - System: Pester service-model control plane -- Version: `v0.1.20` +- Version: `v0.1.21` - Owner: `#2069` - Basis: retained fork promotion dossier under `#2078` - Status: Active @@ -22,7 +22,9 @@ retained artifacts, local replay surfaces, evidence provenance, machine-readable next-requirement guidance, representative retained-artifact replay, local Windows-container surrogate proof, autonomy policy, - machine-readable next-step escalation, and evidence classification. + machine-readable next-step escalation, evidence classification, and the + bounded authority split between Pester harness truth and Windows image-backed + CI product proof. - Out of scope: Legacy monolithic `test-pester.yml` behavior except where it remains the current baseline to compare against. @@ -68,6 +70,7 @@ | REQ-PSM-026 | The local autonomy loop shall consume representative proof checks, including retained-artifact replay and Windows-container surface status, and shall reopen implemented requirements when representative proof checks regress. | Static RTM status alone can overstate maturity; an autonomous loop must reopen the owning requirement when real local proof contradicts the packet. | `priority:pester:local-ci` records proof checks, refuses to report a clean `pass` when representative replay regresses, and selects the owning requirement as the next local target even when the RTM row is already marked implemented. | `TEST-PSM-026` | | REQ-PSM-027 | When the next truthful proof surface is unavailable from the current host, local assurance CI shall emit a machine-readable escalation step that names the blocked requirement, governing requirement, required proof surface, current host state, and exact next commands. | An autonomous loop that stops at an advisory without an explicit escalation packet still depends on human interpretation and cannot hand off cleanly to another agent or host. | `priority:pester:local-ci` emits `pester-service-model-next-step.json`; when no locally actionable requirement remains and the Windows-container surface is unavailable, the next step is an escalation packet rather than `null`, with receipt path, required surface `windows-docker-desktop-ni-image`, and recommended commands. | `TEST-PSM-027` | | REQ-PSM-028 | The Pester packet shall participate in the shared local proof program selector so autonomous development can choose between sibling packets and merge shared-surface escalations into one bounded handoff. | Once more than one local proof packet exists, two independent packet advisories force a human to reconcile them and weaken the value of the autonomy loop. | `priority:program:local-ci` consumes the Pester packet next-step artifact, can select a Pester requirement ahead of sibling packet escalations, and merges shared `windows-docker-desktop-ni-image` escalations from Pester and VI History into one `comparevi-local-program-next-step.json` handoff. | `TEST-PSM-028` | +| REQ-PSM-029 | For Windows image-backed binary-handling CI surfaces, the Pester packet shall remain secondary harness or evidence truth and shall not own the blocking execution gate when the shared Windows NI proof surface is available. | Image-backed compare execution is closer to product truth than the generic dispatcher harness, so the Pester packet should not keep blocking authority over CI surfaces that already have a governed Windows NI proof lane. | `docs/knowledgebase/Pester-Service-Model.md` states that Windows image-backed CI proof remains authoritative on the shared Windows NI path, `vi-binary-gate.yml` routes through `windows-ni-proof-reusable.yml`, and the Pester packet no longer claims that gate as its authoritative executor. | `TEST-PSM-029` | ## Assumptions diff --git a/docs/requirements-vi-history-local-proof-srs.md b/docs/requirements-vi-history-local-proof-srs.md index 24aee3340..70cce0e0c 100644 --- a/docs/requirements-vi-history-local-proof-srs.md +++ b/docs/requirements-vi-history-local-proof-srs.md @@ -3,7 +3,7 @@ ## Document Control - System: VI History local proof control plane -- Version: `v0.1.3` +- Version: `v0.1.6` - Owner: `#2069` - Status: Active @@ -42,3 +42,6 @@ | REQ-VHLP-007 | The VI History packet shall participate in the shared local proof program selector so it can be chosen explicitly against sibling packets and share merged escalation handoffs when the required surface is common. | VI History should be an explicit sibling proof surface, not an orphan packet that a human has to reconcile manually against Pester. | `priority:program:local-ci` consumes `vi-history-local-next-step.json`, can select a VI History requirement ahead of sibling packet escalations, and merges shared `windows-docker-desktop-ni-image` escalations from VI History and Pester into one `comparevi-local-program-next-step.json` handoff. | `TEST-VHLP-007` | | REQ-VHLP-008 | The VI History packet shall govern a clone-backed live-history iteration candidate, and the initial candidate shall be `ni/labview-icon-editor:Tooling/deployment/VIP_Pre-Uninstall Custom Action.vi`. | Retained fixtures and public proof seeds are useful, but they do not replace a real repo clone with actual git lineage when local maintainers need to iterate on VI History behavior. | `tools/priority/vi-history-live-candidate.json` declares the candidate id, repo slug, repo URL, default branch, clone-root override contract, target VI path, and minimum git-history expectation for `VIP_Pre-Uninstall Custom Action.vi`. | `TEST-VHLP-008` | | REQ-VHLP-009 | Local assurance CI shall validate that the governed live-history candidate clone exists locally, contains the target VI, and exposes real git history before Windows replay or hosted proof is chosen as the next step. | There is no truthful VI History local proof for a live-history target if the repo clone, target VI, or commit history are missing. | `vi-history-local-ci.mjs` emits `vi-history-live-candidate-readiness.json` with `ready`, `missing-clone`, `missing-target`, `missing-history`, or `git-failed`, records clone and history facts, and emits a machine-readable clone-preparation escalation when the governed candidate is unavailable. | `TEST-VHLP-009` | +| REQ-VHLP-010 | Local assurance CI shall keep live Windows workflow replay as an explicit next step and shall emit a machine-readable escalation to `priority:workflow:replay:windows:vi-history` instead of invoking the replay lane implicitly during packet selection. | The autonomy selector needs to stay bounded and side-effect-aware; it should choose the next truthful live proof without hanging the whole local loop by launching a Windows replay lane automatically. | When the shared Windows surface and governed live-history candidate are ready, `vi-history-local-ci.mjs` emits `vi-history-local-next-step.json` with required surface `vi-history-windows-workflow-replay`, governing requirement `REQ-VHLP-010`, blocked requirement `REQ-VHLP-001`, and the explicit replay command instead of running the replay lane during selector execution. | `TEST-VHLP-010` | +| REQ-VHLP-011 | The governed Windows workflow replay lane shall terminate or fail closed within bounded helper-process timeouts, and it shall still emit a receipt when those bounds are exceeded. | A replay surface that can emit artifacts yet leave the local loop hanging is not a trustworthy autonomous proof surface. | `windows-workflow-replay-lane.mjs` enforces bounded timeouts on the host-preflight helper and compare helper, returns exit code `124` on timeout, and still writes `windows-workflow-replay-lane@v1` with the timeout failure detail. | `TEST-VHLP-011` | +| REQ-VHLP-012 | Local assurance CI shall consume a successful governed Windows workflow replay receipt and treat that as satisfied local replay proof instead of repeatedly selecting the same replay step. | An autonomous loop should advance once explicit replay proof exists; otherwise it will keep recommending a step that has already been completed. | When `vi-history-scenarios-windows-receipt.json` exists with `result.status=passed`, `vi-history-local-ci.mjs` marks the replay proof check as `pass` and advances to the next truthful packet or program step instead of re-emitting `vi-history-windows-workflow-replay`. | `TEST-VHLP-012` | diff --git a/docs/requirements-windows-docker-shared-surface-srs.md b/docs/requirements-windows-docker-shared-surface-srs.md index 6e024787d..a44e2b159 100644 --- a/docs/requirements-windows-docker-shared-surface-srs.md +++ b/docs/requirements-windows-docker-shared-surface-srs.md @@ -3,7 +3,7 @@ ## Document Control - System: Windows Docker shared local proof surface -- Version: `v0.2.1` +- Version: `v0.2.3` - Owner: `#2069` - Status: Active @@ -15,8 +15,9 @@ chosen. - In scope: Bounded readiness probes, deterministic host bootstrap and preflight, - OneDrive-safe local path hygiene, local assurance CI, and shared local proof - program integration. + OneDrive-safe local path hygiene, local assurance CI, shared local proof + program integration, and authoritative CI proof ownership for Windows + image-backed binary-handling surfaces. - Out of scope: Product-layer Pester execution, VI History replay semantics, and hosted trust routing. @@ -42,3 +43,5 @@ | REQ-WDSS-006 | The shared Windows surface packet shall participate in the shared local proof program selector so it can be chosen explicitly beside Pester and VI History. | Once the Windows surface becomes a first-class packet, the autonomy loop should be able to reason about it directly rather than only through merged packet advisories. | `priority:program:local-ci` consumes the shared-surface next-step artifact, can select a shared-surface requirement ahead of sibling packet escalations, and merges common `windows-docker-desktop-ni-image` handoffs across all packets. | `TEST-WDSS-006` | | REQ-WDSS-007 | When a reachable Windows host bridge exists behind a Unix or WSL coordinator, the shared Windows surface packet shall use that governed bridge to execute Windows-local probe and preflight work before emitting a host-unavailable escalation. | A WSL-based operator should not stop at a human handoff if the actual Windows Docker Desktop + NI image surface is already reachable from the same session. | `windows-host-bridge.mjs` resolves the reachable Windows PowerShell and Node surfaces, the shared-surface local CI uses that bridge to run `Invoke-PesterWindowsContainerSurfaceProbe.ps1` and `Test-WindowsNI2026q1HostPreflight.ps1`, and host-unavailable escalation is only emitted when that bridge is absent or the Windows surface still reports non-ready. | `TEST-WDSS-007` | | REQ-WDSS-008 | When the coordinator is running from a UNC-backed WSL or other non-bindable Windows path, the shared Windows Docker surface shall stage container-bound inputs and output targets into a governed Windows-local mount root and synchronize artifacts back to the requested repo paths. | Windows Docker bind mounts do not reliably accept UNC-backed WSL paths, so local proof needs an explicit staging contract instead of rediscovering mount-spec failures at runtime. | `Run-NIWindowsContainerCompare.ps1` detects UNC-backed container-bound inputs or report paths, emits staging metadata in `ni-windows-container-capture.json`, uses a Windows-local stage root for Docker bind mounts, synchronizes report artifacts back to the requested repo paths, and records cleanup status. | `TEST-WDSS-008` | +| REQ-WDSS-009 | CI gates for Windows image-backed binary-handling surfaces shall use the shared Windows NI image proof as the blocking execution truth; supporting static invariants may run beside that proof, but the gate shall not route through `pester-reusable.yml` as its authoritative executor. | The image-backed surface is closer to product truth than a generic Pester harness, so CI should block on the Windows NI proof path and treat Pester as secondary harness truth for this class of surface. | `.github/workflows/vi-binary-gate.yml` calls `.github/workflows/windows-ni-proof-reusable.yml`, that reusable workflow runs `Test-VIBinaryHandlingInvariants.ps1`, `Test-WindowsNI2026q1HostPreflight.ps1`, and `Run-NIWindowsContainerCompare.ps1`, and neither workflow routes the gate through `.github/workflows/pester-reusable.yml`. | `TEST-WDSS-009` | +| REQ-WDSS-010 | Shared Windows host preflight and runtime-manager Docker operations shall fail closed on bounded command timeouts, and the hosted Windows NI proof workflow shall bound the preflight step independently of the job timeout. | An authoritative Windows image-backed CI gate must classify runner or Docker hangs explicitly instead of stalling until the full job timeout expires. | `Test-WindowsNI2026q1HostPreflight.ps1` and `Invoke-DockerRuntimeManager.ps1` apply explicit Docker command, pull, and runtime-probe timeouts with machine-readable timeout classifications, and `.github/workflows/windows-ni-proof-reusable.yml` sets a bounded timeout on the hosted preflight step. | `TEST-WDSS-010` | diff --git a/docs/rtm-pester-service-model.csv b/docs/rtm-pester-service-model.csv index 0094af634..6df3fc816 100644 --- a/docs/rtm-pester-service-model.csv +++ b/docs/rtm-pester-service-model.csv @@ -27,3 +27,4 @@ REQ-PSM-025,"A local Windows-container surrogate proof surface emits an explicit REQ-PSM-026,"The local autonomy loop consumes proof checks and reopens implemented requirements when representative local proof regresses","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-026,"tools/priority/__tests__/pester-service-model-local-ci.test.mjs;tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs","tools/priority/pester-service-model-local-ci.mjs;docs/schemas/pester-service-model-local-ci-report-v1.schema.json;tools/priority/pester-service-model-autonomy-policy.json",Implemented REQ-PSM-027,"When the next truthful proof surface is unavailable from the current host, local assurance CI emits a machine-readable escalation step instead of a human-only advisory","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-027,"tools/priority/__tests__/pester-service-model-local-ci.test.mjs;tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs","tools/priority/pester-service-model-local-ci.mjs;docs/schemas/pester-service-model-local-ci-report-v1.schema.json;docs/schemas/pester-service-model-next-step-v1.schema.json;package.json",Implemented REQ-PSM-028,"The Pester packet participates in the shared local proof program selector so requirement choice and shared-surface escalations are reconciled once for the whole local loop","docs/requirements-pester-service-model-srs.md",High,TEST-PSM-028,"tools/priority/__tests__/comparevi-local-program-ci.test.mjs;tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs","tools/priority/comparevi-local-program-ci.mjs;tools/priority/pester-service-model-local-ci.mjs;tools/priority/vi-history-local-ci.mjs;docs/schemas/comparevi-local-program-ci-report-v1.schema.json;docs/schemas/comparevi-local-program-next-step-v1.schema.json;package.json",Implemented +REQ-PSM-029,"For Windows image-backed binary-handling CI surfaces, the Pester packet remains secondary harness/evidence truth and does not own the blocking execution gate","docs/requirements-pester-service-model-srs.md",Medium,TEST-PSM-029,"tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs;tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs","docs/knowledgebase/Pester-Service-Model.md;.github/workflows/vi-binary-gate.yml;.github/workflows/windows-ni-proof-reusable.yml",Implemented diff --git a/docs/rtm-vi-history-local-proof.csv b/docs/rtm-vi-history-local-proof.csv index c8b571c53..091ef2857 100644 --- a/docs/rtm-vi-history-local-proof.csv +++ b/docs/rtm-vi-history-local-proof.csv @@ -8,3 +8,6 @@ REQ-VHLP-006,"When the next truthful VI History proof surface is unavailable fro REQ-VHLP-007,"The VI History packet participates in the shared local proof program selector so it can be chosen explicitly against sibling packets and share merged shared-surface handoffs","docs/requirements-vi-history-local-proof-srs.md",High,TEST-VHLP-007,"tools/priority/__tests__/comparevi-local-program-ci.test.mjs;tools/priority/__tests__/vi-history-local-proof-contract.test.mjs","tools/priority/comparevi-local-program-ci.mjs;tools/priority/vi-history-local-ci.mjs;tools/priority/pester-service-model-local-ci.mjs;docs/schemas/comparevi-local-program-ci-report-v1.schema.json;docs/schemas/comparevi-local-program-next-step-v1.schema.json;package.json",Implemented REQ-VHLP-008,"The VI History packet governs a clone-backed live-history iteration candidate, initially ni/labview-icon-editor plus Tooling/deployment/VIP_Pre-Uninstall Custom Action.vi","docs/requirements-vi-history-local-proof-srs.md",High,TEST-VHLP-008,"tools/priority/__tests__/vi-history-local-proof-contract.test.mjs","tools/priority/vi-history-live-candidate.json;docs/schemas/vi-history-live-candidate-v1.schema.json;docs/knowledgebase/VI-History-Local-Proof.md",Implemented REQ-VHLP-009,"Local assurance CI validates clone presence, target path presence, and git history for the governed live-history candidate before Windows replay or hosted proof is chosen","docs/requirements-vi-history-local-proof-srs.md",High,TEST-VHLP-009,"tools/priority/__tests__/vi-history-local-ci.test.mjs;tools/priority/__tests__/vi-history-local-proof-contract.test.mjs","tools/priority/vi-history-local-ci.mjs;tools/priority/vi-history-live-candidate.json;docs/schemas/vi-history-live-candidate-v1.schema.json;docs/schemas/vi-history-live-candidate-readiness-v1.schema.json",Implemented +REQ-VHLP-010,"Local assurance CI keeps live Windows workflow replay as an explicit next step and emits a machine-readable replay escalation instead of invoking the replay lane implicitly during packet selection","docs/requirements-vi-history-local-proof-srs.md",High,TEST-VHLP-010,"tools/priority/__tests__/vi-history-local-ci.test.mjs;tools/priority/__tests__/vi-history-local-proof-contract.test.mjs","tools/priority/vi-history-local-ci.mjs;package.json;docs/knowledgebase/VI-History-Local-Proof.md;docs/architecture/vi-history-local-proof-control-plane.md",Implemented +REQ-VHLP-011,"The governed Windows workflow replay lane terminates or fails closed within bounded helper-process timeouts and still emits a receipt when those bounds are exceeded","docs/requirements-vi-history-local-proof-srs.md",High,TEST-VHLP-011,"tools/priority/__tests__/windows-workflow-replay-lane.test.mjs;tools/priority/__tests__/vi-history-local-proof-contract.test.mjs","tools/priority/windows-workflow-replay-lane.mjs;docs/knowledgebase/VI-History-Local-Proof.md;docs/architecture/vi-history-local-proof-control-plane.md",Implemented +REQ-VHLP-012,"Local assurance CI consumes a successful governed Windows workflow replay receipt and treats that as satisfied local replay proof instead of repeatedly selecting the same replay step","docs/requirements-vi-history-local-proof-srs.md",High,TEST-VHLP-012,"tools/priority/__tests__/vi-history-local-ci.test.mjs;tools/priority/__tests__/vi-history-local-proof-contract.test.mjs","tools/priority/vi-history-local-ci.mjs;tests/results/docker-tools-parity/workflow-replay/vi-history-scenarios-windows-receipt.json;docs/knowledgebase/VI-History-Local-Proof.md",Implemented diff --git a/docs/rtm-windows-docker-shared-surface.csv b/docs/rtm-windows-docker-shared-surface.csv index a8e6bcca9..b32513e08 100644 --- a/docs/rtm-windows-docker-shared-surface.csv +++ b/docs/rtm-windows-docker-shared-surface.csv @@ -7,3 +7,5 @@ REQ-WDSS-005,"When the next truthful shared-surface proof is unavailable from th REQ-WDSS-006,"The shared Windows surface packet participates in the shared local proof program selector beside Pester and VI History","docs/requirements-windows-docker-shared-surface-srs.md",High,TEST-WDSS-006,"tools/priority/__tests__/comparevi-local-program-ci.test.mjs;tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs","tools/priority/comparevi-local-program-ci.mjs;tools/priority/windows-docker-shared-surface-local-ci.mjs;docs/knowledgebase/Local-Proof-Autonomy-Program.md;package.json",Implemented REQ-WDSS-007,"When a reachable Windows host exists behind a Unix or WSL coordinator, the shared Windows surface packet uses a governed bridge before emitting host-unavailable escalation","docs/requirements-windows-docker-shared-surface-srs.md",High,TEST-WDSS-007,"tools/priority/__tests__/windows-host-bridge.test.mjs;tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs","tools/priority/windows-host-bridge.mjs;tools/priority/windows-docker-shared-surface-local-ci.mjs;tools/Test-WindowsNI2026q1HostPreflight.ps1;tools/Run-NIWindowsContainerCompare.ps1",Implemented REQ-WDSS-008,"When the coordinator is running from a UNC-backed WSL or other non-bindable Windows path, the shared Windows Docker surface stages container-bound inputs and output targets into a governed Windows-local mount root and synchronizes artifacts back","docs/requirements-windows-docker-shared-surface-srs.md",High,TEST-WDSS-008,"tests/Run-NIWindowsContainerCompare.Tests.ps1;tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs;tools/priority/__tests__/vi-history-local-proof-contract.test.mjs","tools/Run-NIWindowsContainerCompare.ps1;tools/priority/windows-workflow-replay-lane.mjs;docs/knowledgebase/Windows-Docker-Shared-Surface.md",Implemented +REQ-WDSS-009,"CI gates for Windows image-backed binary-handling surfaces use the shared Windows NI image proof as blocking execution truth and do not route that authority through pester-reusable","docs/requirements-windows-docker-shared-surface-srs.md",High,TEST-WDSS-009,"tools/priority/__tests__/windows-ni-proof-workflow-contract.test.mjs;tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs;tests/ViBinaryHandling.Tests.ps1",".github/workflows/vi-binary-gate.yml;.github/workflows/windows-ni-proof-reusable.yml;tools/Test-VIBinaryHandlingInvariants.ps1;tools/Test-WindowsNI2026q1HostPreflight.ps1;tools/Run-NIWindowsContainerCompare.ps1",Implemented +REQ-WDSS-010,"Shared Windows host preflight and runtime-manager Docker operations fail closed on bounded command timeouts, and the hosted Windows NI proof workflow bounds the preflight step independently of the job timeout","docs/requirements-windows-docker-shared-surface-srs.md",High,TEST-WDSS-010,"tests/Test-WindowsNI2026q1HostPreflight.Tests.ps1;tests/Invoke-DockerRuntimeManager.Tests.ps1;tools/priority/__tests__/windows-ni-proof-workflow-contract.test.mjs","tools/Test-WindowsNI2026q1HostPreflight.ps1;tools/Invoke-DockerRuntimeManager.ps1;.github/workflows/windows-ni-proof-reusable.yml",Implemented diff --git a/docs/testing/pester-service-model-test-plan.md b/docs/testing/pester-service-model-test-plan.md index 686b9fff1..b2115312a 100644 --- a/docs/testing/pester-service-model-test-plan.md +++ b/docs/testing/pester-service-model-test-plan.md @@ -3,7 +3,7 @@ ## Overview - Release or baseline: - Pester service-model assurance packet `v0.1.20` + Pester service-model assurance packet `v0.1.21` - Owner: `#2069` with retained fork basis on `#2078` - Scope: @@ -47,6 +47,7 @@ | `TEST-PSM-026` proof-check aware autonomy coverage | Assurance/Contract | High | Verifies local CI consumes representative replay and Windows-surface proof checks and reopens implemented requirements when representative proof regresses | | `TEST-PSM-027` next-step escalation coverage | Assurance/Contract | High | Verifies local CI emits a machine-readable escalation step when the next truthful proof surface is unavailable from the current host | | `TEST-PSM-028` shared local-program selector coverage | Assurance/Contract | High | Verifies the shared program selector can choose a Pester requirement ahead of sibling escalations and merge the shared Windows Docker Desktop + NI image escalation across packets | +| `TEST-PSM-029` secondary-authority coverage | Workflow/Governance | Medium | Verifies Windows image-backed CI gates use the shared Windows NI proof path as blocking execution truth and that the Pester packet documents itself as secondary on that surface | ## Entry Criteria @@ -93,6 +94,7 @@ | Proof-check aware autonomy coverage | Local CI reopens implemented requirements when representative local proof regresses and records advisory surface status | `tools/priority/pester-service-model-local-ci.mjs`, `TEST-PSM-026` | | Next-step escalation coverage | Local CI emits `pester-service-model-next-step.json` with a governed escalation packet when the next truthful proof surface is unavailable from the current host | `tools/priority/pester-service-model-local-ci.mjs`, `docs/schemas/pester-service-model-next-step-v1.schema.json`, `TEST-PSM-027` | | Shared local-program selector coverage | Shared local CI emits `comparevi-local-program-next-step.json`, chooses requirement work ahead of sibling packet escalations, and merges the shared Windows Docker Desktop + NI image handoff across packets | `tools/priority/__tests__/comparevi-local-program-ci.test.mjs`, `tools/priority/comparevi-local-program-ci.mjs`, `docs/schemas/comparevi-local-program-next-step-v1.schema.json`, `TEST-PSM-028` | +| Secondary-authority coverage | The Pester packet documents Windows image-backed CI proof as authoritative on the shared Windows NI surface, and `vi-binary-gate.yml` no longer routes through `pester-reusable.yml` | `tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs`, `tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs`, `TEST-PSM-029` | | Packet coverage gate | Retained `coverage.xml` and named PR coverage gate | `.github/workflows/pester-service-model-quality.yml` | | Promotion bundle retention | Hosted bundle retains the minimal promotion handoff | `.github/workflows/pester-service-model-release-evidence.yml` | diff --git a/docs/testing/vi-history-local-proof-test-plan.md b/docs/testing/vi-history-local-proof-test-plan.md index 6003839a1..4ebd390eb 100644 --- a/docs/testing/vi-history-local-proof-test-plan.md +++ b/docs/testing/vi-history-local-proof-test-plan.md @@ -3,7 +3,7 @@ ## Document Control - System: VI History local proof control plane -- Version: `v0.1.2` +- Version: `v0.1.5` - Status: Active ## Verification Matrix @@ -19,6 +19,9 @@ | `TEST-VHLP-007` shared local-program selector coverage | Assurance/Contract | High | Verifies the shared program selector can choose VI History requirement work explicitly and merge the shared Windows Docker Desktop + NI image escalation across sibling packets | | `TEST-VHLP-008` clone-backed live-history candidate governance coverage | Assurance/Contract | High | Verifies the packet names `ni/labview-icon-editor` plus `Tooling/deployment/VIP_Pre-Uninstall Custom Action.vi` as the governed clone-backed live-history candidate | | `TEST-VHLP-009` live-history candidate readiness coverage | Assurance/Contract | High | Verifies local VI History CI validates clone presence, target path presence, and git history, then emits a bounded clone-preparation escalation when the candidate is unavailable | +| `TEST-VHLP-010` explicit Windows replay next-step coverage | Assurance/Contract | High | Verifies local VI History CI emits `vi-history-windows-workflow-replay` as the next step when the shared Windows surface and live-history candidate are ready, instead of launching the replay lane during packet selection | +| `TEST-VHLP-011` bounded Windows replay lifecycle coverage | Contract/Replay | High | Verifies the governed Windows workflow replay lane terminates or fails closed within bounded helper-process timeouts and still emits a replay receipt on timeout | +| `TEST-VHLP-012` replay receipt consumption coverage | Assurance/Contract | High | Verifies local VI History CI treats an existing passing `vi-history-scenarios-windows` replay receipt as satisfied local replay proof and advances beyond replay re-selection | ## Entry Criteria @@ -45,3 +48,6 @@ | Shared local-program selector coverage | Shared local CI emits `comparevi-local-program-next-step.json`, selects VI History explicitly when it owns the next requirement, and merges the shared Windows surface handoff across packets | `tools/priority/__tests__/comparevi-local-program-ci.test.mjs`, `tools/priority/comparevi-local-program-ci.mjs`, `docs/schemas/comparevi-local-program-next-step-v1.schema.json`, `TEST-VHLP-007` | | Clone-backed live-history candidate governance coverage | The governed candidate manifest names `ni/labview-icon-editor` and `Tooling/deployment/VIP_Pre-Uninstall Custom Action.vi` explicitly | `tools/priority/__tests__/vi-history-local-proof-contract.test.mjs`, `tools/priority/vi-history-live-candidate.json`, `TEST-VHLP-008` | | Live-history candidate readiness coverage | Local VI History CI emits `vi-history-live-candidate-readiness.json` and escalates clone preparation when the governed target is unavailable | `tools/priority/__tests__/vi-history-local-ci.test.mjs`, `tools/priority/vi-history-local-ci.mjs`, `docs/schemas/vi-history-live-candidate-v1.schema.json`, `docs/schemas/vi-history-live-candidate-readiness-v1.schema.json`, `TEST-VHLP-009` | +| Explicit Windows replay next-step coverage | Local VI History CI emits `vi-history-local-next-step.json` pointing at `priority:workflow:replay:windows:vi-history` when the packet is ready for live replay, instead of invoking the replay lane during selector execution | `tools/priority/__tests__/vi-history-local-ci.test.mjs`, `tools/priority/__tests__/vi-history-local-proof-contract.test.mjs`, `tools/priority/vi-history-local-ci.mjs`, `TEST-VHLP-010` | +| Bounded Windows replay lifecycle coverage | `windows-workflow-replay-lane.mjs` enforces bounded helper-process timeouts and still writes `windows-workflow-replay-lane@v1` when a timeout occurs | `tools/priority/__tests__/windows-workflow-replay-lane.test.mjs`, `tools/priority/windows-workflow-replay-lane.mjs`, `TEST-VHLP-011` | +| Replay receipt consumption coverage | `vi-history-local-ci.mjs` consumes a passing `vi-history-scenarios-windows` replay receipt and stops re-emitting the same replay escalation as the next step | `tools/priority/__tests__/vi-history-local-ci.test.mjs`, `tools/priority/vi-history-local-ci.mjs`, `TEST-VHLP-012` | diff --git a/docs/testing/windows-docker-shared-surface-test-plan.md b/docs/testing/windows-docker-shared-surface-test-plan.md index 8ee00022e..acc9a2edb 100644 --- a/docs/testing/windows-docker-shared-surface-test-plan.md +++ b/docs/testing/windows-docker-shared-surface-test-plan.md @@ -3,7 +3,7 @@ ## Document Control - System: Windows Docker shared local proof surface -- Version: `v0.2.1` +- Version: `v0.2.3` - Status: Active ## Verification Matrix @@ -18,6 +18,8 @@ | `TEST-WDSS-006` shared local-program selector coverage | Assurance/Contract | High | Verifies the shared surface participates explicitly in the program selector beside Pester and VI History | | `TEST-WDSS-007` reachable Windows host bridge coverage | Assurance/Contract | High | Verifies a Unix or WSL coordinator uses a reachable Windows host bridge for probe and preflight work before emitting host-unavailable escalation | | `TEST-WDSS-008` UNC-backed WSL staging coverage | Runtime/Windows Docker | High | Verifies UNC-backed WSL inputs and report paths are staged into a Windows-local mount root, synchronized back, and cleaned up after compare execution | +| `TEST-WDSS-009` authoritative CI gate coverage | Workflow/Contract | High | Verifies Windows image-backed CI gates route through the shared Windows NI proof workflow and not through the generic Pester reusable workflow | +| `TEST-WDSS-010` bounded timeout coverage | Runtime/Workflow | High | Verifies Windows preflight and runtime-manager Docker operations fail closed on timeout and the hosted preflight step carries an explicit workflow timeout | ## Entry Criteria @@ -42,3 +44,5 @@ | Shared local-program selector coverage | Program CI consumes the shared-surface next-step artifact beside Pester and VI History | `tools/priority/__tests__/comparevi-local-program-ci.test.mjs`, `tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs`, `TEST-WDSS-006` | | Reachable Windows host bridge coverage | Shared-surface local CI can consume a reachable Windows host bridge from a Unix or WSL coordinator and only escalates when that bridge is absent or still non-ready | `tools/priority/__tests__/windows-host-bridge.test.mjs`, `tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs`, `TEST-WDSS-007` | | UNC-backed WSL staging coverage | `Run-NIWindowsContainerCompare.ps1` stages UNC-backed or otherwise non-bindable Windows paths into a local Windows mount root, syncs artifacts back, and records staging status in capture output | `tests/Run-NIWindowsContainerCompare.Tests.ps1`, `tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs`, `tools/priority/__tests__/vi-history-local-proof-contract.test.mjs`, `TEST-WDSS-008` | +| Authoritative CI gate coverage | `vi-binary-gate.yml` routes the blocking binary-handling gate through `windows-ni-proof-reusable.yml`, which runs static invariants plus the hosted Windows NI preflight and compare proof without delegating authority to `pester-reusable.yml` | `tools/priority/__tests__/windows-ni-proof-workflow-contract.test.mjs`, `tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs`, `tests/ViBinaryHandling.Tests.ps1`, `TEST-WDSS-009` | +| Bounded timeout coverage | Hosted preflight and shared runtime-manager Docker commands fail closed on pull/probe timeout, and the reusable Windows NI proof workflow bounds the preflight step separately from the job timeout | `tests/Test-WindowsNI2026q1HostPreflight.Tests.ps1`, `tests/Invoke-DockerRuntimeManager.Tests.ps1`, `tools/priority/__tests__/windows-ni-proof-workflow-contract.test.mjs`, `TEST-WDSS-010` | diff --git a/tests/Invoke-DockerRuntimeManager.Tests.ps1 b/tests/Invoke-DockerRuntimeManager.Tests.ps1 index 897e5772c..79753c7b9 100644 --- a/tests/Invoke-DockerRuntimeManager.Tests.ps1 +++ b/tests/Invoke-DockerRuntimeManager.Tests.ps1 @@ -95,6 +95,10 @@ if ($Args[0] -eq 'context' -and $Args.Count -ge 3 -and $Args[1] -eq 'use') { } if ($Args[0] -eq 'info') { + $infoSleep = [Environment]::GetEnvironmentVariable('DOCKER_STUB_INFO_SLEEP_SECONDS') + if (-not [string]::IsNullOrWhiteSpace($infoSleep)) { + Start-Sleep -Seconds ([int]$infoSleep) + } if ([Environment]::GetEnvironmentVariable('DOCKER_STUB_FORCE_INFO_FAILURE') -eq '1') { [Console]::Error.WriteLine('docker info failed') exit 1 @@ -129,6 +133,10 @@ if ($Args[0] -eq 'manifest' -and $Args.Count -ge 3 -and $Args[1] -eq 'inspect') } if ($Args[0] -eq 'image' -and $Args.Count -ge 3 -and $Args[1] -eq 'inspect') { + $inspectSleep = [Environment]::GetEnvironmentVariable('DOCKER_STUB_INSPECT_SLEEP_SECONDS') + if (-not [string]::IsNullOrWhiteSpace($inspectSleep)) { + Start-Sleep -Seconds ([int]$inspectSleep) + } $image = [string]$Args[2] $requirePullWindows = ([Environment]::GetEnvironmentVariable('DOCKER_STUB_REQUIRE_PULL_WINDOWS') -eq '1') $requirePullLinux = ([Environment]::GetEnvironmentVariable('DOCKER_STUB_REQUIRE_PULL_LINUX') -eq '1') @@ -154,6 +162,11 @@ if ($Args[0] -eq 'image' -and $Args.Count -ge 3 -and $Args[1] -eq 'inspect') { if ($Args[0] -eq 'pull' -and $Args.Count -ge 2) { $image = [string]$Args[1] + $pullSleepVar = if (Is-WindowsImage -Image $image) { 'DOCKER_STUB_PULL_SLEEP_WINDOWS' } else { 'DOCKER_STUB_PULL_SLEEP_LINUX' } + $pullSleep = [Environment]::GetEnvironmentVariable($pullSleepVar) + if (-not [string]::IsNullOrWhiteSpace($pullSleep)) { + Start-Sleep -Seconds ([int]$pullSleep) + } if (([Environment]::GetEnvironmentVariable('DOCKER_STUB_PULL_FAIL_WINDOWS') -eq '1') -and (Is-WindowsImage -Image $image)) { [Console]::Error.WriteLine(("pull denied for {0}" -f $image)) exit 1 @@ -173,6 +186,18 @@ if ($Args[0] -eq 'pull' -and $Args.Count -ge 2) { if ($Args[0] -eq 'run') { $joined = ($Args -join ' ') + if ($joined -match '(?i)windows') { + $runSleep = [Environment]::GetEnvironmentVariable('DOCKER_STUB_RUN_SLEEP_WINDOWS') + if (-not [string]::IsNullOrWhiteSpace($runSleep)) { + Start-Sleep -Seconds ([int]$runSleep) + } + } + if ($joined -match '(?i)linux') { + $runSleep = [Environment]::GetEnvironmentVariable('DOCKER_STUB_RUN_SLEEP_LINUX') + if (-not [string]::IsNullOrWhiteSpace($runSleep)) { + Start-Sleep -Seconds ([int]$runSleep) + } + } $runFailWindows = ([Environment]::GetEnvironmentVariable('DOCKER_STUB_RUN_FAIL_WINDOWS') -eq '1') $runFailLinux = ([Environment]::GetEnvironmentVariable('DOCKER_STUB_RUN_FAIL_LINUX') -eq '1') if ($runFailWindows -and $joined -match '(?i)windows') { @@ -198,6 +223,7 @@ exit 0 Set-Content -LiteralPath (Join-Path $binDir 'docker.cmd') -Value $dockerCmd -Encoding ascii $env:PATH = "{0};{1}" -f $binDir, $env:PATH + $env:DOCKER_COMMAND_OVERRIDE = (Join-Path $binDir 'docker.ps1') } } @@ -214,6 +240,13 @@ exit 0 DOCKER_STUB_PULL_FAIL_LINUX = $env:DOCKER_STUB_PULL_FAIL_LINUX DOCKER_STUB_RUN_FAIL_WINDOWS = $env:DOCKER_STUB_RUN_FAIL_WINDOWS DOCKER_STUB_RUN_FAIL_LINUX = $env:DOCKER_STUB_RUN_FAIL_LINUX + DOCKER_STUB_INFO_SLEEP_SECONDS = $env:DOCKER_STUB_INFO_SLEEP_SECONDS + DOCKER_STUB_INSPECT_SLEEP_SECONDS = $env:DOCKER_STUB_INSPECT_SLEEP_SECONDS + DOCKER_STUB_PULL_SLEEP_WINDOWS = $env:DOCKER_STUB_PULL_SLEEP_WINDOWS + DOCKER_STUB_PULL_SLEEP_LINUX = $env:DOCKER_STUB_PULL_SLEEP_LINUX + DOCKER_STUB_RUN_SLEEP_WINDOWS = $env:DOCKER_STUB_RUN_SLEEP_WINDOWS + DOCKER_STUB_RUN_SLEEP_LINUX = $env:DOCKER_STUB_RUN_SLEEP_LINUX + DOCKER_COMMAND_OVERRIDE = $env:DOCKER_COMMAND_OVERRIDE RUNNER_TEMP = $env:RUNNER_TEMP } } @@ -368,6 +401,61 @@ exit 0 $json.probes.windows.status | Should -Be 'success' } + It 'fails closed with image-bootstrap-timeout when a windows image pull exceeds the allowed bound' { + $work = Join-Path $TestDrive 'pull-timeout' + New-Item -ItemType Directory -Path $work -Force | Out-Null + & $script:CreateDockerStub -WorkRoot $work + + Set-Item Env:DOCKER_STUB_STATE_PATH (Join-Path $work 'docker-state.json') + Set-Item Env:DOCKER_STUB_INITIAL_CONTEXT 'desktop-windows' + Set-Item Env:DOCKER_STUB_REQUIRE_PULL_WINDOWS '1' + Set-Item Env:DOCKER_STUB_PULL_SLEEP_WINDOWS '6' + Set-Item Env:RUNNER_TEMP (Join-Path $work 'runner-temp') + + $jsonPath = Join-Path $work 'docker-runtime-manager.json' + $output = @(& pwsh -NoLogo -NoProfile -File $script:ManagerScript ` + -ProbeScope windows ` + -OutputJsonPath $jsonPath ` + -BootstrapPullTimeoutSeconds 5 ` + -SwitchRetryCount 1 ` + -SwitchTimeoutSeconds 30 2>&1) + + $LASTEXITCODE | Should -Not -Be 0 + ($output -join "`n") | Should -Match 'docker pull timed out' + + $json = Get-Content -LiteralPath $jsonPath -Raw | ConvertFrom-Json -Depth 30 + $json.status | Should -Be 'failure' + $json.failureClass | Should -Be 'image-bootstrap-timeout' + $json.probes.windows.bootstrap.pullError | Should -Match 'docker pull timed out' + } + + It 'fails closed with runtime-probe-timeout when a windows runtime probe exceeds the allowed bound' { + $work = Join-Path $TestDrive 'runtime-probe-timeout' + New-Item -ItemType Directory -Path $work -Force | Out-Null + & $script:CreateDockerStub -WorkRoot $work + + Set-Item Env:DOCKER_STUB_STATE_PATH (Join-Path $work 'docker-state.json') + Set-Item Env:DOCKER_STUB_INITIAL_CONTEXT 'desktop-windows' + Set-Item Env:DOCKER_STUB_RUN_SLEEP_WINDOWS '6' + Set-Item Env:RUNNER_TEMP (Join-Path $work 'runner-temp') + + $jsonPath = Join-Path $work 'docker-runtime-manager.json' + $output = @(& pwsh -NoLogo -NoProfile -File $script:ManagerScript ` + -ProbeScope windows ` + -OutputJsonPath $jsonPath ` + -ProbeTimeoutSeconds 5 ` + -SwitchRetryCount 1 ` + -SwitchTimeoutSeconds 30 2>&1) + + $LASTEXITCODE | Should -Not -Be 0 + ($output -join "`n") | Should -Match 'Runtime probe failed' + + $json = Get-Content -LiteralPath $jsonPath -Raw | ConvertFrom-Json -Depth 30 + $json.status | Should -Be 'failure' + $json.failureClass | Should -Be 'runtime-probe-timeout' + $json.probes.windows.probe.status | Should -Be 'timeout' + } + It 'fails with lock timeout when the runtime manager lock is held by another process' { $work = Join-Path $TestDrive 'lock-timeout' New-Item -ItemType Directory -Path $work -Force | Out-Null diff --git a/tests/Test-WindowsNI2026q1HostPreflight.Tests.ps1 b/tests/Test-WindowsNI2026q1HostPreflight.Tests.ps1 index 3935b0882..09e2dc340 100644 --- a/tests/Test-WindowsNI2026q1HostPreflight.Tests.ps1 +++ b/tests/Test-WindowsNI2026q1HostPreflight.Tests.ps1 @@ -77,6 +77,10 @@ if ($Args[0] -eq 'info') { } if ($Args[0] -eq 'image' -and $Args.Count -ge 2 -and $Args[1] -eq 'inspect') { + $inspectSleep = [Environment]::GetEnvironmentVariable('DOCKER_STUB_INSPECT_SLEEP_SECONDS') + if (-not [string]::IsNullOrWhiteSpace($inspectSleep)) { + Start-Sleep -Seconds ([int]$inspectSleep) + } $exists = [Environment]::GetEnvironmentVariable('DOCKER_STUB_IMAGE_EXISTS') if ($exists -eq '1') { Write-Output '{"Id":"sha256:synthetic","RepoDigests":["nationalinstruments/labview:2026q1-windows@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"]}' @@ -87,11 +91,19 @@ if ($Args[0] -eq 'image' -and $Args.Count -ge 2 -and $Args[1] -eq 'inspect') { } if ($Args[0] -eq 'pull') { + $pullSleep = [Environment]::GetEnvironmentVariable('DOCKER_STUB_PULL_SLEEP_SECONDS') + if (-not [string]::IsNullOrWhiteSpace($pullSleep)) { + Start-Sleep -Seconds ([int]$pullSleep) + } Write-Output 'pulled' exit 0 } if ($Args[0] -eq 'run') { + $runSleep = [Environment]::GetEnvironmentVariable('DOCKER_STUB_RUN_SLEEP_SECONDS') + if (-not [string]::IsNullOrWhiteSpace($runSleep)) { + Start-Sleep -Seconds ([int]$runSleep) + } $runStderr = [Environment]::GetEnvironmentVariable('DOCKER_STUB_RUN_STDERR') if (-not [string]::IsNullOrWhiteSpace($runStderr)) { [Console]::Error.WriteLine($runStderr) @@ -139,6 +151,7 @@ exit 0 Set-Content -LiteralPath (Join-Path $binDir 'wsl.cmd') -Value $wslCmd -Encoding ascii $env:PATH = "{0};{1}" -f $binDir, $env:PATH + $env:DOCKER_COMMAND_OVERRIDE = (Join-Path $binDir 'docker.ps1') } } @@ -152,6 +165,10 @@ exit 0 DOCKER_STUB_INFO_EXITCODE = $env:DOCKER_STUB_INFO_EXITCODE DOCKER_STUB_RUN_STDERR = $env:DOCKER_STUB_RUN_STDERR DOCKER_STUB_RUN_EXITCODE = $env:DOCKER_STUB_RUN_EXITCODE + DOCKER_STUB_INSPECT_SLEEP_SECONDS = $env:DOCKER_STUB_INSPECT_SLEEP_SECONDS + DOCKER_STUB_PULL_SLEEP_SECONDS = $env:DOCKER_STUB_PULL_SLEEP_SECONDS + DOCKER_STUB_RUN_SLEEP_SECONDS = $env:DOCKER_STUB_RUN_SLEEP_SECONDS + DOCKER_COMMAND_OVERRIDE = $env:DOCKER_COMMAND_OVERRIDE } } @@ -185,6 +202,7 @@ exit 0 -Image 'nationalinstruments/labview:2026q1-windows' ` -ResultsDir $resultsRoot ` -ExecutionSurface 'github-hosted-windows' ` + -HostPlatformOverride 'Win32NT' ` -OutputJsonPath $outputJsonPath ` -GitHubOutputPath '' ` -StepSummaryPath '' 2>&1 @@ -219,6 +237,7 @@ exit 0 -ResultsDir $resultsRoot ` -ExecutionSurface 'github-hosted-windows' ` -AllowUnavailable ` + -HostPlatformOverride 'Win32NT' ` -OutputJsonPath $outputJsonPath ` -GitHubOutputPath '' ` -StepSummaryPath '' 2>&1 @@ -231,6 +250,69 @@ exit 0 $json.runtimeDeterminism.reason | Should -Be 'docker-daemon-unavailable' } + It 'fails closed with image-bootstrap-timeout when hosted docker pull exceeds the allowed bound' { + $work = Join-Path $TestDrive 'hosted-pull-timeout' + New-Item -ItemType Directory -Path $work -Force | Out-Null + & $script:CreateDockerHostedStubs -WorkRoot $work + + Set-Item Env:DOCKER_STUB_CONTEXT 'default' + Set-Item Env:DOCKER_STUB_OSTYPE 'windows' + Set-Item Env:DOCKER_STUB_INFO_JSON '{"OSType":"windows","OperatingSystem":"Windows Server 2022","Name":"github-hosted","Platform":{"Name":"Docker Engine - Community"}}' + Set-Item Env:DOCKER_STUB_PULL_SLEEP_SECONDS '6' + + $resultsRoot = Join-Path $work 'results' + $outputJsonPath = Join-Path $resultsRoot 'windows-ni-2026q1-host-preflight.json' + + $output = @(& pwsh -NoLogo -NoProfile -File $script:ToolPath ` + -Image 'nationalinstruments/labview:2026q1-windows' ` + -ResultsDir $resultsRoot ` + -ExecutionSurface 'github-hosted-windows' ` + -HostPlatformOverride 'Win32NT' ` + -OutputJsonPath $outputJsonPath ` + -BootstrapPullTimeoutSeconds 5 ` + -GitHubOutputPath '' ` + -StepSummaryPath '' 2>&1) + $LASTEXITCODE | Should -Not -Be 0 + + $json = Get-Content -LiteralPath $outputJsonPath -Raw | ConvertFrom-Json -Depth 20 + $json.status | Should -Be 'failure' + $json.failureClass | Should -Be 'image-bootstrap-timeout' + $json.bootstrap.pullError | Should -Match 'docker pull timed out' + ($output -join "`n") | Should -Match 'docker pull timed out' + } + + It 'fails closed with runtime-probe-timeout when hosted runtime probe exceeds the allowed bound' { + $work = Join-Path $TestDrive 'hosted-runtime-timeout' + New-Item -ItemType Directory -Path $work -Force | Out-Null + & $script:CreateDockerHostedStubs -WorkRoot $work + + Set-Item Env:DOCKER_STUB_CONTEXT 'default' + Set-Item Env:DOCKER_STUB_OSTYPE 'windows' + Set-Item Env:DOCKER_STUB_IMAGE_EXISTS '1' + Set-Item Env:DOCKER_STUB_INFO_JSON '{"OSType":"windows","OperatingSystem":"Windows Server 2022","Name":"github-hosted","Platform":{"Name":"Docker Engine - Community"}}' + Set-Item Env:DOCKER_STUB_RUN_SLEEP_SECONDS '6' + + $resultsRoot = Join-Path $work 'results' + $outputJsonPath = Join-Path $resultsRoot 'windows-ni-2026q1-host-preflight.json' + + $output = @(& pwsh -NoLogo -NoProfile -File $script:ToolPath ` + -Image 'nationalinstruments/labview:2026q1-windows' ` + -ResultsDir $resultsRoot ` + -ExecutionSurface 'github-hosted-windows' ` + -HostPlatformOverride 'Win32NT' ` + -OutputJsonPath $outputJsonPath ` + -RuntimeProbeTimeoutSeconds 5 ` + -GitHubOutputPath '' ` + -StepSummaryPath '' 2>&1) + $LASTEXITCODE | Should -Not -Be 0 + + $json = Get-Content -LiteralPath $outputJsonPath -Raw | ConvertFrom-Json -Depth 20 + $json.status | Should -Be 'failure' + $json.failureClass | Should -Be 'runtime-probe-timeout' + $json.probe.status | Should -Be 'timeout' + ($output -join "`n") | Should -Match 'Hosted Windows runtime probe failed' + } + It 'fails desktop-local fast and quietly when Docker Desktop is still on the Linux engine' { $work = Join-Path $TestDrive 'desktop-local-linux-engine' New-Item -ItemType Directory -Path $work -Force | Out-Null diff --git a/tests/ViBinaryHandling.Tests.ps1 b/tests/ViBinaryHandling.Tests.ps1 index c5f6f42ab..b8818c3dc 100644 --- a/tests/ViBinaryHandling.Tests.ps1 +++ b/tests/ViBinaryHandling.Tests.ps1 @@ -1,33 +1,15 @@ Describe 'VI Binary Handling Invariants' -Tag 'Unit' { - It 'declares *.vi as binary in .gitattributes' { - $attrPath = Join-Path $PSScriptRoot '..' '.gitattributes' - Test-Path $attrPath | Should -BeTrue - $content = Get-Content $attrPath -Raw - ($content -match '(?m)^\*\.vi\s+binary\s*$') | Should -BeTrue - } + It 'passes the standalone invariant contract script' { + $scriptPath = Join-Path $PSScriptRoot '..' 'tools' 'Test-VIBinaryHandlingInvariants.ps1' + $reportPath = Join-Path $TestDrive 'vi-binary-handling-invariants.json' + + { & $scriptPath -OutputJsonPath $reportPath } | Should -Not -Throw - It 'does not attempt textual reads of .vi files in scripts (grep heuristic)' { - $root = Resolve-Path (Join-Path $PSScriptRoot '..') - $psFiles = Get-ChildItem $root -Recurse -Include *.ps1,*.psm1 | Where-Object { - $_.FullName -notmatch '[/\\]tests[/\\]' -and $_.FullName -notmatch '[/\\]tools[/\\]' - } - $badPatterns = @( - 'Get-Content\s+[^\n]*\.vi', - 'ReadAllText\(', - 'StreamReader' - ) - $violations = @() - foreach ($f in $psFiles) { - $text = Get-Content $f.FullName -Raw - foreach ($pat in $badPatterns) { - if ($text -match $pat) { - $violations += [pscustomobject]@{ File=$f.FullName; Pattern=$pat } - } - } - } - if ($violations.Count -gt 0) { - $violations | Format-Table | Out-String | Write-Host - } - $violations.Count | Should -Be 0 + Test-Path -LiteralPath $reportPath | Should -BeTrue + $report = Get-Content -LiteralPath $reportPath -Raw | ConvertFrom-Json -Depth 8 + $report.schema | Should -Be 'comparevi/vi-binary-handling-invariants@v1' + $report.status | Should -Be 'passed' + @($report.checks).Count | Should -BeGreaterThan 1 + (@($report.violations).Count) | Should -Be 0 } } diff --git a/tools/Assert-DockerRuntimeDeterminism.ps1 b/tools/Assert-DockerRuntimeDeterminism.ps1 index 5de4003c7..fa6462244 100644 --- a/tools/Assert-DockerRuntimeDeterminism.ps1 +++ b/tools/Assert-DockerRuntimeDeterminism.ps1 @@ -72,6 +72,8 @@ param( [bool]$AllowHostEngineMutation = $false, + [string]$HostPlatformOverride = '', + [int]$EngineReadyTimeoutSeconds = 120, [int]$EngineReadyPollSeconds = 3, @@ -979,7 +981,15 @@ $observedIsDockerDesktop = $false $runnerOsRaw = $env:RUNNER_OS $runnerOsNormalized = if ([string]::IsNullOrWhiteSpace($runnerOsRaw)) { '' } else { $runnerOsRaw.Trim().ToLowerInvariant() } -$hostIsWindows = [bool]$IsWindows +$hostPlatform = if ([string]::IsNullOrWhiteSpace($HostPlatformOverride)) { + [string][System.Environment]::OSVersion.Platform +} else { + $HostPlatformOverride.Trim() +} +$hostIsWindows = ( + [string]::Equals($hostPlatform, 'Win32NT', [System.StringComparison]::OrdinalIgnoreCase) -or + $hostPlatform -match '(?i)windows' +) $hostAlignmentOk = $true if ($ExpectedOsType -eq 'windows') { if (-not $hostIsWindows) { diff --git a/tools/Invoke-DockerRuntimeManager.ps1 b/tools/Invoke-DockerRuntimeManager.ps1 index 4c4707ac6..fa59acfd2 100644 --- a/tools/Invoke-DockerRuntimeManager.ps1 +++ b/tools/Invoke-DockerRuntimeManager.ps1 @@ -28,6 +28,12 @@ param( [int]$SwitchRetryDelaySeconds = 4, [ValidateRange(5, 600)] [int]$LockWaitSeconds = 90, + [ValidateRange(5, 600)] + [int]$CommandTimeoutSeconds = 45, + [ValidateRange(5, 3600)] + [int]$BootstrapPullTimeoutSeconds = 900, + [ValidateRange(5, 900)] + [int]$ProbeTimeoutSeconds = 180, [string]$WindowsProbeCommand = "[Console]::WriteLine('ni-runtime-probe-ok')", [string]$LinuxProbeCommand = "echo ni-runtime-probe-ok", [string]$OutputJsonPath = 'results/fixture-drift/docker-runtime-manager.json', @@ -61,23 +67,161 @@ function Write-GitHubOutput { Add-Content -LiteralPath $dest -Value ("{0}={1}" -f $Key, ($Value ?? '')) -Encoding utf8 } +function Split-OutputLines { + param([AllowNull()][string]$Text) + + if ([string]::IsNullOrEmpty($Text)) { return @() } + return @($Text -split "(`r`n|`n|`r)" | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) +} + +function Resolve-DockerCommandSource { + $override = $env:DOCKER_COMMAND_OVERRIDE + if (-not [string]::IsNullOrWhiteSpace($override) -and (Test-Path -LiteralPath $override -PathType Leaf)) { + return [System.IO.Path]::GetFullPath($override) + } + + $pathSeparator = [System.IO.Path]::PathSeparator + $pathEntries = @($env:PATH -split [regex]::Escape([string]$pathSeparator)) + $candidates = if ($IsWindows) { + @('docker.exe', 'docker.cmd', 'docker.ps1', 'docker.bat', 'docker') + } else { + @('docker', 'docker.sh', 'docker.exe', 'docker.ps1', 'docker.cmd') + } + + foreach ($entry in $pathEntries) { + if ([string]::IsNullOrWhiteSpace($entry)) { continue } + foreach ($name in $candidates) { + $candidatePath = Join-Path $entry $name + if (Test-Path -LiteralPath $candidatePath -PathType Leaf) { + return [System.IO.Path]::GetFullPath($candidatePath) + } + } + } + + $command = Get-Command -Name 'docker' -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($null -eq $command -or [string]::IsNullOrWhiteSpace([string]$command.Source)) { + return $null + } + + return [System.IO.Path]::GetFullPath([string]$command.Source) +} + +function Invoke-ProcessWithTimeout { + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$FilePath, + [string[]]$Arguments = @(), + [int]$TimeoutSeconds = 45 + ) + + $safeTimeout = [Math]::Max(5, [int]$TimeoutSeconds) + $resolvedFilePath = $FilePath + $effectiveArguments = @($Arguments) + if ([string]::Equals($FilePath, 'docker', [System.StringComparison]::OrdinalIgnoreCase)) { + $dockerCommandSource = Resolve-DockerCommandSource + if (-not [string]::IsNullOrWhiteSpace($dockerCommandSource)) { + $dockerCommandExtension = [System.IO.Path]::GetExtension($dockerCommandSource) + if ([System.StringComparer]::OrdinalIgnoreCase.Equals($dockerCommandExtension, '.ps1')) { + $resolvedFilePath = (Get-Command -Name 'pwsh' -ErrorAction Stop | Select-Object -First 1).Source + $effectiveArguments = @('-NoLogo', '-NoProfile', '-File', $dockerCommandSource) + @($Arguments) + } else { + $resolvedFilePath = $dockerCommandSource + } + } + } + + $argText = if ($effectiveArguments -and $effectiveArguments.Count -gt 0) { + [string]::Join(' ', $effectiveArguments) + } else { + '' + } + $commandText = if ([string]::IsNullOrWhiteSpace($argText)) { $resolvedFilePath } else { "$resolvedFilePath $argText" } + + $result = [ordered]@{ + timedOut = $false + exitCode = $null + stdout = @() + stderr = @() + command = $commandText + exception = '' + } + + $psi = [System.Diagnostics.ProcessStartInfo]::new() + $psi.FileName = $resolvedFilePath + $psi.UseShellExecute = $false + $psi.RedirectStandardOutput = $true + $psi.RedirectStandardError = $true + $psi.CreateNoWindow = $true + foreach ($arg in @($effectiveArguments)) { + [void]$psi.ArgumentList.Add([string]$arg) + } + + $proc = [System.Diagnostics.Process]::new() + $proc.StartInfo = $psi + + try { + [void]$proc.Start() + $completed = $proc.WaitForExit($safeTimeout * 1000) + if (-not $completed) { + $result.timedOut = $true + try { $proc.Kill($true) } catch {} + return [pscustomobject]$result + } + + $result.exitCode = [int]$proc.ExitCode + $result.stdout = @(Split-OutputLines -Text $proc.StandardOutput.ReadToEnd()) + $result.stderr = @(Split-OutputLines -Text $proc.StandardError.ReadToEnd()) + } catch { + $result.exception = [string]$_.Exception.Message + try { + if (-not $proc.HasExited) { + $proc.Kill($true) + } + } catch {} + } finally { + $proc.Dispose() + } + + return [pscustomobject]$result +} + function Invoke-DockerCommand { param( [Parameter(Mandatory)][string[]]$Arguments, + [int]$TimeoutSeconds = $CommandTimeoutSeconds, [switch]$IgnoreExitCode ) - $raw = & docker @Arguments 2>&1 - $exitCode = $LASTEXITCODE - $lines = @($raw | ForEach-Object { [string]$_ }) + $invoke = Invoke-ProcessWithTimeout -FilePath 'docker' -Arguments $Arguments -TimeoutSeconds $TimeoutSeconds + $lines = @(@($invoke.stdout) + @($invoke.stderr) | ForEach-Object { [string]$_ }) $text = ($lines -join "`n") + if ($invoke.timedOut) { + $timeoutMessage = "docker {0} timed out after {1}s." -f ($Arguments -join ' '), [Math]::Max(5, [int]$TimeoutSeconds) + if (-not $IgnoreExitCode) { + throw $timeoutMessage + } + return [pscustomobject]@{ + ExitCode = 124 + TimedOut = $true + Lines = @($timeoutMessage) + Text = $timeoutMessage + } + } + + if ($invoke.exception) { + throw ("docker {0} failed to launch: {1}" -f ($Arguments -join ' '), [string]$invoke.exception) + } + + $exitCode = if ($null -eq $invoke.exitCode) { 1 } else { [int]$invoke.exitCode } + if (-not $IgnoreExitCode -and $exitCode -ne 0) { throw ("docker {0} failed (exit={1}). Output: {2}" -f ($Arguments -join ' '), $exitCode, $text) } return [pscustomobject]@{ ExitCode = [int]$exitCode + TimedOut = $false Lines = $lines Text = $text } @@ -155,7 +299,7 @@ function Set-ContextAndWait { $lastError = $null for ($attempt = 1; $attempt -le $RetryCount; $attempt++) { try { - $useContext = Invoke-DockerCommand -Arguments @('context', 'use', $ContextName) -IgnoreExitCode + $useContext = Invoke-DockerCommand -Arguments @('context', 'use', $ContextName) -TimeoutSeconds $CommandTimeoutSeconds -IgnoreExitCode if ($useContext.ExitCode -ne 0) { $switchError = [string]$useContext.Text $missingContext = ($switchError -match '(?i)context.+not found') -or ($switchError -match '(?i)cannot find the path specified') @@ -168,7 +312,7 @@ function Set-ContextAndWait { $deadline = (Get-Date).ToUniversalTime().AddSeconds($TimeoutSeconds) do { - $osProbe = Invoke-DockerCommand -Arguments @('info', '--format', '{{.OSType}}') -IgnoreExitCode + $osProbe = Invoke-DockerCommand -Arguments @('info', '--format', '{{.OSType}}') -TimeoutSeconds $CommandTimeoutSeconds -IgnoreExitCode if ($osProbe.ExitCode -eq 0) { $osType = $osProbe.Text.Trim().ToLowerInvariant() if ($osType -eq $ExpectedOsType.Trim().ToLowerInvariant()) { @@ -200,7 +344,7 @@ function Get-ImageProbeResult { [Parameter(Mandatory)][string]$ExpectedOs ) - $manifestOut = Invoke-DockerCommand -Arguments @('manifest', 'inspect', $Image) + $manifestOut = Invoke-DockerCommand -Arguments @('manifest', 'inspect', $Image) -TimeoutSeconds $CommandTimeoutSeconds $manifest = $manifestOut.Text | ConvertFrom-Json -Depth 30 $digest = '' @@ -281,17 +425,27 @@ function Ensure-LocalImageAvailability { pullError = '' } - $inspect = Invoke-DockerCommand -Arguments @('image', 'inspect', $Image, '--format', '{{json .}}') -IgnoreExitCode + $inspect = Invoke-DockerCommand -Arguments @('image', 'inspect', $Image, '--format', '{{json .}}') -TimeoutSeconds $CommandTimeoutSeconds -IgnoreExitCode + if ($inspect.TimedOut) { + throw ("docker image inspect timed out for '{0}' after {1}s." -f $Image, [Math]::Max(5, [int]$CommandTimeoutSeconds)) + } if ($inspect.ExitCode -ne 0 -and $BootstrapIfMissing) { $pullStart = Get-Date - $pull = Invoke-DockerCommand -Arguments @('pull', $Image) -IgnoreExitCode + $pull = Invoke-DockerCommand -Arguments @('pull', $Image) -TimeoutSeconds $BootstrapPullTimeoutSeconds -IgnoreExitCode $result.pullDurationMs = [int]([Math]::Round(((Get-Date) - $pullStart).TotalMilliseconds)) + if ($pull.TimedOut) { + $result.pullError = ("docker pull timed out for '{0}' after {1}s." -f $Image, [Math]::Max(5, [int]$BootstrapPullTimeoutSeconds)) + throw $result.pullError + } if ($pull.ExitCode -ne 0) { $result.pullError = [string]$pull.Text throw ("docker pull failed for '{0}' (exit={1}). Output: {2}" -f $Image, $pull.ExitCode, $pull.Text) } $result.pulled = $true - $inspect = Invoke-DockerCommand -Arguments @('image', 'inspect', $Image, '--format', '{{json .}}') -IgnoreExitCode + $inspect = Invoke-DockerCommand -Arguments @('image', 'inspect', $Image, '--format', '{{json .}}') -TimeoutSeconds $CommandTimeoutSeconds -IgnoreExitCode + if ($inspect.TimedOut) { + throw ("docker image inspect timed out for '{0}' after pull (limit {1}s)." -f $Image, [Math]::Max(5, [int]$CommandTimeoutSeconds)) + } } if ($inspect.ExitCode -ne 0) { @@ -339,21 +493,25 @@ function Invoke-ContainerRuntimeProbe { } $start = Get-Date - $probe = Invoke-DockerCommand -Arguments $args -IgnoreExitCode + $probe = Invoke-DockerCommand -Arguments $args -TimeoutSeconds $ProbeTimeoutSeconds -IgnoreExitCode $durationMs = [int]([Math]::Round(((Get-Date) - $start).TotalMilliseconds)) - $text = [string]$probe.Text + $text = if ($probe.TimedOut) { + "docker run timed out after {0}s." -f [Math]::Max(5, [int]$ProbeTimeoutSeconds) + } else { + [string]$probe.Text + } if ($text.Length -gt 2000) { $text = $text.Substring(0, 2000) } return [ordered]@{ attempted = $true - status = if ($probe.ExitCode -eq 0) { 'success' } else { 'failure' } - exitCode = [int]$probe.ExitCode + status = if ($probe.TimedOut) { 'timeout' } elseif ($probe.ExitCode -eq 0) { 'success' } else { 'failure' } + exitCode = if ($probe.TimedOut) { 124 } else { [int]$probe.ExitCode } durationMs = $durationMs output = $text command = ($args -join ' ') - error = if ($probe.ExitCode -eq 0) { '' } else { $text } + error = if ($probe.TimedOut -or $probe.ExitCode -ne 0) { $text } else { '' } } } @@ -414,6 +572,9 @@ $summary = [ordered]@{ switchRetryCount = [int]$SwitchRetryCount switchRetryDelaySeconds = [int]$SwitchRetryDelaySeconds lockWaitSeconds = [int]$LockWaitSeconds + commandTimeoutSeconds = [int]$CommandTimeoutSeconds + bootstrapPullTimeoutSeconds = [int]$BootstrapPullTimeoutSeconds + probeTimeoutSeconds = [int]$ProbeTimeoutSeconds } lock = [ordered]@{ path = '' @@ -510,7 +671,7 @@ try { $contextStart = '' try { - $contextStart = (Invoke-DockerCommand -Arguments @('context', 'show')).Text.Trim() + $contextStart = (Invoke-DockerCommand -Arguments @('context', 'show') -TimeoutSeconds $CommandTimeoutSeconds).Text.Trim() } catch { $contextStart = 'unknown' } @@ -518,7 +679,7 @@ try { $summary.contexts.start = $contextStart try { - $startOsProbe = Invoke-DockerCommand -Arguments @('info', '--format', '{{.OSType}}') -IgnoreExitCode + $startOsProbe = Invoke-DockerCommand -Arguments @('info', '--format', '{{.OSType}}') -TimeoutSeconds $CommandTimeoutSeconds -IgnoreExitCode if ($startOsProbe.ExitCode -eq 0) { $summary.contexts.startOsType = $startOsProbe.Text.Trim().ToLowerInvariant() } @@ -597,7 +758,13 @@ try { $caught = $_ $message = $_.Exception.Message $summary.status = 'failure' - if ($message -match '(?i)failed to switch docker context|did not reach expected ostype|docker engine switch|timed out waiting for docker manager lock') { + if ($message -match '(?i)docker pull timed out') { + $summary.failureClass = 'image-bootstrap-timeout' + } elseif ($message -match '(?i)runtime probe failed.+exit=124|docker run timed out') { + $summary.failureClass = 'runtime-probe-timeout' + } elseif ($message -match '(?i)docker .+timed out') { + $summary.failureClass = 'docker-command-timeout' + } elseif ($message -match '(?i)failed to switch docker context|did not reach expected ostype|docker engine switch|timed out waiting for docker manager lock') { $summary.failureClass = 'runtime-determinism' } elseif ($message -match '(?i)docker pull failed|local image inspect failed') { $summary.failureClass = 'image-bootstrap' @@ -608,6 +775,32 @@ try { } $summary.failureMessage = $message + if ($message -match '(?i)docker pull timed out') { + if ($includeWindows -and [string]::IsNullOrWhiteSpace([string]$summary.probes.windows.bootstrap.pullError)) { + $summary.probes.windows.bootstrap.attempted = [bool]$BootstrapWindowsImage + $summary.probes.windows.bootstrap.pullError = $message + } + if ($includeLinux -and [string]::IsNullOrWhiteSpace([string]$summary.probes.linux.bootstrap.pullError)) { + $summary.probes.linux.bootstrap.attempted = [bool]$BootstrapLinuxImage + $summary.probes.linux.bootstrap.pullError = $message + } + } + + if ($message -match '(?i)runtime probe failed.+exit=124|docker run timed out') { + if ($includeWindows -and [string]::Equals([string]$summary.probes.windows.probe.status, 'not-run', [System.StringComparison]::OrdinalIgnoreCase)) { + $summary.probes.windows.probe.attempted = $true + $summary.probes.windows.probe.status = 'timeout' + $summary.probes.windows.probe.exitCode = 124 + $summary.probes.windows.probe.error = $message + } + if ($includeLinux -and [string]::Equals([string]$summary.probes.linux.probe.status, 'not-run', [System.StringComparison]::OrdinalIgnoreCase)) { + $summary.probes.linux.probe.attempted = $true + $summary.probes.linux.probe.status = 'timeout' + $summary.probes.linux.probe.exitCode = 124 + $summary.probes.linux.probe.error = $message + } + } + if ($includeWindows -and $summary.probes.windows.status -eq 'pending') { $summary.probes.windows.status = 'failure' $summary.probes.windows.error = $message diff --git a/tools/Test-VIBinaryHandlingInvariants.ps1 b/tools/Test-VIBinaryHandlingInvariants.ps1 new file mode 100644 index 000000000..d7a85a164 --- /dev/null +++ b/tools/Test-VIBinaryHandlingInvariants.ps1 @@ -0,0 +1,169 @@ +#Requires -Version 7.0 +[CmdletBinding()] +param( + [string]$RepoRoot = (Join-Path $PSScriptRoot '..'), + [string]$OutputJsonPath = '', + [string]$StepSummaryPath = '', + [switch]$PassThru +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Resolve-AbsolutePath { + param([Parameter(Mandatory)][string]$Path) + return $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($Path) +} + +function ConvertTo-PortablePath { + param([AllowNull()][string]$Value) + if ([string]::IsNullOrWhiteSpace($Value)) { return '' } + return ($Value -replace '\\', '/') +} + +function New-TextualReadPattern { + param( + [Parameter(Mandatory)][string]$Id, + [Parameter(Mandatory)][string]$Pattern, + [Parameter(Mandatory)][string]$Description + ) + + return [pscustomobject]@{ + id = $Id + regex = [regex]::new($Pattern, [System.Text.RegularExpressions.RegexOptions]::IgnoreCase) + description = $Description + } +} + +$repoRootResolved = Resolve-AbsolutePath -Path $RepoRoot +$attrPath = Join-Path $repoRootResolved '.gitattributes' +$selfPath = Resolve-AbsolutePath -Path $PSCommandPath +$scannedRoots = @( + Join-Path $repoRootResolved 'scripts' + Join-Path $repoRootResolved 'tools' +) + +$textualReadPatterns = @( + (New-TextualReadPattern -Id 'get-content-vi' -Pattern 'Get-Content\s+[^\r\n]*\.vi(\b|[''"`])' -Description 'Direct Get-Content against a .vi path'), + (New-TextualReadPattern -Id 'readalltext-vi' -Pattern 'ReadAllText\s*\([^\r\n]*\.vi(\b|[''"`])' -Description 'Direct ReadAllText against a .vi path'), + (New-TextualReadPattern -Id 'open-text-vi' -Pattern 'OpenText\s*\([^\r\n]*\.vi(\b|[''"`])' -Description 'Direct OpenText against a .vi path'), + (New-TextualReadPattern -Id 'streamreader-vi' -Pattern 'StreamReader(?:\]::new|\s*\()[^\r\n]*\.vi(\b|[''"`])' -Description 'Direct StreamReader against a .vi path') +) + +$checks = @() +$violations = @() + +$attrOk = $false +$attrMessage = '' +if (Test-Path -LiteralPath $attrPath -PathType Leaf) { + $attrContent = Get-Content -LiteralPath $attrPath -Raw + $attrOk = [regex]::IsMatch($attrContent, '(?m)^\*\.vi\s+binary\s*$') + if (-not $attrOk) { + $attrMessage = '*.vi binary declaration is missing from .gitattributes.' + } +} else { + $attrMessage = '.gitattributes is missing.' +} + +$checks += [pscustomobject]@{ + id = 'gitattributes-vi-binary' + status = if ($attrOk) { 'passed' } else { 'failed' } + message = if ($attrOk) { '*.vi is declared as binary.' } else { $attrMessage } + path = '' +} + +$filesScanned = 0 +foreach ($root in $scannedRoots) { + if (-not (Test-Path -LiteralPath $root -PathType Container)) { + continue + } + + $psFiles = Get-ChildItem -LiteralPath $root -Recurse -Include *.ps1,*.psm1 -File | + Where-Object { + $_.FullName -notmatch '[/\\]tests[/\\]' -and + (Resolve-AbsolutePath -Path $_.FullName) -ne $selfPath + } + foreach ($file in $psFiles) { + $filesScanned += 1 + $content = Get-Content -LiteralPath $file.FullName -Raw + foreach ($pattern in $textualReadPatterns) { + $matches = $pattern.regex.Matches($content) + foreach ($match in $matches) { + $violations += [pscustomobject]@{ + file = $file.FullName + patternId = $pattern.id + description = $pattern.description + excerpt = [string]$match.Value + } + } + } + } +} + +$checks[0].path = ConvertTo-PortablePath -Value $attrPath +foreach ($violation in $violations) { + $violation.file = ConvertTo-PortablePath -Value $violation.file +} + +$portableRepoRoot = ConvertTo-PortablePath -Value $repoRootResolved +$portableScannedRoots = @($scannedRoots | ForEach-Object { ConvertTo-PortablePath -Value $_ }) + +$checks += [pscustomobject]@{ + id = 'no-textual-vi-reads' + status = if ($violations.Count -eq 0) { 'passed' } else { 'failed' } + message = if ($violations.Count -eq 0) { + 'No direct textual .vi reads were detected in scripts/ or tools/.' + } else { + ('Detected {0} direct textual .vi read pattern(s).' -f $violations.Count) + } + scannedRoots = $portableScannedRoots + filesScanned = $filesScanned +} + +$status = if (@($checks | Where-Object { $_.status -ne 'passed' }).Count -eq 0) { 'passed' } else { 'failed' } +$report = [pscustomobject]@{ + schema = 'comparevi/vi-binary-handling-invariants@v1' + generatedAtUtc = (Get-Date).ToUniversalTime().ToString('o') + status = $status + repoRoot = $portableRepoRoot + checks = @($checks) + violationCount = @($violations).Count + violations = @($violations) +} + +if (-not [string]::IsNullOrWhiteSpace($OutputJsonPath)) { + $outputPathResolved = Resolve-AbsolutePath -Path $OutputJsonPath + $outputParent = Split-Path -Parent $outputPathResolved + if ($outputParent -and -not (Test-Path -LiteralPath $outputParent -PathType Container)) { + New-Item -ItemType Directory -Path $outputParent -Force | Out-Null + } + $report | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $outputPathResolved -Encoding utf8 +} + +if (-not [string]::IsNullOrWhiteSpace($StepSummaryPath)) { + $summaryPathResolved = Resolve-AbsolutePath -Path $StepSummaryPath + $summaryLines = @( + '### VI Binary Handling Invariants', + '', + ('- status: `{0}`' -f $status), + ('- repo_root: `{0}`' -f (ConvertTo-PortablePath -Value $repoRootResolved)), + ('- files_scanned: `{0}`' -f $filesScanned), + ('- violation_count: `{0}`' -f $violations.Count) + ) + if ($violations.Count -gt 0) { + $summaryLines += '' + $summaryLines += '#### Violations' + foreach ($violation in $violations) { + $summaryLines += ('- `{0}` `{1}`' -f $violation.file, $violation.patternId) + } + } + $summaryLines -join "`n" | Out-File -LiteralPath $summaryPathResolved -Encoding utf8 -Append +} + +if ($PassThru) { + $report +} + +if ($status -ne 'passed') { + throw ('VI binary handling invariants failed. See report schema comparevi/vi-binary-handling-invariants@v1.') +} diff --git a/tools/Test-WindowsNI2026q1HostPreflight.ps1 b/tools/Test-WindowsNI2026q1HostPreflight.ps1 index 68f1e658d..81a863e47 100644 --- a/tools/Test-WindowsNI2026q1HostPreflight.ps1 +++ b/tools/Test-WindowsNI2026q1HostPreflight.ps1 @@ -23,6 +23,13 @@ param( [string]$ExecutionSurface = 'desktop-local', [bool]$ManageDockerEngine = $false, [bool]$AllowHostEngineMutation = $false, + [string]$HostPlatformOverride = '', + [ValidateRange(5, 600)] + [int]$CommandTimeoutSeconds = 45, + [ValidateRange(5, 3600)] + [int]$BootstrapPullTimeoutSeconds = 900, + [ValidateRange(5, 900)] + [int]$RuntimeProbeTimeoutSeconds = 180, [switch]$AllowUnavailable, [string]$OutputJsonPath = '', [string]$GitHubOutputPath = $env:GITHUB_OUTPUT, @@ -72,23 +79,161 @@ function Test-IsHostedRuntimeUnavailableMessage { return $false } +function Split-OutputLines { + param([AllowNull()][string]$Text) + + if ([string]::IsNullOrEmpty($Text)) { return @() } + return @($Text -split "(`r`n|`n|`r)" | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) +} + +function Resolve-DockerCommandSource { + $override = $env:DOCKER_COMMAND_OVERRIDE + if (-not [string]::IsNullOrWhiteSpace($override) -and (Test-Path -LiteralPath $override -PathType Leaf)) { + return [System.IO.Path]::GetFullPath($override) + } + + $pathSeparator = [System.IO.Path]::PathSeparator + $pathEntries = @($env:PATH -split [regex]::Escape([string]$pathSeparator)) + $candidates = if ($IsWindows) { + @('docker.exe', 'docker.cmd', 'docker.ps1', 'docker.bat', 'docker') + } else { + @('docker', 'docker.sh', 'docker.exe', 'docker.ps1', 'docker.cmd') + } + + foreach ($entry in $pathEntries) { + if ([string]::IsNullOrWhiteSpace($entry)) { continue } + foreach ($name in $candidates) { + $candidatePath = Join-Path $entry $name + if (Test-Path -LiteralPath $candidatePath -PathType Leaf) { + return [System.IO.Path]::GetFullPath($candidatePath) + } + } + } + + $command = Get-Command -Name 'docker' -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($null -eq $command -or [string]::IsNullOrWhiteSpace([string]$command.Source)) { + return $null + } + + return [System.IO.Path]::GetFullPath([string]$command.Source) +} + +function Invoke-ProcessWithTimeout { + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$FilePath, + [string[]]$Arguments = @(), + [int]$TimeoutSeconds = 45 + ) + + $safeTimeout = [Math]::Max(5, [int]$TimeoutSeconds) + $resolvedFilePath = $FilePath + $effectiveArguments = @($Arguments) + if ([string]::Equals($FilePath, 'docker', [System.StringComparison]::OrdinalIgnoreCase)) { + $dockerCommandSource = Resolve-DockerCommandSource + if (-not [string]::IsNullOrWhiteSpace($dockerCommandSource)) { + $dockerCommandExtension = [System.IO.Path]::GetExtension($dockerCommandSource) + if ([System.StringComparer]::OrdinalIgnoreCase.Equals($dockerCommandExtension, '.ps1')) { + $resolvedFilePath = (Get-Command -Name 'pwsh' -ErrorAction Stop | Select-Object -First 1).Source + $effectiveArguments = @('-NoLogo', '-NoProfile', '-File', $dockerCommandSource) + @($Arguments) + } else { + $resolvedFilePath = $dockerCommandSource + } + } + } + + $argText = if ($effectiveArguments -and $effectiveArguments.Count -gt 0) { + [string]::Join(' ', $effectiveArguments) + } else { + '' + } + $commandText = if ([string]::IsNullOrWhiteSpace($argText)) { $resolvedFilePath } else { "$resolvedFilePath $argText" } + + $result = [ordered]@{ + timedOut = $false + exitCode = $null + stdout = @() + stderr = @() + command = $commandText + exception = '' + } + + $psi = [System.Diagnostics.ProcessStartInfo]::new() + $psi.FileName = $resolvedFilePath + $psi.UseShellExecute = $false + $psi.RedirectStandardOutput = $true + $psi.RedirectStandardError = $true + $psi.CreateNoWindow = $true + foreach ($arg in @($effectiveArguments)) { + [void]$psi.ArgumentList.Add([string]$arg) + } + + $proc = [System.Diagnostics.Process]::new() + $proc.StartInfo = $psi + + try { + [void]$proc.Start() + $completed = $proc.WaitForExit($safeTimeout * 1000) + if (-not $completed) { + $result.timedOut = $true + try { $proc.Kill($true) } catch {} + return [pscustomobject]$result + } + + $result.exitCode = [int]$proc.ExitCode + $result.stdout = @(Split-OutputLines -Text $proc.StandardOutput.ReadToEnd()) + $result.stderr = @(Split-OutputLines -Text $proc.StandardError.ReadToEnd()) + } catch { + $result.exception = [string]$_.Exception.Message + try { + if (-not $proc.HasExited) { + $proc.Kill($true) + } + } catch {} + } finally { + $proc.Dispose() + } + + return [pscustomobject]$result +} + function Invoke-DockerCommand { param( [Parameter(Mandatory)][string[]]$Arguments, + [int]$TimeoutSeconds = $CommandTimeoutSeconds, [switch]$IgnoreExitCode ) - $raw = & docker @Arguments 2>&1 - $exitCode = $LASTEXITCODE - $lines = @($raw | ForEach-Object { [string]$_ }) + $invoke = Invoke-ProcessWithTimeout -FilePath 'docker' -Arguments $Arguments -TimeoutSeconds $TimeoutSeconds + $lines = @(@($invoke.stdout) + @($invoke.stderr) | ForEach-Object { [string]$_ }) $text = ($lines -join "`n") + if ($invoke.timedOut) { + $timeoutMessage = "docker {0} timed out after {1}s." -f ($Arguments -join ' '), [Math]::Max(5, [int]$TimeoutSeconds) + if (-not $IgnoreExitCode) { + throw $timeoutMessage + } + return [pscustomobject]@{ + ExitCode = 124 + TimedOut = $true + Lines = @($timeoutMessage) + Text = $timeoutMessage + } + } + + if ($invoke.exception) { + throw ("docker {0} failed to launch: {1}" -f ($Arguments -join ' '), [string]$invoke.exception) + } + + $exitCode = if ($null -eq $invoke.exitCode) { 1 } else { [int]$invoke.exitCode } + if (-not $IgnoreExitCode -and $exitCode -ne 0) { throw ("docker {0} failed (exit={1}). Output: {2}" -f ($Arguments -join ' '), $exitCode, $text) } return [pscustomobject]@{ ExitCode = [int]$exitCode + TimedOut = $false Lines = $lines Text = $text } @@ -134,17 +279,27 @@ function Ensure-LocalImageAvailability { pullError = '' } - $inspect = Invoke-DockerCommand -Arguments @('image', 'inspect', $Image, '--format', '{{json .}}') -IgnoreExitCode + $inspect = Invoke-DockerCommand -Arguments @('image', 'inspect', $Image, '--format', '{{json .}}') -TimeoutSeconds $CommandTimeoutSeconds -IgnoreExitCode + if ($inspect.TimedOut) { + throw ("docker image inspect timed out for '{0}' after {1}s." -f $Image, [Math]::Max(5, [int]$CommandTimeoutSeconds)) + } if ($inspect.ExitCode -ne 0) { $pullStart = Get-Date - $pull = Invoke-DockerCommand -Arguments @('pull', $Image) -IgnoreExitCode + $pull = Invoke-DockerCommand -Arguments @('pull', $Image) -TimeoutSeconds $BootstrapPullTimeoutSeconds -IgnoreExitCode $result.pullDurationMs = [int]([Math]::Round(((Get-Date) - $pullStart).TotalMilliseconds)) + if ($pull.TimedOut) { + $result.pullError = ("docker pull timed out for '{0}' after {1}s." -f $Image, [Math]::Max(5, [int]$BootstrapPullTimeoutSeconds)) + throw $result.pullError + } if ($pull.ExitCode -ne 0) { $result.pullError = [string]$pull.Text throw ("docker pull failed for '{0}' (exit={1}). Output: {2}" -f $Image, $pull.ExitCode, $pull.Text) } $result.pulled = $true - $inspect = Invoke-DockerCommand -Arguments @('image', 'inspect', $Image, '--format', '{{json .}}') -IgnoreExitCode + $inspect = Invoke-DockerCommand -Arguments @('image', 'inspect', $Image, '--format', '{{json .}}') -TimeoutSeconds $CommandTimeoutSeconds -IgnoreExitCode + if ($inspect.TimedOut) { + throw ("docker image inspect timed out for '{0}' after pull (limit {1}s)." -f $Image, [Math]::Max(5, [int]$CommandTimeoutSeconds)) + } } if ($inspect.ExitCode -ne 0) { @@ -188,21 +343,25 @@ function Invoke-ContainerRuntimeProbe { ) $start = Get-Date - $probe = Invoke-DockerCommand -Arguments $args -IgnoreExitCode + $probe = Invoke-DockerCommand -Arguments $args -TimeoutSeconds $RuntimeProbeTimeoutSeconds -IgnoreExitCode $durationMs = [int]([Math]::Round(((Get-Date) - $start).TotalMilliseconds)) - $text = [string]$probe.Text + $text = if ($probe.TimedOut) { + "docker run timed out after {0}s." -f [Math]::Max(5, [int]$RuntimeProbeTimeoutSeconds) + } else { + [string]$probe.Text + } if ($text.Length -gt 2000) { $text = $text.Substring(0, 2000) } return [ordered]@{ attempted = $true - status = if ($probe.ExitCode -eq 0) { 'success' } else { 'failure' } - exitCode = [int]$probe.ExitCode + status = if ($probe.TimedOut) { 'timeout' } elseif ($probe.ExitCode -eq 0) { 'success' } else { 'failure' } + exitCode = if ($probe.TimedOut) { 124 } else { [int]$probe.ExitCode } durationMs = $durationMs output = $text command = ($args -join ' ') - error = if ($probe.ExitCode -eq 0) { '' } else { $text } + error = if ($probe.TimedOut -or $probe.ExitCode -ne 0) { $text } else { '' } } } @@ -307,6 +466,9 @@ try { -WindowsImage $Image ` -BootstrapWindowsImage:$true ` -BootstrapLinuxImage:$false ` + -CommandTimeoutSeconds $CommandTimeoutSeconds ` + -BootstrapPullTimeoutSeconds $BootstrapPullTimeoutSeconds ` + -ProbeTimeoutSeconds $RuntimeProbeTimeoutSeconds ` -RestoreContext 'desktop-windows' ` -OutputJsonPath $jsonPathResolved ` -GitHubOutputPath '' ` @@ -356,6 +518,8 @@ try { -AutoRepair:$true ` -ManageDockerEngine:$ManageDockerEngine ` -AllowHostEngineMutation:$AllowHostEngineMutation ` + -HostPlatformOverride $HostPlatformOverride ` + -CommandTimeoutSeconds $CommandTimeoutSeconds ` -SnapshotPath $snapshotPath ` -GitHubOutputPath '' if ($LASTEXITCODE -ne 0) { @@ -442,7 +606,19 @@ try { if (-not $handledUnavailable) { $summary.status = 'failure' - if ([string]::IsNullOrWhiteSpace([string]$summary.failureClass) -or [string]::Equals([string]$summary.failureClass, 'none', [System.StringComparison]::OrdinalIgnoreCase)) { + if ($failureMessage -match '(?i)docker pull timed out') { + $summary.failureClass = 'image-bootstrap-timeout' + $summary.bootstrap.attempted = $true + $summary.bootstrap.pullError = $failureMessage + } elseif ($failureMessage -match '(?i)runtime probe timed out|docker run timed out|Hosted Windows runtime probe failed.+exit=124') { + $summary.failureClass = 'runtime-probe-timeout' + $summary.probe.attempted = $true + $summary.probe.status = 'timeout' + $summary.probe.exitCode = 124 + $summary.probe.error = $failureMessage + } elseif ($failureMessage -match '(?i)docker .+timed out') { + $summary.failureClass = 'docker-command-timeout' + } elseif ([string]::IsNullOrWhiteSpace([string]$summary.failureClass) -or [string]::Equals([string]$summary.failureClass, 'none', [System.StringComparison]::OrdinalIgnoreCase)) { $summary.failureClass = 'preflight-failed' } if ([string]::IsNullOrWhiteSpace([string]$summary.failureMessage)) { diff --git a/tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs b/tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs index 217914b6b..0e3fa4688 100644 --- a/tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs +++ b/tools/priority/__tests__/pester-service-model-local-harness-contract.test.mjs @@ -185,6 +185,8 @@ test('assurance packet records forward execution-pack, path-hygiene, replay, and assert.match(srs, /machine-readable escalation step/i); assert.match(srs, /REQ-PSM-028/); assert.match(srs, /shared local proof program selector/i); + assert.match(srs, /REQ-PSM-029/); + assert.match(srs, /secondary harness or evidence truth/i); assert.match(rtm, /REQ-PSM-012/); assert.match(rtm, /TEST-PSM-012/); @@ -220,6 +222,8 @@ test('assurance packet records forward execution-pack, path-hygiene, replay, and assert.match(rtm, /TEST-PSM-027/); assert.match(rtm, /REQ-PSM-028/); assert.match(rtm, /TEST-PSM-028/); + assert.match(rtm, /REQ-PSM-029/); + assert.match(rtm, /TEST-PSM-029/); assert.match(plan, /TEST-PSM-012[\s\S]*named-pack and execution-group coverage/i); assert.match(plan, /TEST-PSM-013[\s\S]*local path-hygiene coverage/i); @@ -242,6 +246,7 @@ test('assurance packet records forward execution-pack, path-hygiene, replay, and assert.match(plan, /TEST-PSM-026[\s\S]*proof-check aware autonomy coverage/i); assert.match(plan, /TEST-PSM-027[\s\S]*next-step escalation coverage/i); assert.match(plan, /TEST-PSM-028[\s\S]*shared local-program selector coverage/i); + assert.match(plan, /TEST-PSM-029[\s\S]*secondary-authority coverage/i); assert.match(doc, /named execution pack or test group/i); assert.match(doc, /OneDrive-like paths are path-hygiene risk/i); @@ -276,6 +281,10 @@ test('assurance packet records forward execution-pack, path-hygiene, replay, and assert.match(doc, /machine-readable escalation step/i); assert.match(doc, /comparevi-local-program-next-step\.json/i); assert.match(doc, /shared `windows-docker-desktop-ni-image` escalations should merge/i); + assert.match(doc, /Windows image-backed binary-handling CI surfaces/i); + assert.match(doc, /secondary harness and evidence truth/i); + assert.match(doc, /vi-binary-gate\.yml/i); + assert.match(doc, /windows-ni-proof-reusable\.yml/i); }); test('assurance packet records failure-detail producer consistency as an implemented execution requirement', () => { diff --git a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs index 24a088974..97ea88815 100644 --- a/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs +++ b/tools/priority/__tests__/pester-service-model-workflow-contract.test.mjs @@ -219,6 +219,7 @@ test('pester evidence distinguishes context-blocked, selection-blocked, and read test('knowledgebase documents the additive service model and keeps the monolith as the current baseline', () => { const doc = readRepoFile('docs/knowledgebase/Pester-Service-Model.md'); + const gateWorkflow = readRepoFile('.github/workflows/vi-binary-gate.yml'); assert.match(doc, /legacy Pester control plane couples four concerns into one self-hosted transaction/i); assert.match(doc, /pester-context\.yml/); @@ -236,6 +237,11 @@ test('knowledgebase documents the additive service model and keeps the monolith assert.match(doc, /release-evidence-provenance\.json/i); assert.match(doc, /promotion-dossier-provenance\.json/i); assert.match(doc, /existing required gate remains in place/i); + assert.match(doc, /Windows image-backed binary-handling CI surfaces/i); + assert.match(doc, /Pester is secondary harness and evidence truth/i); + assert.match(doc, /windows-ni-proof-reusable\.yml/); + assert.match(gateWorkflow, /uses:\s+\.\s*\/\.github\/workflows\/windows-ni-proof-reusable\.yml/); + assert.doesNotMatch(gateWorkflow, /pester-reusable\.yml/); }); test('trusted PR pilot router only runs self-hosted service-model proof for workflow dispatch or same-owner labeled PR heads', () => { diff --git a/tools/priority/__tests__/vi-history-local-ci.test.mjs b/tools/priority/__tests__/vi-history-local-ci.test.mjs index 1b1a14e39..aa95a5b30 100644 --- a/tools/priority/__tests__/vi-history-local-ci.test.mjs +++ b/tools/priority/__tests__/vi-history-local-ci.test.mjs @@ -6,7 +6,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { spawnSync } from 'node:child_process'; -import { applyAutonomyPolicy, deriveEscalations, determinePhase, parseCsv, parseRequirementNumber, rankProofRegressions, rankRequirementGaps, runLiveHistoryCandidateProof, selectNextStep } from '../vi-history-local-ci.mjs'; +import { applyAutonomyPolicy, deriveEscalations, determinePhase, parseCsv, parseRequirementNumber, rankProofRegressions, rankRequirementGaps, runLiveHistoryCandidateProof, runWindowsWorkflowReplayProof, selectNextStep } from '../vi-history-local-ci.mjs'; test('parseRequirementNumber extracts numeric VI History local-proof ids', () => { assert.equal(parseRequirementNumber('REQ-VHLP-001'), 1); @@ -130,6 +130,30 @@ test('deriveEscalations emits a shared Windows-surface escalation for VI History assert.equal(escalations[0].required_surface, 'windows-docker-desktop-ni-image'); }); +test('deriveEscalations emits an explicit Windows workflow replay next step when the shared surface is already ready', () => { + const escalations = deriveEscalations([ + { + id: 'windows-workflow-replay', + owner_requirement: 'REQ-VHLP-001', + status: 'advisory', + blocking: false, + summary: 'The governed VI History Windows workflow replay lane is ready and must be invoked explicitly as the next live-proof step.', + current_surface_status: 'ready-for-explicit-replay', + current_host_platform: 'Windows', + receipt_path: 'tests/results/docker-tools-parity/workflow-replay/vi-history-scenarios-windows-receipt.json', + reason: 'Local VI History CI keeps live Windows workflow replay as an explicit next step instead of running it implicitly during packet selection.', + recommended_commands: [ + 'npm run priority:workflow:replay:windows:vi-history' + ] + } + ]); + + assert.equal(escalations.length, 1); + assert.equal(escalations[0].governing_requirement, 'REQ-VHLP-010'); + assert.equal(escalations[0].blocked_requirement, 'REQ-VHLP-001'); + assert.equal(escalations[0].required_surface, 'vi-history-windows-workflow-replay'); +}); + test('deriveEscalations emits a clone-backed live-history escalation for VI History', () => { const escalations = deriveEscalations([ { @@ -191,6 +215,35 @@ test('runLiveHistoryCandidateProof validates a clone-backed target with real git assert.match(check.summary, /ready for local iteration/i); }); +test('runWindowsWorkflowReplayProof consumes an existing passing replay receipt instead of re-requesting replay', async () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'vi-history-replay-pass-')); + const receiptPath = path.join(tempRoot, 'tests', 'results', 'docker-tools-parity', 'workflow-replay', 'vi-history-scenarios-windows-receipt.json'); + fs.mkdirSync(path.dirname(receiptPath), { recursive: true }); + fs.writeFileSync(receiptPath, JSON.stringify({ + schema: 'windows-workflow-replay-lane@v1', + schemaVersion: '1.0.0', + replay: { mode: 'vi-history-scenarios-windows' }, + result: { status: 'passed', errorMessage: null }, + }, null, 2), 'utf8'); + + const check = await runWindowsWorkflowReplayProof(tempRoot, path.join(tempRoot, 'results'), { + runSharedWindowsSurfaceProofFn: async () => ({ + status: 'pass', + blocking: false, + current_surface_status: 'ready', + current_host_platform: 'Windows', + coordinator_host_platform: 'Unix', + bridge_mode: 'wsl-windows', + receipt_path: 'tests/results/_agent/windows-docker-shared-surface/local-ci/windows-surface/pester-windows-container-surface.json', + reason: 'ready', + }), + }); + + assert.equal(check.status, 'pass'); + assert.equal(check.current_surface_status, 'passed'); + assert.match(check.summary, /already passed/i); +}); + test('selectNextStep prefers requirements before VI History escalations', () => { const requirement = { req_id: 'REQ-VHLP-002', diff --git a/tools/priority/__tests__/vi-history-local-proof-contract.test.mjs b/tools/priority/__tests__/vi-history-local-proof-contract.test.mjs index 470f63613..4adf58e02 100644 --- a/tools/priority/__tests__/vi-history-local-proof-contract.test.mjs +++ b/tools/priority/__tests__/vi-history-local-proof-contract.test.mjs @@ -68,6 +68,13 @@ test('VI History local-proof packet traces requirements, tests, and shared Windo assert.match(srs, /VIP_Pre-Uninstall Custom Action\.vi/); assert.match(srs, /REQ-VHLP-009/); assert.match(srs, /clone exists locally, contains the target VI, and exposes real git history/i); + assert.match(srs, /REQ-VHLP-010/); + assert.match(srs, /explicit next step/i); + assert.match(srs, /priority:workflow:replay:windows:vi-history/); + assert.match(srs, /REQ-VHLP-011/); + assert.match(srs, /bounded helper-process timeouts/i); + assert.match(srs, /REQ-VHLP-012/); + assert.match(srs, /consume a successful governed Windows workflow replay receipt/i); assert.match(rtm, /REQ-VHLP-006/); assert.match(rtm, /TEST-VHLP-006/); @@ -77,6 +84,12 @@ test('VI History local-proof packet traces requirements, tests, and shared Windo assert.match(rtm, /TEST-VHLP-008/); assert.match(rtm, /REQ-VHLP-009/); assert.match(rtm, /TEST-VHLP-009/); + assert.match(rtm, /REQ-VHLP-010/); + assert.match(rtm, /TEST-VHLP-010/); + assert.match(rtm, /REQ-VHLP-011/); + assert.match(rtm, /TEST-VHLP-011/); + assert.match(rtm, /REQ-VHLP-012/); + assert.match(rtm, /TEST-VHLP-012/); assert.match(plan, /TEST-VHLP-001/); assert.match(plan, /TEST-VHLP-006/); @@ -87,6 +100,12 @@ test('VI History local-proof packet traces requirements, tests, and shared Windo assert.match(plan, /ni\/labview-icon-editor/i); assert.match(plan, /TEST-VHLP-009/); assert.match(plan, /clone presence, target path presence, and git history/i); + assert.match(plan, /TEST-VHLP-010/); + assert.match(plan, /explicit Windows replay next-step coverage/i); + assert.match(plan, /TEST-VHLP-011/); + assert.match(plan, /bounded Windows replay lifecycle coverage/i); + assert.match(plan, /TEST-VHLP-012/); + assert.match(plan, /replay receipt consumption coverage/i); assert.match(doc, /priority:workflow:replay:windows:vi-history/); assert.match(doc, /history:local:proof/); @@ -99,16 +118,27 @@ test('VI History local-proof packet traces requirements, tests, and shared Windo assert.match(doc, /VIP_Pre-Uninstall Custom Action\.vi/); assert.match(doc, /UNC-backed WSL checkout/i); assert.match(doc, /staged into a Windows-local mount root/i); + assert.match(doc, /vi-history-windows-workflow-replay/); + assert.match(doc, /should not launch the live Windows workflow replay lane/i); + assert.match(doc, /terminate or fail closed within bounded helper-process timeouts/i); + assert.match(doc, /consume[\s\S]*same replay step/i); assert.match(arch, /Windows workflow replay surface/); assert.match(arch, /Local autonomy surface/); assert.match(arch, /Clone-backed live-history candidate surface/); + assert.match(arch, /explicit next-step escalation/i); + assert.match(arch, /bounded helper-process[\s\S]*timeouts/i); + assert.match(arch, /passing governed Windows workflow replay receipt should be consumable/i); assert.match(localCi, /REQ-VHLP-006/); assert.match(localCi, /windows-docker-desktop-ni-image/); assert.match(localCi, /priority:workflow:replay:windows:vi-history/); - assert.match(localCi, /windows-host-bridge/i); + assert.match(localCi, /runSharedWindowsDockerSurfaceProof/); + assert.match(localCi, /windows-docker-shared-surface-local-ci/); assert.match(localCi, /REQ-VHLP-009/); assert.match(localCi, /clone-backed-live-history-candidate/); assert.match(localCi, /vi-history-live-candidate-readiness\.json/); + assert.match(localCi, /REQ-VHLP-010/); + assert.match(localCi, /ready-for-explicit-replay/); + assert.match(localCi, /vi-history-windows-workflow-replay/); }); diff --git a/tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs b/tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs index ca5329b36..120d7d85c 100644 --- a/tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs +++ b/tools/priority/__tests__/windows-docker-shared-surface-contract.test.mjs @@ -48,6 +48,9 @@ test('Windows shared-surface packet traces requirements, tests, and shared-progr assert.match(srs, /REQ-WDSS-008/); assert.match(srs, /UNC-backed WSL/i); assert.match(srs, /Windows-local mount root/i); + assert.match(srs, /REQ-WDSS-009/); + assert.match(srs, /blocking execution truth/i); + assert.match(srs, /pester-reusable\.yml/); assert.match(rtm, /REQ-WDSS-003/); assert.match(rtm, /TEST-WDSS-003/); @@ -57,6 +60,8 @@ test('Windows shared-surface packet traces requirements, tests, and shared-progr assert.match(rtm, /TEST-WDSS-007/); assert.match(rtm, /REQ-WDSS-008/); assert.match(rtm, /TEST-WDSS-008/); + assert.match(rtm, /REQ-WDSS-009/); + assert.match(rtm, /TEST-WDSS-009/); assert.match(plan, /TEST-WDSS-001/); assert.match(plan, /TEST-WDSS-002/); @@ -67,6 +72,8 @@ test('Windows shared-surface packet traces requirements, tests, and shared-progr assert.match(plan, /reachable Windows host bridge/i); assert.match(plan, /TEST-WDSS-008/); assert.match(plan, /UNC-backed WSL staging coverage/i); + assert.match(plan, /TEST-WDSS-009/); + assert.match(plan, /authoritative CI gate coverage/i); assert.match(doc, /priority:windows-surface:local-ci/); assert.match(doc, /tests:windows-surface:probe/); @@ -76,6 +83,9 @@ test('Windows shared-surface packet traces requirements, tests, and shared-progr assert.match(doc, /ExecutionPolicy Bypass/i); assert.match(doc, /stage container-bound inputs and output targets/i); assert.match(doc, /ni-windows-container-capture\.json/); + assert.match(doc, /authoritative[\s\S]*execution-truth plane/i); + assert.match(doc, /windows-ni-proof-reusable\.yml/); + assert.match(doc, /Test-VIBinaryHandlingInvariants\.ps1/); assert.match(arch, /Readiness probe surface/); assert.match(arch, /Path-hygiene surface/); @@ -83,6 +93,8 @@ test('Windows shared-surface packet traces requirements, tests, and shared-progr assert.match(arch, /Bridge surface/); assert.match(arch, /Windows-local staging surface/); assert.match(arch, /UNC-backed WSL repo paths should never be passed straight to Docker bind\s+mounts/i); + assert.match(arch, /Hosted CI authority surface/); + assert.match(arch, /windows-ni-proof-reusable\.yml/); assert.match(programDoc, /Windows Docker Shared Surface/i); @@ -97,4 +109,8 @@ test('Windows shared-surface packet traces requirements, tests, and shared-progr assert.match(compareScript, /Test-PathRequiresWindowsDockerLocalStage/); assert.match(compareScript, /outputSyncStatus/); assert.match(compareScript, /cleanupStatus/); + + const invariantsScript = readRepoFile('tools/Test-VIBinaryHandlingInvariants.ps1'); + assert.match(invariantsScript, /comparevi\/vi-binary-handling-invariants@v1/); + assert.match(invariantsScript, /no-textual-vi-reads/); }); diff --git a/tools/priority/__tests__/windows-ni-proof-workflow-contract.test.mjs b/tools/priority/__tests__/windows-ni-proof-workflow-contract.test.mjs new file mode 100644 index 000000000..750522157 --- /dev/null +++ b/tools/priority/__tests__/windows-ni-proof-workflow-contract.test.mjs @@ -0,0 +1,69 @@ +#!/usr/bin/env node + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { readFileSync } from 'node:fs'; + +const repoRoot = process.cwd(); + +function readRepoFile(relativePath) { + return readFileSync(path.join(repoRoot, relativePath), 'utf8'); +} + +test('reusable Windows NI proof workflow owns hosted preflight, compare, artifact upload, and optional VI-binary invariants', () => { + const workflow = readRepoFile('.github/workflows/windows-ni-proof-reusable.yml'); + + assert.match(workflow, /name:\s+Windows NI proof \(reusable\)/); + assert.match(workflow, /workflow_call:/); + assert.match(workflow, /run_binary_invariants:/); + assert.match(workflow, /group:\s+\$\{\{\s*github\.workflow\s*\}\}-windows-ni-proof-\$\{\{\s*inputs\.sample_id \|\| github\.ref\s*\}\}/); + assert.match(workflow, /runs-on:\s+windows-2022/); + assert.match(workflow, /permissions:\s*\r?\n\s+contents:\s+read/); + assert.match(workflow, /uses:\s+actions\/checkout@v5/); + assert.match( + workflow, + /repository:\s+\$\{\{\s*github\.event_name == 'pull_request' && github\.event\.pull_request\.head\.repo\.full_name \|\| github\.repository\s*\}\}/ + ); + assert.match( + workflow, + /ref:\s+\$\{\{\s*github\.event_name == 'pull_request' && github\.event\.pull_request\.head\.sha \|\| github\.sha\s*\}\}/ + ); + assert.match(workflow, /Validate VI binary-handling invariants/); + assert.match(workflow, /Test-VIBinaryHandlingInvariants\.ps1/); + assert.match(workflow, /Prepare NI Windows image and hosted runtime/); + assert.match(workflow, /Prepare NI Windows image and hosted runtime[\s\S]*?timeout-minutes:\s+20/); + assert.match(workflow, /Test-WindowsNI2026q1HostPreflight\.ps1/); + assert.match(workflow, /-ExecutionSurface 'github-hosted-windows'/); + assert.match(workflow, /Run NI Windows create comparison report/); + assert.match(workflow, /Run-NIWindowsContainerCompare\.ps1/); + assert.match(workflow, /runtime-manager-compare-windows\.json/); + assert.match(workflow, /Validate Windows comparison report artifact contract/); + assert.match(workflow, /vi-history\/windows-compare-artifact-summary@v1/); + assert.match(workflow, /Upload Windows NI proof artifacts/); + assert.match(workflow, /vi-binary-handling-invariants\.json/); + assert.doesNotMatch(workflow, /pester-reusable\.yml/); +}); + +test('VI binary gate routes through the reusable Windows NI proof workflow instead of pester-reusable', () => { + const workflow = readRepoFile('.github/workflows/vi-binary-gate.yml'); + + assert.match(workflow, /name:\s+VI Binary Handling Gate/); + assert.match(workflow, /uses:\s+\.\s*\/\.github\/workflows\/windows-ni-proof-reusable\.yml/); + assert.match(workflow, /base_vi:\s+fixtures\/vi-stage\/control-rename\/Base\.vi/); + assert.match(workflow, /head_vi:\s+fixtures\/vi-stage\/control-rename\/Head\.vi/); + assert.match(workflow, /run_binary_invariants:\s+true/); + assert.match(workflow, /results_root:\s+tests\/results\/vi-binary-gate/); + assert.doesNotMatch(workflow, /pester-reusable\.yml/); +}); + +test('manual Windows hosted parity workflow reuses the same hosted Windows NI proof contract', () => { + const workflow = readRepoFile('.github/workflows/windows-hosted-parity.yml'); + + assert.match(workflow, /name:\s+Windows Hosted NI Proof \(Manual\)/); + assert.match(workflow, /workflow_dispatch:/); + assert.match(workflow, /uses:\s+\.\s*\/\.github\/workflows\/windows-ni-proof-reusable\.yml/); + assert.match(workflow, /results_root:\s+tests\/results\/windows-hosted-parity/); + assert.match(workflow, /artifact_name:\s+windows-hosted-ni-proof/); + assert.doesNotMatch(workflow, /pester-reusable\.yml/); +}); diff --git a/tools/priority/__tests__/windows-workflow-replay-lane.test.mjs b/tools/priority/__tests__/windows-workflow-replay-lane.test.mjs index cad6900d7..022661129 100644 --- a/tools/priority/__tests__/windows-workflow-replay-lane.test.mjs +++ b/tools/priority/__tests__/windows-workflow-replay-lane.test.mjs @@ -88,6 +88,7 @@ test('buildReplayCommand forwards the deterministic Windows preflight locations' }); assert.equal(command.helperPath.replace(/\\/g, '/'), 'tools/Test-WindowsNI2026q1HostPreflight.ps1'); + assert.equal(command.helperTimeoutSeconds, 300); assert.deepEqual(command.command, [ '-NoLogo', '-NoProfile', @@ -127,6 +128,8 @@ test('buildReplayCommand exposes the local vi-history-scenarios-windows compare assert.equal(command.kind, 'preflight-compare'); assert.equal(command.compareHelperPath.replace(/\\/g, '/'), 'tools/Run-NIWindowsContainerCompare.ps1'); + assert.equal(command.helperTimeoutSeconds, 300); + assert.equal(command.compareProcessTimeoutSeconds, 900); assert.deepEqual(command.compareCommand, [ '-NoLogo', '-NoProfile', @@ -431,3 +434,33 @@ test('runWindowsWorkflowReplayLane fails closed and still writes a receipt when assert.match(result.receipt.result.errorMessage, /preflight failed/i); runSchemaValidate(repoRoot, replaySchemaPath, path.join(tmpDir, options.receiptPath)); }); + +test('runWindowsWorkflowReplayLane fails closed when the helper exceeds the bounded timeout', async () => { + const { parseArgs, runWindowsWorkflowReplayLane } = await loadModule(); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'windows-workflow-replay-timeout-')); + const options = parseArgs(['node', modulePath], tmpDir); + + const result = await runWindowsWorkflowReplayLane(options, { + repoRoot: tmpDir, + resolveRepoGitStateFn: () => ({ + headSha: 'abc123', + branch: 'issue/upstream-2088-windows-replay-timeout', + upstreamDevelopMergeBase: 'base123', + dirtyTracked: false, + }), + runProcessFn: () => ({ + status: null, + stdout: '', + stderr: '', + error: Object.assign(new Error('spawnSync pwsh ETIMEDOUT'), { code: 'ETIMEDOUT' }), + signal: 'SIGKILL', + timedOut: true, + }), + }); + + assert.equal(result.status, 'failed'); + assert.equal(result.receipt.result.status, 'failed'); + assert.equal(result.receipt.result.exitCode, 124); + assert.match(result.receipt.result.errorMessage, /bounded timeout of 300s/i); + runSchemaValidate(repoRoot, replaySchemaPath, path.join(tmpDir, options.receiptPath)); +}); diff --git a/tools/priority/vi-history-local-ci.mjs b/tools/priority/vi-history-local-ci.mjs index fb5c9163b..fa376eba6 100644 --- a/tools/priority/vi-history-local-ci.mjs +++ b/tools/priority/vi-history-local-ci.mjs @@ -9,11 +9,6 @@ import Ajv2020 from 'ajv/dist/2020.js'; import addFormats from 'ajv-formats'; import yaml from 'js-yaml'; -import { - buildWindowsNodeBridgeSpec, - detectWindowsHostBridge, - runBridgeSpec, -} from './windows-host-bridge.mjs'; import { runWindowsSurfaceProof as runSharedWindowsDockerSurfaceProof } from './windows-docker-shared-surface-local-ci.mjs'; const repoRootDefault = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); @@ -139,6 +134,17 @@ async function readJson(filePath) { return JSON.parse(await fs.readFile(filePath, 'utf8')); } +async function readJsonIfExists(filePath) { + try { + return await readJson(filePath); + } catch (error) { + if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') { + return null; + } + throw error; + } +} + function relativeFrom(repoRoot, filePath) { return path.relative(repoRoot, filePath).split(path.sep).join('/'); } @@ -374,6 +380,34 @@ function deriveEscalations(proofChecks) { }; } + if (check.current_surface_status === 'ready-for-explicit-replay') { + return { + type: 'escalation', + escalation_id: 'vi-history-windows-workflow-replay', + governing_requirement: 'REQ-VHLP-010', + blocked_requirement: 'REQ-VHLP-001', + proof_check_id: check.id, + status: 'required', + mode: 'escalate', + why_now: 'The shared Windows surface and clone-backed live-history candidate are ready, so the next truthful move is the explicit governed VI History Windows workflow replay lane.', + reason: check.reason, + required_surface: 'vi-history-windows-workflow-replay', + current_surface_status: check.current_surface_status ?? 'unknown', + current_host_platform: check.current_host_platform ?? 'unknown', + receipt_path: check.receipt_path ?? null, + suggested_loop: [ + 'Run the governed Windows workflow replay lane explicitly instead of expecting packet selection to invoke it.', + 'Inspect the workflow-grade receipt and compare artifacts the replay lane emits.', + 'If the replay lane fails, reopen the owning VI History requirement from the emitted receipt.' + ], + recommended_commands: check.recommended_commands ?? [], + stop_conditions: [ + 'Stop once the VI History Windows workflow replay lane receipt reaches status=passed.', + 'Stop if the replay lane exposes a new blocking host, image, or artifact defect.' + ] + }; + } + return { type: 'escalation', escalation_id: 'windows-docker-desktop-ni-image', @@ -739,8 +773,14 @@ async function runSharedWindowsSurfaceProof(repoRoot, resultsDir) { }; } -async function runWindowsWorkflowReplayProof(repoRoot, resultsDir) { - const surfaceProof = await runSharedWindowsSurfaceProof(repoRoot, resultsDir); +async function runWindowsWorkflowReplayProof( + repoRoot, + resultsDir, + { + runSharedWindowsSurfaceProofFn = runSharedWindowsSurfaceProof, + } = {}, +) { + const surfaceProof = await runSharedWindowsSurfaceProofFn(repoRoot, resultsDir); if (surfaceProof.status !== 'pass') { return { id: 'windows-workflow-replay', @@ -759,96 +799,67 @@ async function runWindowsWorkflowReplayProof(repoRoot, resultsDir) { } const receiptPath = path.join('tests', 'results', 'docker-tools-parity', 'workflow-replay', 'vi-history-scenarios-windows-receipt.json'); - const result = process.platform === 'win32' - ? spawnSync('node', [ - 'tools/priority/windows-workflow-replay-lane.mjs', - '--mode', - 'vi-history-scenarios-windows', - '--allow-unavailable' - ], { - cwd: repoRoot, - encoding: 'utf8', - maxBuffer: 20 * 1024 * 1024 - }) - : (() => { - const bridge = detectWindowsHostBridge(repoRoot); - if (bridge.status !== 'reachable') { - throw new Error(bridge.reason); - } - const bridgeSpec = buildWindowsNodeBridgeSpec({ - bridge, - scriptRelativePath: path.join('tools', 'priority', 'windows-workflow-replay-lane.mjs'), - scriptArgs: ['--mode', 'vi-history-scenarios-windows', '--allow-unavailable'] - }); - return runBridgeSpec(bridgeSpec, { cwd: repoRoot, maxBuffer: 64 * 1024 * 1024 }); - })(); - - const base = { - id: 'windows-workflow-replay', - owner_requirement: 'REQ-VHLP-001', - receipt_path: receiptPath - }; - - let receipt; - try { - receipt = await readJson(path.join(repoRoot, receiptPath)); - } catch (error) { - return { - ...base, - status: 'fail', - blocking: true, - summary: 'VI History Windows workflow replay did not emit its governed receipt.', - details: [result.stdout?.trim(), result.stderr?.trim(), error instanceof Error ? error.message : String(error)].filter(Boolean) - }; - } + const receipt = await readJsonIfExists(path.join(repoRoot, receiptPath)); + const receiptStatus = receipt?.result?.status ?? null; - const receiptStatus = receipt?.result?.status ?? 'unknown'; if (receiptStatus === 'passed') { return { - ...base, + id: 'windows-workflow-replay', + owner_requirement: 'REQ-VHLP-001', status: 'pass', blocking: false, - summary: 'VI History Windows workflow replay passed and emitted a workflow-grade receipt.', - current_surface_status: receiptStatus, - current_host_platform: 'Windows', - coordinator_host_platform: process.platform === 'win32' ? 'Windows' : 'Unix', - bridge_mode: process.platform === 'win32' ? 'native-windows' : 'wsl-windows', + summary: 'The governed VI History Windows workflow replay lane already passed and emitted a workflow-grade receipt.', + current_surface_status: 'passed', + current_host_platform: surfaceProof.current_host_platform, + coordinator_host_platform: surfaceProof.coordinator_host_platform, + bridge_mode: surfaceProof.bridge_mode, + reason: 'A passing VI History Windows workflow replay receipt is already present for the governed lane.', + receipt_path: receiptPath, recommended_commands: [ 'npm run priority:workflow:replay:windows:vi-history' ] }; } - if (receiptStatus === 'unavailable') { + + if (receiptStatus === 'failed') { return { - ...base, - status: 'advisory', - blocking: false, - summary: 'VI History Windows workflow replay is unavailable from the current host; use the shared Windows Docker Desktop + NI image surface.', - current_surface_status: receiptStatus, - current_host_platform: 'Windows', - coordinator_host_platform: process.platform === 'win32' ? 'Windows' : 'Unix', - bridge_mode: process.platform === 'win32' ? 'native-windows' : 'wsl-windows', + id: 'windows-workflow-replay', + owner_requirement: 'REQ-VHLP-001', + status: 'fail', + blocking: true, + summary: 'The governed VI History Windows workflow replay lane emitted a failing receipt.', + current_surface_status: 'failed', + current_host_platform: surfaceProof.current_host_platform, + coordinator_host_platform: surfaceProof.coordinator_host_platform, + bridge_mode: surfaceProof.bridge_mode, + reason: receipt?.result?.errorMessage ?? 'The replay receipt recorded a failure.', + receipt_path: receiptPath, + details: [receipt?.result?.errorMessage].filter(Boolean), recommended_commands: [ - 'npm run docker:ni:windows:bootstrap', - 'npm run compare:docker:ni:windows:probe', 'npm run priority:workflow:replay:windows:vi-history' ] }; } + return { - ...base, - status: 'fail', - blocking: true, - summary: 'VI History Windows workflow replay failed on the current packet.', - current_surface_status: receiptStatus, - current_host_platform: 'Windows', - coordinator_host_platform: process.platform === 'win32' ? 'Windows' : 'Unix', - bridge_mode: process.platform === 'win32' ? 'native-windows' : 'wsl-windows', - details: [result.stdout?.trim(), result.stderr?.trim()].filter(Boolean) + id: 'windows-workflow-replay', + owner_requirement: 'REQ-VHLP-001', + status: 'advisory', + blocking: false, + summary: 'The governed VI History Windows workflow replay lane is ready and must be invoked explicitly as the next live-proof step.', + current_surface_status: 'ready-for-explicit-replay', + current_host_platform: surfaceProof.current_host_platform, + coordinator_host_platform: surfaceProof.coordinator_host_platform, + bridge_mode: surfaceProof.bridge_mode, + reason: 'Local VI History CI keeps live Windows workflow replay as an explicit next step instead of running it implicitly during packet selection.', + receipt_path: receiptPath, + recommended_commands: [ + 'npm run priority:workflow:replay:windows:vi-history' + ] }; } -export { parseCsv, parseRequirementNumber, determinePhase, rankRequirementGaps, rankProofRegressions, deriveEscalations, selectNextStep, applyAutonomyPolicy, runLiveHistoryCandidateProof }; +export { parseCsv, parseRequirementNumber, determinePhase, rankRequirementGaps, rankProofRegressions, deriveEscalations, selectNextStep, applyAutonomyPolicy, runLiveHistoryCandidateProof, runWindowsWorkflowReplayProof }; export async function runVIHistoryLocalCi({ repoRoot = repoRootDefault, diff --git a/tools/priority/windows-workflow-replay-lane.mjs b/tools/priority/windows-workflow-replay-lane.mjs index e557b29c4..7a3fe5089 100644 --- a/tools/priority/windows-workflow-replay-lane.mjs +++ b/tools/priority/windows-workflow-replay-lane.mjs @@ -19,6 +19,7 @@ export const MODE_CONFIG = Object.freeze({ 'windows-ni-2026q1-host-preflight': Object.freeze({ kind: 'preflight-only', helperPath: path.join('tools', 'Test-WindowsNI2026q1HostPreflight.ps1'), + helperTimeoutSeconds: 300, preflightReportPath: path.join( DEFAULT_RESULTS_ROOT, 'windows-ni-2026q1-host-preflight', @@ -29,6 +30,8 @@ export const MODE_CONFIG = Object.freeze({ kind: 'preflight-compare', helperPath: path.join('tools', 'Test-WindowsNI2026q1HostPreflight.ps1'), compareHelperPath: path.join('tools', 'Run-NIWindowsContainerCompare.ps1'), + helperTimeoutSeconds: 300, + compareProcessTimeoutSeconds: 900, preflightReportPath: path.join( DEFAULT_RESULTS_ROOT, 'vi-history-scenarios-windows', @@ -128,12 +131,16 @@ function runProcess(command, args, options = {}) { encoding: 'utf8', env: options.env ?? process.env, maxBuffer: options.maxBuffer ?? DEFAULT_REPLAY_MAX_BUFFER_BYTES, + timeout: options.timeoutMs, + killSignal: options.killSignal ?? 'SIGKILL', }); return { status: typeof result.status === 'number' ? result.status : null, stdout: result.stdout ?? '', stderr: result.stderr ?? '', error: result.error ?? null, + signal: result.signal ?? null, + timedOut: result.error?.code === 'ETIMEDOUT', }; } @@ -331,12 +338,14 @@ export function buildReplayCommand(options) { const replayCommand = { kind: config.kind, helperPath: config.helperPath, + helperTimeoutSeconds: config.helperTimeoutSeconds ?? 300, command, modePaths, }; if (config.kind === 'preflight-compare') { replayCommand.compareHelperPath = config.compareHelperPath; + replayCommand.compareProcessTimeoutSeconds = config.compareProcessTimeoutSeconds ?? (config.timeoutSeconds + 300); replayCommand.compareCommand = [ '-NoLogo', '-NoProfile', @@ -491,8 +500,17 @@ export async function runWindowsWorkflowReplayLane( cwd: repoRoot, env, maxBuffer: DEFAULT_REPLAY_MAX_BUFFER_BYTES, + timeoutMs: replayCommand.helperTimeoutSeconds * 1000, + killSignal: 'SIGKILL', }); + if (helperResult.timedOut) { + return failClosed( + `Windows replay helper exceeded the bounded timeout of ${replayCommand.helperTimeoutSeconds}s.`, + 124, + ); + } + if (!fs.existsSync(preflightReportResolvedPath)) { return failClosed( normalizeText(helperResult.stderr) ?? @@ -526,7 +544,15 @@ export async function runWindowsWorkflowReplayLane( cwd: repoRoot, env, maxBuffer: DEFAULT_REPLAY_MAX_BUFFER_BYTES, + timeoutMs: replayCommand.compareProcessTimeoutSeconds * 1000, + killSignal: 'SIGKILL', }); + if (compareResult.timedOut) { + return failClosed( + `Windows compare helper exceeded the bounded timeout of ${replayCommand.compareProcessTimeoutSeconds}s.`, + 124, + ); + } const compareExitCode = compareResult.status ?? 0; const reportResolvedPath = resolveRepoPath(repoRoot, replayCommand.modePaths.reportPath); const captureResolvedPath = resolveRepoPath(repoRoot, replayCommand.modePaths.capturePath); From 5c837a9209c8f8c965d45a693d81594059a9a23f Mon Sep 17 00:00:00 2001 From: svelderrainruiz Date: Wed, 1 Apr 2026 05:37:27 -0700 Subject: [PATCH 36/44] release: defer downstream proof for non-develop tags --- .github/workflows/release.yml | 51 ++++++++++++++++++- docs/DOWNSTREAM_DEVELOP_PROMOTION_CONTRACT.md | 7 +++ docs/RELEASE_PROMOTION_CONTRACT.md | 10 ++++ ...rkflow-pwsh-continuation-contract.test.mjs | 32 ++++++++++-- 4 files changed, 96 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 15818add8..70c1b281a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -774,6 +774,55 @@ jobs: source_sha="$(git rev-parse "${RELEASE_TAG}^{commit}")" echo "source_sha=$source_sha" >> "$GITHUB_OUTPUT" + - name: Resolve current develop head + id: current_develop + shell: bash + run: | + set -euo pipefail + git fetch --force origin "refs/heads/develop:refs/remotes/origin/develop" + develop_sha="$(git rev-parse "refs/remotes/origin/develop^{commit}")" + echo "develop_sha=$develop_sha" >> "$GITHUB_OUTPUT" + + - name: Decide downstream proving enforcement mode + id: downstream_proving_policy + if: always() + shell: bash + run: | + set -euo pipefail + source_sha='${{ steps.release_source.outputs.source_sha }}' + develop_sha='${{ steps.current_develop.outputs.develop_sha }}' + mode='advisory-replay' + reason='release-publication-mode-not-publish' + + if [[ "${RELEASE_PUBLICATION_MODE}" == 'publish' ]]; then + if [[ "$source_sha" == "$develop_sha" ]]; then + mode='required' + reason='release-source-matches-current-develop' + else + mode='deferred-release-source-not-current-develop' + reason='release-source-differs-from-current-develop' + fi + fi + + { + echo "mode=$mode" + echo "reason=$reason" + } >> "$GITHUB_OUTPUT" + + { + echo '## Downstream Proving Gate' + echo + echo "- Publication mode: \`${RELEASE_PUBLICATION_MODE}\`" + echo "- Release source SHA: \`$source_sha\`" + echo "- Current develop SHA: \`$develop_sha\`" + echo "- Enforcement mode: \`$mode\`" + echo "- Reason: \`$reason\`" + if [[ "$mode" == 'deferred-release-source-not-current-develop' ]]; then + echo '- Exact-SHA downstream proving is deferred because `downstream-promotion.yml` proves `develop`, not a release-only branch head.' + echo '- Keep the selection artifact for evidence, then enforce exact downstream proof after finalize or back-merge aligns `develop` to the release source.' + fi + } >> "$GITHUB_STEP_SUMMARY" + - name: Resolve downstream proving artifact selection id: downstream_proving_artifacts if: always() @@ -956,7 +1005,7 @@ jobs: --trust tests/results/_agent/supply-chain/release-trust-gate.json --output tests/results/_agent/release/release-scorecard.json ) - if [[ "${RELEASE_PUBLICATION_MODE}" == 'publish' ]]; then + if [[ "${{ steps.downstream_proving_policy.outputs.mode }}" == 'required' ]]; then scorecard_args+=( --downstream-proving-selection "${downstream_proving_selection_path}" --require-downstream-proving diff --git a/docs/DOWNSTREAM_DEVELOP_PROMOTION_CONTRACT.md b/docs/DOWNSTREAM_DEVELOP_PROMOTION_CONTRACT.md index d9290631a..075414afb 100644 --- a/docs/DOWNSTREAM_DEVELOP_PROMOTION_CONTRACT.md +++ b/docs/DOWNSTREAM_DEVELOP_PROMOTION_CONTRACT.md @@ -212,6 +212,13 @@ That means release should select the `downstream-promotion.yml` artifact whose Release evidence should retain the machine-readable selection report that points back to the exact downstream promotion run and scorecard artifact used. +Because this contract proves `upstream/develop`, release automation must not +fail closed on exact-SHA downstream proving during the first publication of a +tag whose source commit does not yet match the current `develop` head. In that +case the selection report remains required evidence, but release scorecard +blocking is deferred until finalize/back-merge or another replay aligns +`develop` to the released source commit. + The proving artifact is authoritative even when the workflow cannot update `downstream/develop` directly because repository rules require the branch PR or merge-queue path. In that case the workflow records the handoff in the step diff --git a/docs/RELEASE_PROMOTION_CONTRACT.md b/docs/RELEASE_PROMOTION_CONTRACT.md index c5b217cd6..20313240b 100644 --- a/docs/RELEASE_PROMOTION_CONTRACT.md +++ b/docs/RELEASE_PROMOTION_CONTRACT.md @@ -316,6 +316,16 @@ Scorecard blockers are fail-closed when any gate regresses: - downstream proving selection report missing, invalid, or not aligned to the selected downstream promotion scorecard when downstream proving is required +Downstream proving is required during initial release publication only when the +release source commit matches the current `develop` head that +`downstream-promotion.yml` can actually prove. When publication targets a +release-branch commit that has not yet been re-aligned with `develop`, release +workflows must still emit the downstream proving selection artifact, but the +scorecard must treat that proof as deferred evidence instead of an immediate +blocking gate. Exact-SHA downstream proving becomes blocking again after +finalize or back-merge restores `develop` alignment, or during a dedicated +post-publication replay that runs against the aligned source. + The SLO artifact now has two distinct surfaces: - `breaches` diff --git a/tools/priority/__tests__/workflow-pwsh-continuation-contract.test.mjs b/tools/priority/__tests__/workflow-pwsh-continuation-contract.test.mjs index 034d4ca98..340325399 100644 --- a/tools/priority/__tests__/workflow-pwsh-continuation-contract.test.mjs +++ b/tools/priority/__tests__/workflow-pwsh-continuation-contract.test.mjs @@ -130,6 +130,8 @@ test('release workflow resolves downloaded artifacts through the shared helper b const releaseContractJob = workflow?.jobs?.['release-contract']; const releaseContractSteps = releaseContractJob?.steps ?? []; const releaseSourceIndex = releaseContractSteps.findIndex((step) => step?.name === 'Resolve release source commit'); + const currentDevelopIndex = releaseContractSteps.findIndex((step) => step?.name === 'Resolve current develop head'); + const downstreamPolicyIndex = releaseContractSteps.findIndex((step) => step?.name === 'Decide downstream proving enforcement mode'); const downstreamResolveIndex = releaseContractSteps.findIndex((step) => step?.name === 'Resolve downstream proving artifact selection'); const buildScorecardIndex = releaseContractSteps.findIndex((step) => step?.name === 'Build release scorecard'); @@ -139,7 +141,15 @@ test('release workflow resolves downloaded artifacts through the shared helper b assert.ok(uploadScenarioIndex > writeScenarioIndex, 'validate-cli-artifacts should upload scenario summaries after writing them'); assert.ok(enforceValidationIndex > uploadScenarioIndex, 'validate-cli-artifacts should fail only after scenario artifacts upload'); assert.ok(releaseSourceIndex >= 0, 'release-contract should resolve the expected release source commit'); - assert.ok(downstreamResolveIndex > releaseSourceIndex, 'release-contract should resolve downstream proving artifacts after the release source is known'); + assert.ok(currentDevelopIndex > releaseSourceIndex, 'release-contract should resolve the current develop head after the release source is known'); + assert.ok( + downstreamPolicyIndex > currentDevelopIndex, + 'release-contract should decide downstream proving enforcement after resolving the current develop head' + ); + assert.ok( + downstreamResolveIndex > downstreamPolicyIndex, + 'release-contract should resolve downstream proving artifacts after the enforcement mode is known' + ); assert.ok(buildScorecardIndex > downstreamResolveIndex, 'release-contract should build the release scorecard after downstream proving resolution'); assert.match(workflowRaw, /name: Resolve validation artifact paths/); assert.match(workflowRaw, /Resolve-DownloadedArtifactPath\.ps1/); @@ -156,6 +166,16 @@ test('release workflow resolves downloaded artifacts through the shared helper b assert.match(workflowRaw, /name: Resolve release source commit/); assert.match(workflowRaw, /git fetch --force --tags origin "refs\/tags\/\$\{RELEASE_TAG\}:refs\/tags\/\$\{RELEASE_TAG\}"/); assert.match(workflowRaw, /git rev-parse "\$\{RELEASE_TAG\}\^\{commit\}"/); + assert.match(workflowRaw, /name: Resolve current develop head/); + assert.match(workflowRaw, /git fetch --force origin "refs\/heads\/develop:refs\/remotes\/origin\/develop"/); + assert.match(workflowRaw, /git rev-parse "refs\/remotes\/origin\/develop\^\{commit\}"/); + assert.match(workflowRaw, /name: Decide downstream proving enforcement mode/); + assert.match(workflowRaw, /mode='advisory-replay'/); + assert.match(workflowRaw, /if \[\[ "\$\{RELEASE_PUBLICATION_MODE\}" == 'publish' \]\]; then/); + assert.match(workflowRaw, /if \[\[ "\$source_sha" == "\$develop_sha" \]\]; then/); + assert.match(workflowRaw, /mode='required'/); + assert.match(workflowRaw, /mode='deferred-release-source-not-current-develop'/); + assert.match(workflowRaw, /## Downstream Proving Gate/); assert.match(workflowRaw, /RELEASE_PUBLICATION_MODE:\s*\$\{\{\s*needs\.certification-matrix\.outputs\.publication_mode\s*\}\}/); assert.match(workflowRaw, /name: Download supply-chain trust artifact/); assert.match(workflowRaw, /path:\s*tests\/results\/_agent/); @@ -169,8 +189,14 @@ test('release workflow resolves downloaded artifacts through the shared helper b assert.match(workflowRaw, /steps\.downstream_proving_artifacts\.outputs\.downstream_proving_scorecard_path/); assert.match(workflowRaw, /downstream_promotion_path="\$\{\{\s*steps\.downstream_proving_artifacts\.outputs\.downstream_proving_scorecard_path\s*\}\}"/); assert.match(workflowRaw, /downstream_proving_selection_path='tests\/results\/_agent\/release\/downstream-proving-selection\.json'/); - assert.match(workflowRaw, /if \[\[ "\$\{RELEASE_PUBLICATION_MODE\}" == 'publish' \]\]; then/); - assert.match(workflowRaw, /scorecard_args\+=\(\s*--downstream-proving-selection "\$\{downstream_proving_selection_path\}"\s*--require-downstream-proving\s*\)/ms); + assert.match( + workflowRaw, + /if \[\[ "\$\{\{\s*steps\.downstream_proving_policy\.outputs\.mode\s*\}\}" == 'required' \]\]; then/ + ); + assert.match( + workflowRaw, + /scorecard_args\+=\(\s*--downstream-proving-selection "\$\{downstream_proving_selection_path\}"\s*--require-downstream-proving\s*\)/ms + ); assert.match(workflowRaw, /if \[ -n "\$\{downstream_promotion_path\}" \]; then/); assert.match(workflowRaw, /scorecard_args\+=\(--downstream-promotion "\$\{downstream_promotion_path\}"\)/); assert.match(workflowRaw, /tools\/release-review\/Evaluate-ReleaseReviewPolicy\.ps1/); From 7f07a2136eccefffff1cf0b7e2cd65f6c7631a1a Mon Sep 17 00:00:00 2001 From: svelderrainruiz Date: Wed, 1 Apr 2026 05:50:38 -0700 Subject: [PATCH 37/44] release: carry conductor fixes onto develop --- .github/workflows/release-conductor.yml | 9 - .../release-conductor-report-v1.schema.json | 43 +-- .../release-conductor-schema.test.mjs | 1 - .../__tests__/release-conductor.test.mjs | 325 +++++++++--------- tools/priority/release-conductor.mjs | 233 ++++--------- 5 files changed, 232 insertions(+), 379 deletions(-) diff --git a/.github/workflows/release-conductor.yml b/.github/workflows/release-conductor.yml index 4c9f692ef..726ae7e19 100644 --- a/.github/workflows/release-conductor.yml +++ b/.github/workflows/release-conductor.yml @@ -25,10 +25,6 @@ on: description: 'Release version proposal (for example 0.8.0 or 0.8.0-rc.1)' required: false type: string - dwell_minutes: - description: 'Green dwell minutes (default 60)' - required: false - type: string quarantine_stale_hours: description: 'Quarantine stale threshold hours (default 24)' required: false @@ -196,11 +192,6 @@ jobs: $args += @('--version', $versionInput.Trim()) } - $dwellInput = '${{ inputs.dwell_minutes }}' - if (-not [string]::IsNullOrWhiteSpace($dwellInput)) { - $args += @('--dwell-minutes', $dwellInput.Trim()) - } - $quarantineInput = '${{ inputs.quarantine_stale_hours }}' if (-not [string]::IsNullOrWhiteSpace($quarantineInput)) { $args += @('--quarantine-stale-hours', $quarantineInput.Trim()) diff --git a/docs/schemas/release-conductor-report-v1.schema.json b/docs/schemas/release-conductor-report-v1.schema.json index 4d15593b9..2b0b2639d 100644 --- a/docs/schemas/release-conductor-report-v1.schema.json +++ b/docs/schemas/release-conductor-report-v1.schema.json @@ -241,61 +241,20 @@ "reportPath", "queueReportPath", "policySnapshotPath", - "dwellMinutes", "quarantineStaleHours" ], "properties": { "reportPath": { "type": "string" }, "queueReportPath": { "type": "string" }, "policySnapshotPath": { "type": "string" }, - "dwellMinutes": { "type": "integer", "minimum": 0 }, "quarantineStaleHours": { "type": "integer", "minimum": 0 } } }, "gates": { "type": "object", "additionalProperties": false, - "required": ["greenDwell", "queueHealth", "policySnapshot", "quarantine"], + "required": ["queueHealth", "policySnapshot", "quarantine"], "properties": { - "greenDwell": { - "type": "object", - "additionalProperties": false, - "required": ["status", "dwellMinutes", "evaluatedAt", "reasons", "workflows"], - "properties": { - "status": { "type": "string", "enum": ["pass", "fail"] }, - "dwellMinutes": { "type": "integer", "minimum": 0 }, - "evaluatedAt": { "type": "string", "format": "date-time" }, - "reasons": { - "type": "array", - "items": { "type": "string" } - }, - "workflows": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "required": [ - "name", - "file", - "runCount", - "inWindowCount", - "successInWindow", - "failureInWindow", - "latestSuccessAt" - ], - "properties": { - "name": { "type": "string" }, - "file": { "type": "string" }, - "runCount": { "type": "integer", "minimum": 0 }, - "inWindowCount": { "type": "integer", "minimum": 0 }, - "successInWindow": { "type": "boolean" }, - "failureInWindow": { "type": "boolean" }, - "latestSuccessAt": { "type": ["string", "null"] } - } - } - } - } - }, "queueHealth": { "type": "object", "additionalProperties": false, diff --git a/tools/priority/__tests__/release-conductor-schema.test.mjs b/tools/priority/__tests__/release-conductor-schema.test.mjs index 0f389d894..090eb511e 100644 --- a/tools/priority/__tests__/release-conductor-schema.test.mjs +++ b/tools/priority/__tests__/release-conductor-schema.test.mjs @@ -68,7 +68,6 @@ test('release conductor report validates schema', async () => { stream: 'comparevi-cli', channel: 'stable', version: '0.8.0', - dwellMinutes: 60, quarantineStaleHours: 24, help: false }, diff --git a/tools/priority/__tests__/release-conductor.test.mjs b/tools/priority/__tests__/release-conductor.test.mjs index 78f2a2c9f..f3c944a49 100644 --- a/tools/priority/__tests__/release-conductor.test.mjs +++ b/tools/priority/__tests__/release-conductor.test.mjs @@ -3,7 +3,6 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { - evaluateGreenDwell, evaluatePolicySnapshotGate, evaluateQuarantineGate, evaluateQueueHealthGate, @@ -11,38 +10,12 @@ import { runReleaseConductor } from '../release-conductor.mjs'; -function makeWorkflowRunsResponse(workflowFile) { - if (workflowFile.includes('validate.yml')) { - return { - workflow_runs: [ - { - id: 1, - status: 'completed', - conclusion: 'success', - updated_at: '2026-03-06T11:45:00Z' - } - ] - }; - } - return { - workflow_runs: [ - { - id: 2, - status: 'completed', - conclusion: 'success', - updated_at: '2026-03-06T11:40:00Z' - } - ] - }; -} - test('parseArgs applies defaults and supports burst-style apply flags', () => { const defaults = parseArgs(['node', 'release-conductor.mjs']); assert.equal(defaults.apply, false); assert.equal(defaults.dryRun, true); assert.equal(defaults.repairExistingTag, false); assert.equal(defaults.channel, 'stable'); - assert.equal(defaults.dwellMinutes, 60); const parsed = parseArgs([ 'node', @@ -55,8 +28,6 @@ test('parseArgs applies defaults and supports burst-style apply flags', () => { 'rc', '--version', '0.8.0-rc.1', - '--dwell-minutes', - '30', '--quarantine-stale-hours', '12' ]); @@ -66,22 +37,11 @@ test('parseArgs applies defaults and supports burst-style apply flags', () => { assert.equal(parsed.repo, 'owner/repo'); assert.equal(parsed.channel, 'rc'); assert.equal(parsed.version, '0.8.0-rc.1'); - assert.equal(parsed.dwellMinutes, 30); assert.equal(parsed.quarantineStaleHours, 12); }); test('gate evaluators classify pass/fail deterministically', () => { const now = new Date('2026-03-06T12:00:00.000Z'); - const green = evaluateGreenDwell({ - now, - dwellMinutes: 60, - workflowRunsByName: { - Validate: [{ status: 'completed', conclusion: 'success', updated_at: '2026-03-06T11:30:00Z' }], - 'Policy Guard (Upstream)': [{ status: 'completed', conclusion: 'success', updated_at: '2026-03-06T11:35:00Z' }] - } - }); - assert.equal(green.status, 'pass'); - const queueFail = evaluateQueueHealthGate({ exists: true, error: null, @@ -99,18 +59,22 @@ test('gate evaluators classify pass/fail deterministically', () => { payload: { paused: true, pausedReasons: ['success-rate-below-threshold'], - throughputController: { mode: 'stabilize' }, - runtimeFleet: { - totals: { - queued: 0, - inProgress: 0, - stalled: 0 - } + controls: { + pausedByVariable: false, + queueAutopilotPaused: false }, + throughputController: { mode: 'stabilize' }, queueInventory: { - mergeQueueOccupancy: 0, + mergeQueueOccupancy: 2, readyQueuedCount: 0 }, + burst: { + active: false, + backoffActive: true, + triggerSignals: { + releaseBranchPullRequest: true + } + }, summary: { quarantinedCount: 0 } @@ -119,7 +83,45 @@ test('gate evaluators classify pass/fail deterministically', () => { assert.equal(queueIdlePass.status, 'pass'); assert.equal(queueIdlePass.paused, true); assert.equal(queueIdlePass.controllerMode, 'stabilize'); - assert.deepEqual(queueIdlePass.reasons, ['release-safe-idle-queue-pause']); + assert.deepEqual(queueIdlePass.reasons, ['release-safe-generic-stabilize-pause']); + + const explicitReleasePause = evaluateQueueHealthGate({ + exists: true, + error: null, + payload: { + paused: true, + pausedReasons: ['success-rate-below-threshold'], + controls: { + pausedByVariable: true, + queueAutopilotPaused: false + }, + throughputController: { mode: 'stabilize' } + } + }); + assert.equal(explicitReleasePause.status, 'fail'); + assert.ok(explicitReleasePause.reasons.includes('release-queue-explicit-pause')); + + const activeReleaseQueue = evaluateQueueHealthGate({ + exists: true, + error: null, + payload: { + paused: true, + pausedReasons: ['success-rate-below-threshold'], + controls: { + pausedByVariable: false, + queueAutopilotPaused: false + }, + throughputController: { mode: 'stabilize' }, + burst: { + active: true, + triggerSignals: { + releaseBranchPullRequest: true + } + } + } + }); + assert.equal(activeReleaseQueue.status, 'fail'); + assert.ok(activeReleaseQueue.reasons.includes('release-queue-activity-active')); const policyPass = evaluatePolicySnapshotGate({ exists: true, @@ -219,7 +221,6 @@ test('runReleaseConductor blocks apply when release conductor flag is disabled', stream: 'comparevi-cli', channel: 'stable', version: '0.8.0', - dwellMinutes: 60, quarantineStaleHours: 24, help: false }, @@ -262,22 +263,6 @@ test('runReleaseConductor keeps dry-run proposal-only when queue evidence is mis }; }; - const runGhJsonFn = (args) => { - if (args[0] !== 'api') { - throw new Error(`unexpected gh args: ${args.join(' ')}`); - } - return { - workflow_runs: [ - { - id: 1, - status: 'completed', - conclusion: 'success', - updated_at: '2026-03-06T09:00:00Z' - } - ] - }; - }; - const { report, exitCode } = await runReleaseConductor({ repoRoot: process.cwd(), now: new Date('2026-03-06T12:00:00.000Z'), @@ -291,7 +276,6 @@ test('runReleaseConductor keeps dry-run proposal-only when queue evidence is mis stream: 'comparevi-cli', channel: 'stable', version: '0.8.0', - dwellMinutes: 60, quarantineStaleHours: 24, help: false }, @@ -299,7 +283,6 @@ test('runReleaseConductor keeps dry-run proposal-only when queue evidence is mis GITHUB_REPOSITORY: 'owner/repo', RELEASE_CONDUCTOR_ENABLED: '0' }, - runGhJsonFn, runCommandFn: () => ({ status: 0, stdout: '', stderr: '' }), readJsonOptionalFn, writeReportFn: async (reportPath) => reportPath @@ -308,89 +291,12 @@ test('runReleaseConductor keeps dry-run proposal-only when queue evidence is mis assert.equal(exitCode, 0); assert.equal(report.decision.status, 'pass'); assert.equal(report.release.proposalOnly, true); - assert.equal(report.gates.greenDwell.status, 'fail'); assert.equal(report.gates.queueHealth.status, 'fail'); assert.equal(report.gates.quarantine.status, 'fail'); assert.equal(report.decision.blockerCount, 0); - assert.ok(report.decision.advisories.some((entry) => entry.code === 'green-dwell-no-recent-success')); assert.ok(report.decision.advisories.some((entry) => entry.code === 'queue-report-unavailable-dry-run')); }); -test('runReleaseConductor still blocks dry-run when the dwell window contains workflow failures', async () => { - const readJsonOptionalFn = async (filePath) => { - const normalized = String(filePath); - if (normalized.includes('queue-supervisor-report.json')) { - return { - exists: true, - error: null, - path: filePath, - payload: { - paused: false, - throughputController: { mode: 'healthy' }, - retryHistory: {} - } - }; - } - return { - exists: true, - error: null, - path: filePath, - payload: { - schema: 'priority/policy-live-state@v1', - generatedAt: '2026-03-06T10:00:00Z', - state: {} - } - }; - }; - - const runGhJsonFn = (args) => { - if (args[0] !== 'api') { - throw new Error(`unexpected gh args: ${args.join(' ')}`); - } - return { - workflow_runs: [ - { - id: 1, - status: 'completed', - conclusion: 'failure', - updated_at: '2026-03-06T11:45:00Z' - } - ] - }; - }; - - const { report, exitCode } = await runReleaseConductor({ - repoRoot: process.cwd(), - now: new Date('2026-03-06T12:00:00.000Z'), - args: { - apply: false, - dryRun: true, - reportPath: 'tests/results/_agent/release/release-conductor-report.json', - queueReportPath: 'tests/results/_agent/queue/queue-supervisor-report.json', - policySnapshotPath: 'tests/results/_agent/policy/policy-state-snapshot.json', - repo: 'owner/repo', - stream: 'comparevi-cli', - channel: 'stable', - version: '0.8.0', - dwellMinutes: 60, - quarantineStaleHours: 24, - help: false - }, - environment: { - GITHUB_REPOSITORY: 'owner/repo', - RELEASE_CONDUCTOR_ENABLED: '0' - }, - runGhJsonFn, - runCommandFn: () => ({ status: 0, stdout: '', stderr: '' }), - readJsonOptionalFn, - writeReportFn: async (reportPath) => reportPath - }); - - assert.equal(exitCode, 1); - assert.equal(report.decision.status, 'fail'); - assert.ok(report.decision.blockers.some((entry) => entry.code === 'green-dwell-failed')); -}); - test('runReleaseConductor allows proposal-only dry-run without an explicit version', async () => { const readJsonOptionalFn = async (filePath) => { const normalized = String(filePath); @@ -439,7 +345,6 @@ test('runReleaseConductor allows proposal-only dry-run without an explicit versi stream: 'comparevi-cli', channel: 'stable', version: null, - dwellMinutes: 60, quarantineStaleHours: 24, help: false }, @@ -545,7 +450,6 @@ test('runReleaseConductor creates and publishes a signed tag when apply is enabl stream: 'comparevi-cli', channel: 'stable', version: '0.8.0', - dwellMinutes: 60, quarantineStaleHours: 24, help: false }, @@ -566,11 +470,35 @@ test('runReleaseConductor creates and publishes a signed tag when apply is enabl assert.equal(report.release.tagPushed, true); assert.equal(report.release.tagPushRemote.remoteName, 'upstream'); assert.equal(report.release.signingMaterial.backend, 'ssh'); + assert.equal(report.release.publicationReplay.requested, true); + assert.equal(report.release.publicationReplay.status, 'dispatched'); + assert.equal(report.release.publicationReplay.dispatched, true); + assert.equal(report.release.publicationReplay.ref, 'develop'); + assert.equal(report.release.publicationReplay.tagInputName, 'release_tag'); + assert.equal(report.release.publicationReplay.tagInputValue, 'v0.8.0'); + assert.equal(report.release.publicationReplay.modeInputName, 'publication_mode'); + assert.equal(report.release.publicationReplay.modeInputValue, 'publish'); assert.ok(commandCalls.some((entry) => entry.command === 'git' && entry.args[0] === 'tag')); assert.ok(commandCalls.some((entry) => entry.command === 'git' && entry.args[0] === 'push')); + assert.equal( + commandCalls.some( + (entry) => + entry.command === 'gh' && + entry.args[0] === 'workflow' && + entry.args[1] === 'run' && + entry.args[2] === 'release.yml' && + entry.args[3] === '--ref' && + entry.args[4] === 'develop' && + entry.args[5] === '-f' && + entry.args[6] === 'release_tag=v0.8.0' && + entry.args[7] === '-f' && + entry.args[8] === 'publication_mode=publish' + ), + true + ); }); -test('runReleaseConductor allows apply when queue pause is only an idle success-rate throttle', async () => { +test('runReleaseConductor allows apply when queue pause is only a generic success-rate throttle', async () => { const readJsonOptionalFn = async (filePath) => { const normalized = String(filePath); if (normalized.includes('queue-supervisor-report.json')) { @@ -581,18 +509,22 @@ test('runReleaseConductor allows apply when queue pause is only an idle success- payload: { paused: true, pausedReasons: ['success-rate-below-threshold'], - throughputController: { mode: 'stabilize' }, - runtimeFleet: { - totals: { - queued: 0, - inProgress: 0, - stalled: 0 - } + controls: { + pausedByVariable: false, + queueAutopilotPaused: false }, + throughputController: { mode: 'stabilize' }, queueInventory: { - mergeQueueOccupancy: 0, + mergeQueueOccupancy: 2, readyQueuedCount: 0 }, + burst: { + active: false, + backoffActive: true, + triggerSignals: { + releaseBranchPullRequest: true + } + }, summary: { quarantinedCount: 0 }, @@ -657,7 +589,6 @@ test('runReleaseConductor allows apply when queue pause is only an idle success- stream: 'comparevi-cli', channel: 'rc', version: '0.8.0-rc.1', - dwellMinutes: 60, quarantineStaleHours: 24, help: false }, @@ -673,12 +604,82 @@ test('runReleaseConductor allows apply when queue pause is only an idle success- assert.equal(exitCode, 0); assert.equal(report.gates.queueHealth.status, 'pass'); - assert.deepEqual(report.gates.queueHealth.reasons, ['release-safe-idle-queue-pause']); + assert.deepEqual(report.gates.queueHealth.reasons, ['release-safe-generic-stabilize-pause']); assert.equal(report.release.proposalOnly, false); assert.ok(commandCalls.some((entry) => entry.command === 'git' && entry.args[0] === 'tag')); assert.ok(commandCalls.some((entry) => entry.command === 'git' && entry.args[0] === 'push')); }); +test('runReleaseConductor blocks apply when queue pause reflects active release queue activity', async () => { + const readJsonOptionalFn = async (filePath) => { + const normalized = String(filePath); + if (normalized.includes('queue-supervisor-report.json')) { + return { + exists: true, + error: null, + path: filePath, + payload: { + paused: true, + pausedReasons: ['success-rate-below-threshold'], + controls: { + pausedByVariable: false, + queueAutopilotPaused: false + }, + throughputController: { mode: 'stabilize' }, + burst: { + active: true, + triggerSignals: { + releaseBranchPullRequest: true + } + }, + retryHistory: {} + } + }; + } + return { + exists: true, + error: null, + path: filePath, + payload: { + schema: 'priority/policy-live-state@v1', + generatedAt: '2026-03-06T10:00:00Z', + state: {} + } + }; + }; + + const { report, exitCode } = await runReleaseConductor({ + repoRoot: process.cwd(), + now: new Date('2026-03-06T12:00:00.000Z'), + args: { + apply: true, + dryRun: false, + repairExistingTag: false, + reportPath: 'tests/results/_agent/release/release-conductor-report.json', + queueReportPath: 'tests/results/_agent/queue/queue-supervisor-report.json', + policySnapshotPath: 'tests/results/_agent/policy/policy-state-snapshot.json', + repo: 'owner/repo', + stream: 'comparevi-cli', + channel: 'stable', + version: '0.8.0', + quarantineStaleHours: 24, + help: false + }, + environment: { + GITHUB_REPOSITORY: 'owner/repo', + RELEASE_CONDUCTOR_ENABLED: '1' + }, + runCommandFn: () => ({ status: 0, stdout: '', stderr: '' }), + readJsonOptionalFn, + writeReportFn: async (reportPath) => reportPath + }); + + assert.equal(exitCode, 1); + assert.equal(report.gates.queueHealth.status, 'fail'); + assert.ok(report.gates.queueHealth.reasons.includes('release-queue-activity-active')); + assert.ok(report.decision.blockers.some((entry) => entry.code === 'queue-health-failed')); +}); + test('runReleaseConductor blocks apply when authoritative tag already exists and repair mode is not requested', async () => { const readJsonOptionalFn = async (filePath) => { const normalized = String(filePath); @@ -758,7 +759,6 @@ test('runReleaseConductor blocks apply when authoritative tag already exists and stream: 'comparevi-cli', channel: 'rc', version: '0.8.0-rc.1', - dwellMinutes: 60, quarantineStaleHours: 24, help: false }, @@ -851,7 +851,6 @@ test('runReleaseConductor reports a repair plan in dry-run for an existing autho stream: 'comparevi-cli', channel: 'rc', version: '0.8.0-rc.1', - dwellMinutes: 60, quarantineStaleHours: 24, help: false }, @@ -965,7 +964,6 @@ test('runReleaseConductor repairs an existing authoritative tag when repair mode stream: 'comparevi-cli', channel: 'rc', version: '0.8.0-rc.1', - dwellMinutes: 60, quarantineStaleHours: 24, help: false }, @@ -1128,7 +1126,6 @@ test('runReleaseConductor dispatches protected-tag-safe replay when the publishe stream: 'comparevi-cli', channel: 'rc', version: '0.8.0-rc.1', - dwellMinutes: 60, quarantineStaleHours: 24, help: false }, @@ -1272,7 +1269,6 @@ test('runReleaseConductor reports equivalent replay availability in dry-run for stream: 'comparevi-cli', channel: 'rc', version: '0.8.0-rc.1', - dwellMinutes: 60, quarantineStaleHours: 24, help: false }, @@ -1387,7 +1383,6 @@ test('runReleaseConductor fails apply when repaired tag publication replay dispa stream: 'comparevi-cli', channel: 'rc', version: '0.8.0-rc.1', - dwellMinutes: 60, quarantineStaleHours: 24, help: false }, @@ -1481,7 +1476,6 @@ test('runReleaseConductor fails apply when signed tag push remote is unavailable stream: 'comparevi-cli', channel: 'stable', version: '0.8.0', - dwellMinutes: 60, quarantineStaleHours: 24, help: false }, @@ -1560,7 +1554,6 @@ test('runReleaseConductor blocks apply when signing material is unavailable', as stream: 'comparevi-cli', channel: 'stable', version: '0.8.0', - dwellMinutes: 60, quarantineStaleHours: 24, help: false }, diff --git a/tools/priority/release-conductor.mjs b/tools/priority/release-conductor.mjs index 649ded3ec..abf7ac0f0 100644 --- a/tools/priority/release-conductor.mjs +++ b/tools/priority/release-conductor.mjs @@ -11,7 +11,6 @@ export const REPORT_SCHEMA = 'release/release-conductor-report@v1'; export const DEFAULT_REPORT_PATH = path.join('tests', 'results', '_agent', 'release', 'release-conductor-report.json'); export const DEFAULT_QUEUE_REPORT_PATH = path.join('tests', 'results', '_agent', 'queue', 'queue-supervisor-report.json'); export const DEFAULT_POLICY_SNAPSHOT_PATH = path.join('tests', 'results', '_agent', 'policy', 'policy-state-snapshot.json'); -export const DEFAULT_DWELL_MINUTES = 60; export const DEFAULT_QUARANTINE_STALE_HOURS = 24; export const RELEASE_PUBLICATION_WORKFLOW = 'release.yml'; export const RELEASE_PUBLICATION_WORKFLOW_REF = 'develop'; @@ -20,11 +19,6 @@ export const RELEASE_PUBLICATION_MODE_INPUT = 'publication_mode'; export const RELEASE_PUBLICATION_MODE_PUBLISH = 'publish'; export const RELEASE_PUBLICATION_MODE_VERIFY_EXISTING_RELEASE = 'verify-existing-release'; -const REQUIRED_DWELL_WORKFLOWS = Object.freeze([ - { name: 'Validate', file: 'validate.yml' }, - { name: 'Policy Guard (Upstream)', file: 'policy-guard-upstream.yml' } -]); - function printUsage() { console.log('Usage: node tools/priority/release-conductor.mjs [options]'); console.log(''); @@ -38,7 +32,6 @@ function printUsage() { console.log(' --stream Release stream name (default: comparevi-cli).'); console.log(' --channel Release channel (default: stable).'); console.log(' --version Proposed version used for release tag proposal (optional).'); - console.log(` --dwell-minutes Required green dwell window in minutes (default: ${DEFAULT_DWELL_MINUTES}).`); console.log(` --quarantine-stale-hours Fail when queue quarantine is stale beyond N hours (default: ${DEFAULT_QUARANTINE_STALE_HOURS}).`); console.log(' --dry-run Force dry-run mode.'); console.log(' -h, --help Show this help text and exit.'); @@ -111,7 +104,6 @@ export function parseArgs(argv = process.argv) { stream: 'comparevi-cli', channel: 'stable', version: null, - dwellMinutes: DEFAULT_DWELL_MINUTES, quarantineStaleHours: DEFAULT_QUARANTINE_STALE_HOURS, help: false }; @@ -145,7 +137,6 @@ export function parseArgs(argv = process.argv) { token === '--stream' || token === '--channel' || token === '--version' || - token === '--dwell-minutes' || token === '--quarantine-stale-hours' ) { const next = args[index + 1]; @@ -160,7 +151,6 @@ export function parseArgs(argv = process.argv) { if (token === '--stream') options.stream = next; if (token === '--channel') options.channel = next.trim().toLowerCase(); if (token === '--version') options.version = next; - if (token === '--dwell-minutes') options.dwellMinutes = parseIntStrict(next, { label: '--dwell-minutes' }); if (token === '--quarantine-stale-hours') { options.quarantineStaleHours = parseIntStrict(next, { label: '--quarantine-stale-hours' }); } @@ -238,66 +228,6 @@ async function readJsonOptional(filePath) { } } -function normalizeUpdatedAt(entry) { - const updatedAt = new Date(entry?.updated_at ?? entry?.updatedAt ?? 0); - return Number.isNaN(updatedAt.valueOf()) ? null : updatedAt; -} - -function normalizeConclusion(entry) { - return String(entry?.conclusion ?? '').trim().toLowerCase(); -} - -export function evaluateGreenDwell({ - workflowRunsByName = {}, - now = new Date(), - dwellMinutes = DEFAULT_DWELL_MINUTES -} = {}) { - const dwellStartMs = now.valueOf() - dwellMinutes * 60 * 1000; - const failureConclusions = new Set(['failure', 'cancelled', 'timed_out', 'action_required', 'startup_failure']); - const details = []; - const reasons = []; - - for (const spec of REQUIRED_DWELL_WORKFLOWS) { - const runs = Array.isArray(workflowRunsByName[spec.name]) ? workflowRunsByName[spec.name] : []; - const inWindow = runs.filter((entry) => { - const updatedAt = normalizeUpdatedAt(entry); - return updatedAt && updatedAt.valueOf() >= dwellStartMs; - }); - const successInWindow = inWindow.some((entry) => normalizeConclusion(entry) === 'success'); - const failureInWindow = inWindow.some((entry) => failureConclusions.has(normalizeConclusion(entry))); - const latestSuccess = runs - .filter((entry) => normalizeConclusion(entry) === 'success') - .map((entry) => normalizeUpdatedAt(entry)?.toISOString() ?? null) - .find(Boolean); - - const status = successInWindow && !failureInWindow ? 'pass' : 'fail'; - if (!successInWindow) { - reasons.push(`no-success-${spec.file}`); - } - if (failureInWindow) { - reasons.push(`failure-in-window-${spec.file}`); - } - - details.push({ - name: spec.name, - file: spec.file, - runCount: runs.length, - inWindowCount: inWindow.length, - successInWindow, - failureInWindow, - latestSuccessAt: latestSuccess - }); - } - - return { - status: reasons.length === 0 ? 'pass' : 'fail', - dwellMinutes, - evaluatedAt: now.toISOString(), - reasons: [...new Set(reasons)], - workflows: details - }; -} - export function evaluateQueueHealthGate(queueReportEnvelope) { if (!queueReportEnvelope.exists || queueReportEnvelope.error || !queueReportEnvelope.payload) { return { @@ -310,45 +240,45 @@ export function evaluateQueueHealthGate(queueReportEnvelope) { const queueReport = queueReportEnvelope.payload; const paused = Boolean(queueReport?.paused); + const controls = + queueReport?.controls && typeof queueReport.controls === 'object' ? queueReport.controls : {}; + const burst = + queueReport?.burst && typeof queueReport.burst === 'object' ? queueReport.burst : {}; + const triggerSignals = + burst?.triggerSignals && typeof burst.triggerSignals === 'object' ? burst.triggerSignals : {}; const controllerMode = queueReport?.throughputController?.mode ?? queueReport?.adaptiveInflight?.mode ?? null; const pausedReasons = Array.isArray(queueReport?.pausedReasons) ? queueReport.pausedReasons : []; - const runtimeTotals = - queueReport?.runtimeFleet && typeof queueReport.runtimeFleet === 'object' ? queueReport.runtimeFleet.totals : null; - const mergeQueueOccupancy = - queueReport?.queueInventory?.mergeQueueOccupancy ?? - queueReport?.summary?.mergeQueueOccupancy ?? - null; - const readyQueuedCount = - queueReport?.queueInventory?.readyQueuedCount ?? - queueReport?.summary?.readyQueuedCount ?? - null; - const quarantinedCount = queueReport?.summary?.quarantinedCount ?? null; - const idleSuccessRatePause = + const successRateThrottlePause = paused && controllerMode === 'stabilize' && pausedReasons.length > 0 && - pausedReasons.every((reason) => reason === 'success-rate-below-threshold') && - Number(mergeQueueOccupancy ?? 0) === 0 && - Number(readyQueuedCount ?? 0) === 0 && - Number(runtimeTotals?.queued ?? 0) === 0 && - Number(runtimeTotals?.inProgress ?? 0) === 0 && - Number(runtimeTotals?.stalled ?? 0) === 0 && - Number(quarantinedCount ?? 0) === 0; + pausedReasons.every((reason) => reason === 'success-rate-below-threshold'); + const explicitOperatorPause = + Boolean(controls?.pausedByVariable) || + Boolean(controls?.queueAutopilotPaused) || + controllerMode === 'pause'; + const activeReleaseQueueActivity = + Boolean(burst?.active) && + (Boolean(triggerSignals?.releaseWindow) || + Boolean(triggerSignals?.releaseBranchPullRequest) || + Boolean(triggerSignals?.releaseBurstLabel)); const reasons = []; - if (idleSuccessRatePause) { - reasons.push('release-safe-idle-queue-pause'); + if (successRateThrottlePause && !explicitOperatorPause && !activeReleaseQueueActivity) { + reasons.push('release-safe-generic-stabilize-pause'); } - if (paused) reasons.push('queue-paused'); - if (controllerMode === 'stabilize') reasons.push('queue-stabilize-mode'); + if (explicitOperatorPause) reasons.push('release-queue-explicit-pause'); + if (activeReleaseQueueActivity) reasons.push('release-queue-activity-active'); + if (paused && !successRateThrottlePause && !explicitOperatorPause) reasons.push('queue-paused'); + if (controllerMode === 'stabilize' && !successRateThrottlePause) reasons.push('queue-stabilize-mode'); - if (idleSuccessRatePause) { + if (successRateThrottlePause && !explicitOperatorPause && !activeReleaseQueueActivity) { return { status: 'pass', - reasons: ['release-safe-idle-queue-pause'], + reasons: ['release-safe-generic-stabilize-pause'], paused, controllerMode }; @@ -457,12 +387,18 @@ function isQueueReportUnavailableGate(gate) { return reasons.length > 0 && reasons.every((reason) => reason === 'queue-report-unavailable'); } -function isDryRunGreenDwellAdvisory(gate) { - if (gate?.status !== 'fail') { - return false; - } +function describeQueueHealthBlocker(gate) { const reasons = Array.isArray(gate?.reasons) ? gate.reasons : []; - return reasons.length > 0 && reasons.every((reason) => reason.startsWith('no-success-')); + if (reasons.includes('release-queue-explicit-pause')) { + return 'Queue supervisor reported an explicit release queue pause.'; + } + if (reasons.includes('release-queue-activity-active')) { + return 'Queue supervisor reported active release queue activity.'; + } + if (reasons.includes('queue-report-unavailable')) { + return 'Queue supervisor evidence is unavailable.'; + } + return 'Queue supervisor reported release-relevant queue risk.'; } function pushUniqueDecisionEntry(entries, entry) { @@ -471,31 +407,6 @@ function pushUniqueDecisionEntry(entries, entry) { } } -function fetchWorkflowRunsByName({ runGhJsonFn, repository, branch, sampleSize, cwd }) { - const workflowRunsByName = {}; - const fetchErrors = []; - - for (const workflow of REQUIRED_DWELL_WORKFLOWS) { - const endpoint = `repos/${repository}/actions/workflows/${workflow.file}/runs?branch=${encodeURIComponent(branch)}&per_page=${sampleSize}`; - try { - const response = runGhJsonFn(['api', endpoint], { cwd }) ?? {}; - workflowRunsByName[workflow.name] = Array.isArray(response.workflow_runs) ? response.workflow_runs : []; - } catch (error) { - workflowRunsByName[workflow.name] = []; - fetchErrors.push({ - workflow: workflow.name, - file: workflow.file, - message: error?.message ?? String(error) - }); - } - } - - return { - workflowRunsByName, - fetchErrors - }; -} - function detectSigningMaterial({ runCommandFn, repoRoot, environment = process.env }) { const keyResult = runCommandFn('git', ['config', '--get', 'user.signingkey'], { cwd: repoRoot, @@ -793,19 +704,6 @@ export async function runReleaseConductor(options = {}) { const repository = resolveRepositorySlug(repoRoot, args.repo, environment); const queueReportEnvelope = await readJsonOptionalFn(path.resolve(repoRoot, args.queueReportPath)); const policySnapshotEnvelope = await readJsonOptionalFn(path.resolve(repoRoot, args.policySnapshotPath)); - const { workflowRunsByName, fetchErrors } = fetchWorkflowRunsByName({ - runGhJsonFn, - repository, - branch: 'develop', - sampleSize: 20, - cwd: repoRoot - }); - - const greenDwellGate = evaluateGreenDwell({ - workflowRunsByName, - now, - dwellMinutes: args.dwellMinutes - }); const queueHealthGate = evaluateQueueHealthGate(queueReportEnvelope); const policySnapshotGate = evaluatePolicySnapshotGate(policySnapshotEnvelope); const quarantineGate = evaluateQuarantineGate({ @@ -817,25 +715,6 @@ export async function runReleaseConductor(options = {}) { const applyRequested = Boolean(args.apply && !args.dryRun); const blockers = []; const advisories = []; - if (fetchErrors.length > 0) { - blockers.push({ - code: 'workflow-fetch-failed', - message: 'Unable to fetch required workflow run history for dwell gate.' - }); - } - if (greenDwellGate.status !== 'pass') { - if (!applyRequested && isDryRunGreenDwellAdvisory(greenDwellGate)) { - pushUniqueDecisionEntry(advisories, { - code: 'green-dwell-no-recent-success', - message: `No successful required workflow run was observed in the last ${args.dwellMinutes} minutes; dry-run remains proposal-only.` - }); - } else { - blockers.push({ - code: 'green-dwell-failed', - message: `Required workflows were not continuously green for ${args.dwellMinutes} minutes.` - }); - } - } if (queueHealthGate.status !== 'pass') { if (!applyRequested && isQueueReportUnavailableGate(queueHealthGate)) { pushUniqueDecisionEntry(advisories, { @@ -845,7 +724,7 @@ export async function runReleaseConductor(options = {}) { } else { blockers.push({ code: 'queue-health-failed', - message: 'Queue supervisor reported paused/stabilize state.' + message: describeQueueHealthBlocker(queueHealthGate) }); } } @@ -1169,6 +1048,40 @@ export async function runReleaseConductor(options = {}) { if (pushResult.status === 0) { tagPushed = true; proposalOnly = false; + publicationReplay.requested = true; + publicationReplay.modeInputValue = RELEASE_PUBLICATION_MODE_PUBLISH; + const dispatchResult = runCommandFn( + 'gh', + [ + 'workflow', + 'run', + RELEASE_PUBLICATION_WORKFLOW, + '--ref', + RELEASE_PUBLICATION_WORKFLOW_REF, + '-f', + `${RELEASE_PUBLICATION_TAG_INPUT}=${targetTag}`, + '-f', + `${RELEASE_PUBLICATION_MODE_INPUT}=${RELEASE_PUBLICATION_MODE_PUBLISH}` + ], + { + cwd: repoRoot, + allowFailure: true + } + ); + if (dispatchResult.status === 0) { + publicationReplay.dispatched = true; + publicationReplay.status = 'dispatched'; + } else { + publicationReplay.status = 'dispatch-failed'; + publicationReplay.error = + asOptional(dispatchResult.stderr) ?? + asOptional(dispatchResult.stdout) ?? + 'release workflow dispatch failed'; + blockers.push({ + code: 'release-replay-dispatch-failed', + message: `Release publication replay dispatch failed for ${targetTag}: ${publicationReplay.error}` + }); + } } else { tagPushError = asOptional(pushResult.stderr) ?? asOptional(pushResult.stdout) ?? 'tag push failed'; blockers.push({ @@ -1217,16 +1130,14 @@ export async function runReleaseConductor(options = {}) { reportPath: args.reportPath, queueReportPath: args.queueReportPath, policySnapshotPath: args.policySnapshotPath, - dwellMinutes: args.dwellMinutes, quarantineStaleHours: args.quarantineStaleHours }, gates: { - greenDwell: greenDwellGate, queueHealth: queueHealthGate, policySnapshot: policySnapshotGate, quarantine: quarantineGate }, - workflowFetchErrors: fetchErrors, + workflowFetchErrors: [], decision: { status, blockerCount: blockers.length, From 781839a035c04a1f8de86e387aedaf7df2a3f801 Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Wed, 1 Apr 2026 07:31:04 -0700 Subject: [PATCH 38/44] fix: normalize vi history paths on windows proof surfaces (#2089) * fix: normalize vi history paths on windows proof surfaces * fix: satisfy release workflow shell lint * fix: move release expressions out of shell lint path * chore: teach actionlint repo runner labels * docs: satisfy markdownlint on local proof packet files * ci: fix lychee packet workflow configuration --------- Co-authored-by: svelderrainruiz --- .github/actionlint.yaml | 10 +- .../pester-service-model-quality.yml | 1 + .../pester-service-model-release-evidence.yml | 5 +- .github/workflows/release.yml | 22 +- docs/DOWNSTREAM_DEVELOP_PROMOTION_CONTRACT.md | 11 +- ...-2078-pester-service-model-requirements.md | 1 + .../pester-service-model-control-plane.md | 30 ++- .../local-proof-autonomy-program-test-plan.md | 8 +- .../vi-history-local-proof-test-plan.md | 27 +-- ...windows-docker-shared-surface-test-plan.md | 23 +- tests/CompareVI.GitRefs.VI2.Tests.ps1 | 65 +++++- tests/TestFileExistsAtRef.Tests.ps1 | 200 ++++++++++++++++++ tools/Compare-RefsToTemp.ps1 | 55 ++++- tools/Compare-VIHistory.ps1 | 110 +++++++--- tools/Render-VIHistoryReport.ps1 | 40 +++- 15 files changed, 510 insertions(+), 98 deletions(-) diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 725de74d4..e3e499ed6 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -2,9 +2,11 @@ self-hosted-runner: labels: - comparevi - capability-ingress + - docker-lane - labview-2026 - lv32 - - docker-lane - - teststand - - self-hosted-docker-linux - - hosted-docker-linux + +paths: + .github/workflows/release.yml: + ignore: + - 'SC2016:info:.*Expressions don''t expand in single quotes' diff --git a/.github/workflows/pester-service-model-quality.yml b/.github/workflows/pester-service-model-quality.yml index 6cda6daa7..2b1d0f562 100644 --- a/.github/workflows/pester-service-model-quality.yml +++ b/.github/workflows/pester-service-model-quality.yml @@ -111,6 +111,7 @@ jobs: uses: lycheeverse/lychee-action@v2 with: fail: true + failIfEmpty: false args: >- --no-progress docs/knowledgebase/Pester-Service-Model.md diff --git a/.github/workflows/pester-service-model-release-evidence.yml b/.github/workflows/pester-service-model-release-evidence.yml index ef77845f7..33ae0375e 100644 --- a/.github/workflows/pester-service-model-release-evidence.yml +++ b/.github/workflows/pester-service-model-release-evidence.yml @@ -119,10 +119,11 @@ jobs: uses: lycheeverse/lychee-action@v2 with: fail: true + failIfEmpty: false + format: json + output: tests/results/_agent/pester-service-model/docs-link-check.json args: >- --no-progress - --format json - --output tests/results/_agent/pester-service-model/docs-link-check.json docs/knowledgebase/Pester-Service-Model.md docs/architecture/pester-service-model-control-plane.md docs/architecture/ADR-2078-pester-service-model-requirements.md diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 70c1b281a..2eafe7c81 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -787,20 +787,24 @@ jobs: id: downstream_proving_policy if: always() shell: bash + env: + RELEASE_SOURCE_SHA: ${{ steps.release_source.outputs.source_sha }} + CURRENT_DEVELOP_SHA: ${{ steps.current_develop.outputs.develop_sha }} run: | + # shellcheck disable=SC2016 set -euo pipefail - source_sha='${{ steps.release_source.outputs.source_sha }}' - develop_sha='${{ steps.current_develop.outputs.develop_sha }}' - mode='advisory-replay' - reason='release-publication-mode-not-publish' + source_sha="${RELEASE_SOURCE_SHA}" + develop_sha="${CURRENT_DEVELOP_SHA}" + mode="advisory-replay" + reason="release-publication-mode-not-publish" - if [[ "${RELEASE_PUBLICATION_MODE}" == 'publish' ]]; then + if [[ "${RELEASE_PUBLICATION_MODE}" == "publish" ]]; then if [[ "$source_sha" == "$develop_sha" ]]; then - mode='required' - reason='release-source-matches-current-develop' + mode="required" + reason="release-source-matches-current-develop" else - mode='deferred-release-source-not-current-develop' - reason='release-source-differs-from-current-develop' + mode="deferred-release-source-not-current-develop" + reason="release-source-differs-from-current-develop" fi fi diff --git a/docs/DOWNSTREAM_DEVELOP_PROMOTION_CONTRACT.md b/docs/DOWNSTREAM_DEVELOP_PROMOTION_CONTRACT.md index 075414afb..a5d719968 100644 --- a/docs/DOWNSTREAM_DEVELOP_PROMOTION_CONTRACT.md +++ b/docs/DOWNSTREAM_DEVELOP_PROMOTION_CONTRACT.md @@ -19,8 +19,15 @@ It is not a second feature-development branch. - Optional LV32 shadow proof receipt output: `tests/results/_agent/promotion/vi-history-lv32-shadow-proof-receipt.json` - Template-agent verification lane report: `tests/results/_agent/promotion/template-agent-verification-report.json` -- Authoritative template verification overlay: `tests/results/_agent/promotion/template-agent-verification-report.local.json`, projected during bootstrap from the latest matching downstream proving artifact for the current `develop` source SHA -- Supported template-proof authority synthesis: `tests/results/_agent/promotion/template-agent-verification-report.supported.json`, projected from the latest supported `template-smoke` `workflow_dispatch` proof on a supported consumer fork when that proof is aligned to the current canonical template head +- Authoritative template verification overlay: + `tests/results/_agent/promotion/template-agent-verification-report.local.json`, + projected during bootstrap from the latest matching downstream proving + artifact for the current `develop` source SHA +- Supported template-proof authority synthesis: + `tests/results/_agent/promotion/template-agent-verification-report.supported.json`, + projected from the latest supported `template-smoke` `workflow_dispatch` + proof on a supported consumer fork when that proof is aligned to the current + canonical template head - Selection resolver: `tools/priority/resolve-downstream-proving-artifact.mjs` - Selection schema: `docs/schemas/downstream-proving-selection-v1.schema.json` - Selection output: `tests/results/_agent/release/downstream-proving-selection.json` diff --git a/docs/architecture/ADR-2078-pester-service-model-requirements.md b/docs/architecture/ADR-2078-pester-service-model-requirements.md index ccc3edeb1..b19c993d7 100644 --- a/docs/architecture/ADR-2078-pester-service-model-requirements.md +++ b/docs/architecture/ADR-2078-pester-service-model-requirements.md @@ -14,6 +14,7 @@ yet fully specified or traceable as a control plane. ## Decision Create a dedicated assurance packet for the Pester service model with: + - an SRS - an RTM - an architecture packet diff --git a/docs/architecture/pester-service-model-control-plane.md b/docs/architecture/pester-service-model-control-plane.md index 40ae086da..41b9ff7df 100644 --- a/docs/architecture/pester-service-model-control-plane.md +++ b/docs/architecture/pester-service-model-control-plane.md @@ -103,10 +103,15 @@ `REQ-PSM-012` maps to selection plus explicit execution-pack entrypoints. `REQ-PSM-013` maps to `tools/PesterPathHygiene.ps1`, local harness path hygiene, and session-lock safety before dispatch. - `REQ-PSM-014` maps to `tools/Replay-PesterServiceModelArtifacts.Local.ps1`, `tools/Invoke-PesterEvidenceClassification.ps1`, and local replay of retained postprocess and evidence layers. + `REQ-PSM-014` maps to + `tools/Replay-PesterServiceModelArtifacts.Local.ps1`, + `tools/Invoke-PesterEvidenceClassification.ps1`, and local replay of + retained postprocess and evidence layers. `REQ-PSM-015` maps to the dispatch/finalize/postprocess/evidence split. `REQ-PSM-016` maps to durable execution telemetry. - `REQ-PSM-017` maps to schema-governed retained artifacts and readers, including explicit `unsupported-schema` outcomes for postprocess, evidence, and local replay. + `REQ-PSM-017` maps to schema-governed retained artifacts and readers, + including explicit `unsupported-schema` outcomes for postprocess, evidence, + and local replay. `REQ-PSM-018` maps to `pester-service-model-promotion-comparison.json`, release-evidence bundles, and representative baseline comparison rendered into the promotion dossier. @@ -115,13 +120,22 @@ `release-evidence-provenance.json`, and `promotion-dossier-provenance.json` across evidence, local replay, and promotion views. - `REQ-PSM-021` maps to operator-explainable gate outcomes, including `pester-operator-outcome.json` and shared summary or top-failure rendering from the same outcome contract. - `REQ-PSM-022` maps to the local autonomy loop that selects the next bounded requirement slice. + `REQ-PSM-021` maps to operator-explainable gate outcomes, including + `pester-operator-outcome.json` and shared summary or top-failure rendering + from the same outcome contract. + `REQ-PSM-022` maps to the local autonomy loop that selects the next bounded + requirement slice. `REQ-PSM-023` maps to the explicit autonomy policy and stop-condition surface. - `REQ-PSM-024` maps to representative retained-artifact replay compatibility, including schema-lite summary repair and legacy receipt tolerance. - `REQ-PSM-025` maps to the bounded local Windows-container surrogate for Docker Desktop Windows engine plus the pinned NI Windows image. - `REQ-PSM-026` maps to proof-check aware autonomy that reopens implemented requirements when representative replay regresses. - `REQ-PSM-027` maps to the machine-readable next-step escalation packet that hands off to the required proof surface when the current host cannot satisfy it. + `REQ-PSM-024` maps to representative retained-artifact replay + compatibility, including schema-lite summary repair and legacy receipt + tolerance. + `REQ-PSM-025` maps to the bounded local Windows-container surrogate for + Docker Desktop Windows engine plus the pinned NI Windows image. + `REQ-PSM-026` maps to proof-check aware autonomy that reopens implemented + requirements when representative replay regresses. + `REQ-PSM-027` maps to the machine-readable next-step escalation packet that + hands off to the required proof surface when the current host cannot satisfy + it. - Decision rationale: The service model exists to separate concerns and make failures classifiable by layer instead of inferred from one coupled self-hosted run. diff --git a/docs/testing/local-proof-autonomy-program-test-plan.md b/docs/testing/local-proof-autonomy-program-test-plan.md index a35f4217b..ad63b9144 100644 --- a/docs/testing/local-proof-autonomy-program-test-plan.md +++ b/docs/testing/local-proof-autonomy-program-test-plan.md @@ -10,10 +10,10 @@ | Test ID | Coverage | Layer | Priority | Notes | | --- | --- | --- | --- | --- | -| `TEST-LPAP-001` packet aggregation and requirement ranking coverage | Assurance/Program | High | Verifies the program consumes sibling packet next-step artifacts and ranks requirement work ahead of escalation work | -| `TEST-LPAP-002` shared-surface escalation merge coverage | Assurance/Program | High | Verifies shared escalations to the same external surface collapse into one bounded handoff packet | -| `TEST-LPAP-003` post-local promotion escalation coverage | Assurance/Program | High | Verifies the program emits a machine-readable promotion escalation instead of `null` when local packets are complete | -| `TEST-LPAP-004` concurrent bundle workspace safety coverage | Assurance/Program | High | Verifies packet-local CI surfaces use run-scoped audit bundle roots instead of deleting a shared `surface-bundle` workspace | +| `TEST-LPAP-001` | Packet aggregation and requirement ranking coverage | Assurance/Program | High | Verifies the program consumes sibling packet next-step artifacts and ranks requirement work ahead of escalation work | +| `TEST-LPAP-002` | Shared-surface escalation merge coverage | Assurance/Program | High | Verifies shared escalations to the same external surface collapse into one bounded handoff packet | +| `TEST-LPAP-003` | Post-local promotion escalation coverage | Assurance/Program | High | Verifies the program emits a machine-readable promotion escalation instead of `null` when local packets are complete | +| `TEST-LPAP-004` | Concurrent bundle workspace safety coverage | Assurance/Program | High | Verifies packet-local CI surfaces use run-scoped audit bundle roots instead of deleting a shared `surface-bundle` workspace | ## Entry Criteria diff --git a/docs/testing/vi-history-local-proof-test-plan.md b/docs/testing/vi-history-local-proof-test-plan.md index 4ebd390eb..fbc8c4c31 100644 --- a/docs/testing/vi-history-local-proof-test-plan.md +++ b/docs/testing/vi-history-local-proof-test-plan.md @@ -10,18 +10,18 @@ | Test ID | Coverage | Layer | Priority | Notes | | --- | --- | --- | --- | --- | -| `TEST-VHLP-001` local Windows workflow replay coverage | Contract/Replay | High | Verifies the governed `vi-history-scenarios-windows` replay lane emits a bounded receipt and compare artifact paths, including bridge-backed Windows launch from Unix or WSL coordinators | -| `TEST-VHLP-002` local refinement profile coverage | Execution/Local | High | Verifies `proof`, `dev-fast`, `warm-dev`, and `windows-mirror-proof` local refinement behavior and retained receipts | -| `TEST-VHLP-003` local operator-session coverage | Operator/Local | High | Verifies local operator-session wrappers consume the refinement helper and emit canonical local session contracts | -| `TEST-VHLP-004` workflow-readiness envelope coverage | Evidence/Decision | High | Verifies the VI History workflow-readiness envelope captures lane lifecycle, verdict, and recommendation | -| `TEST-VHLP-005` local autonomy-loop coverage | Assurance/Contract | High | Verifies local VI History CI emits a report, ranked backlog, and next-step artifact | -| `TEST-VHLP-006` next-step escalation coverage | Assurance/Contract | High | Verifies local VI History CI emits a machine-readable escalation step to the shared Windows Docker Desktop + NI image surface only after native or reachable bridge-backed Windows checks cannot satisfy it | -| `TEST-VHLP-007` shared local-program selector coverage | Assurance/Contract | High | Verifies the shared program selector can choose VI History requirement work explicitly and merge the shared Windows Docker Desktop + NI image escalation across sibling packets | -| `TEST-VHLP-008` clone-backed live-history candidate governance coverage | Assurance/Contract | High | Verifies the packet names `ni/labview-icon-editor` plus `Tooling/deployment/VIP_Pre-Uninstall Custom Action.vi` as the governed clone-backed live-history candidate | -| `TEST-VHLP-009` live-history candidate readiness coverage | Assurance/Contract | High | Verifies local VI History CI validates clone presence, target path presence, and git history, then emits a bounded clone-preparation escalation when the candidate is unavailable | -| `TEST-VHLP-010` explicit Windows replay next-step coverage | Assurance/Contract | High | Verifies local VI History CI emits `vi-history-windows-workflow-replay` as the next step when the shared Windows surface and live-history candidate are ready, instead of launching the replay lane during packet selection | -| `TEST-VHLP-011` bounded Windows replay lifecycle coverage | Contract/Replay | High | Verifies the governed Windows workflow replay lane terminates or fails closed within bounded helper-process timeouts and still emits a replay receipt on timeout | -| `TEST-VHLP-012` replay receipt consumption coverage | Assurance/Contract | High | Verifies local VI History CI treats an existing passing `vi-history-scenarios-windows` replay receipt as satisfied local replay proof and advances beyond replay re-selection | +| `TEST-VHLP-001` | Local Windows workflow replay coverage | Contract/Replay | High | Verifies the governed `vi-history-scenarios-windows` replay lane emits a bounded receipt and compare artifact paths, including bridge-backed Windows launch from Unix or WSL coordinators | +| `TEST-VHLP-002` | Local refinement profile coverage | Execution/Local | High | Verifies `proof`, `dev-fast`, `warm-dev`, and `windows-mirror-proof` local refinement behavior and retained receipts | +| `TEST-VHLP-003` | Local operator-session coverage | Operator/Local | High | Verifies local operator-session wrappers consume the refinement helper and emit canonical local session contracts | +| `TEST-VHLP-004` | Workflow-readiness envelope coverage | Evidence/Decision | High | Verifies the VI History workflow-readiness envelope captures lane lifecycle, verdict, and recommendation | +| `TEST-VHLP-005` | Local autonomy-loop coverage | Assurance/Contract | High | Verifies local VI History CI emits a report, ranked backlog, and next-step artifact | +| `TEST-VHLP-006` | Next-step escalation coverage | Assurance/Contract | High | Verifies local VI History CI emits a machine-readable escalation step to the shared Windows Docker Desktop + NI image surface only after native or reachable bridge-backed Windows checks cannot satisfy it | +| `TEST-VHLP-007` | Shared local-program selector coverage | Assurance/Contract | High | Verifies the shared program selector can choose VI History requirement work explicitly and merge the shared Windows Docker Desktop + NI image escalation across sibling packets | +| `TEST-VHLP-008` | Clone-backed live-history candidate governance coverage | Assurance/Contract | High | Verifies the packet names `ni/labview-icon-editor` plus `Tooling/deployment/VIP_Pre-Uninstall Custom Action.vi` as the governed clone-backed live-history candidate | +| `TEST-VHLP-009` | Live-history candidate readiness coverage | Assurance/Contract | High | Verifies local VI History CI validates clone presence, target path presence, and git history, then emits a bounded clone-preparation escalation when the candidate is unavailable | +| `TEST-VHLP-010` | Explicit Windows replay next-step coverage | Assurance/Contract | High | Verifies local VI History CI emits `vi-history-windows-workflow-replay` as the next step when the shared Windows surface and live-history candidate are ready, instead of launching the replay lane during packet selection | +| `TEST-VHLP-011` | Bounded Windows replay lifecycle coverage | Contract/Replay | High | Verifies the governed Windows workflow replay lane terminates or fails closed within bounded helper-process timeouts and still emits a replay receipt on timeout | +| `TEST-VHLP-012` | Replay receipt consumption coverage | Assurance/Contract | High | Verifies local VI History CI treats an existing passing `vi-history-scenarios-windows` replay receipt as satisfied local replay proof and advances beyond replay re-selection | ## Entry Criteria @@ -33,7 +33,8 @@ - Contract tests and PowerShell tests covering the packet pass. - The local VI History CI emits a machine-readable next step. - The clone-backed live-history candidate is governed explicitly. -- If the current host cannot satisfy the Windows replay lane, the next step is an explicit escalation packet rather than a prose-only advisory. +- If the current host cannot satisfy the Windows replay lane, the next step is + an explicit escalation packet rather than a prose-only advisory. ## Traceability Notes diff --git a/docs/testing/windows-docker-shared-surface-test-plan.md b/docs/testing/windows-docker-shared-surface-test-plan.md index acc9a2edb..6e7b7dc97 100644 --- a/docs/testing/windows-docker-shared-surface-test-plan.md +++ b/docs/testing/windows-docker-shared-surface-test-plan.md @@ -10,16 +10,16 @@ | Test ID | Coverage | Layer | Priority | Notes | | --- | --- | --- | --- | --- | -| `TEST-WDSS-001` shared-surface readiness probe coverage | Probe/Receipt | High | Verifies the shared Windows readiness probe emits bounded readiness states and a machine-readable receipt | -| `TEST-WDSS-002` bootstrap and preflight contract coverage | Bootstrap/Host | High | Verifies the deterministic Windows host bootstrap/preflight commands remain explicit and stable | -| `TEST-WDSS-003` path-hygiene coverage | Safety/Local | High | Verifies the shared surface detects OneDrive-like managed roots and emits a relocation escalation | -| `TEST-WDSS-004` local assurance-loop coverage | Assurance/Contract | High | Verifies the shared-surface local CI emits a report, proof checks, and next-step artifact | -| `TEST-WDSS-005` host-unavailable escalation coverage | Assurance/Contract | High | Verifies local CI emits a machine-readable escalation to `windows-docker-desktop-ni-image` when the current host cannot satisfy the surface | -| `TEST-WDSS-006` shared local-program selector coverage | Assurance/Contract | High | Verifies the shared surface participates explicitly in the program selector beside Pester and VI History | -| `TEST-WDSS-007` reachable Windows host bridge coverage | Assurance/Contract | High | Verifies a Unix or WSL coordinator uses a reachable Windows host bridge for probe and preflight work before emitting host-unavailable escalation | -| `TEST-WDSS-008` UNC-backed WSL staging coverage | Runtime/Windows Docker | High | Verifies UNC-backed WSL inputs and report paths are staged into a Windows-local mount root, synchronized back, and cleaned up after compare execution | -| `TEST-WDSS-009` authoritative CI gate coverage | Workflow/Contract | High | Verifies Windows image-backed CI gates route through the shared Windows NI proof workflow and not through the generic Pester reusable workflow | -| `TEST-WDSS-010` bounded timeout coverage | Runtime/Workflow | High | Verifies Windows preflight and runtime-manager Docker operations fail closed on timeout and the hosted preflight step carries an explicit workflow timeout | +| `TEST-WDSS-001` | Shared-surface readiness probe coverage | Probe/Receipt | High | Verifies the shared Windows readiness probe emits bounded readiness states and a machine-readable receipt | +| `TEST-WDSS-002` | Bootstrap and preflight contract coverage | Bootstrap/Host | High | Verifies the deterministic Windows host bootstrap/preflight commands remain explicit and stable | +| `TEST-WDSS-003` | Path-hygiene coverage | Safety/Local | High | Verifies the shared surface detects OneDrive-like managed roots and emits a relocation escalation | +| `TEST-WDSS-004` | Local assurance-loop coverage | Assurance/Contract | High | Verifies the shared-surface local CI emits a report, proof checks, and next-step artifact | +| `TEST-WDSS-005` | Host-unavailable escalation coverage | Assurance/Contract | High | Verifies local CI emits a machine-readable escalation to `windows-docker-desktop-ni-image` when the current host cannot satisfy the surface | +| `TEST-WDSS-006` | Shared local-program selector coverage | Assurance/Contract | High | Verifies the shared surface participates explicitly in the program selector beside Pester and VI History | +| `TEST-WDSS-007` | Reachable Windows host bridge coverage | Assurance/Contract | High | Verifies a Unix or WSL coordinator uses a reachable Windows host bridge for probe and preflight work before emitting host-unavailable escalation | +| `TEST-WDSS-008` | UNC-backed WSL staging coverage | Runtime/Windows Docker | High | Verifies UNC-backed WSL inputs and report paths are staged into a Windows-local mount root, synchronized back, and cleaned up after compare execution | +| `TEST-WDSS-009` | Authoritative CI gate coverage | Workflow/Contract | High | Verifies Windows image-backed CI gates route through the shared Windows NI proof workflow and not through the generic Pester reusable workflow | +| `TEST-WDSS-010` | Bounded timeout coverage | Runtime/Workflow | High | Verifies Windows preflight and runtime-manager Docker operations fail closed on timeout and the hosted preflight step carries an explicit workflow timeout | ## Entry Criteria @@ -30,7 +30,8 @@ - Contract and local-CI tests covering the packet pass. - The shared-surface local CI emits a machine-readable next step. -- On a non-Windows host with no reachable Windows bridge, the next step is an explicit escalation packet rather than prose-only guidance. +- On a non-Windows host with no reachable Windows bridge, the next step is an + explicit escalation packet rather than prose-only guidance. ## Traceability Notes diff --git a/tests/CompareVI.GitRefs.VI2.Tests.ps1 b/tests/CompareVI.GitRefs.VI2.Tests.ps1 index ed7e687a6..14b6e1e2a 100644 --- a/tests/CompareVI.GitRefs.VI2.Tests.ps1 +++ b/tests/CompareVI.GitRefs.VI2.Tests.ps1 @@ -2,24 +2,36 @@ Describe 'CompareVI with Git refs (VI2.vi at two commits)' -Tag 'CompareVI','Integration' { BeforeAll { $ErrorActionPreference = 'Stop' + $script:_skipCompareVIGitRefsReason = $null try { git --version | Out-Null } catch { throw 'git is required for this test' } $repoRoot = (Get-Location).Path $target = 'VI2.vi' if (-not (Test-Path -LiteralPath (Join-Path $repoRoot $target))) { - Set-ItResult -Skipped -Because "Target file not found: $target" + $script:_skipCompareVIGitRefsReason = "Target file not found: $target" + return } $revList = & git rev-list --max-count=50 HEAD -- $target - if (-not $revList) { Set-ItResult -Skipped -Because 'No history for target'; return } + if (-not $revList) { + $script:_skipCompareVIGitRefsReason = 'No history for target' + return + } $pairs = @() foreach ($a in $revList) { foreach ($b in $revList) { if ($a -ne $b) { $pairs += [pscustomobject]@{ A=$a; B=$b } } } } - if (-not $pairs) { Set-ItResult -Skipped -Because 'Not enough refs' } + if (-not $pairs) { + $script:_skipCompareVIGitRefsReason = 'Not enough refs' + return + } Set-Variable -Name '_repo' -Value $repoRoot -Scope Script Set-Variable -Name '_pairs' -Value $pairs -Scope Script Set-Variable -Name '_target' -Value $target -Scope Script } It 'produces exec and summary JSON from two refs (non-failing check)' { + if ($script:_skipCompareVIGitRefsReason) { + Set-ItResult -Skipped -Because $script:_skipCompareVIGitRefsReason + return + } $pair = $null foreach ($p in $_pairs) { & git show --no-renames -- "$($p.A):$_target" 1>$null 2>$null; $okA = ($LASTEXITCODE -eq 0) @@ -56,4 +68,51 @@ Describe 'CompareVI with Git refs (VI2.vi at two commits)' -Tag 'CompareVI','Int "VI2 refs: A=$($pair.A) B=$($pair.B) expectDiff=$($s.computed.expectDiff) cliDiff=$($s.cli.diff) exit=$($s.cli.exitCode)" | Write-Host } + + It 'strips provider-qualified results paths from emitted ref-compare artifacts' { + if ($script:_skipCompareVIGitRefsReason) { + Set-ItResult -Skipped -Because $script:_skipCompareVIGitRefsReason + return + } + $pair = $null + foreach ($p in $_pairs) { + & git show --no-renames -- "$($p.A):$_target" 1>$null 2>$null; $okA = ($LASTEXITCODE -eq 0) + & git show --no-renames -- "$($p.B):$_target" 1>$null 2>$null; $okB = ($LASTEXITCODE -eq 0) + if ($okA -and $okB) { $pair = $p; break } + } + if (-not $pair) { Set-ItResult -Skipped -Because 'No valid ref pair with content'; return } + + $rd = Join-Path $TestDrive 'ref-compare-vi2-provider' + New-Item -ItemType Directory -Path $rd -Force | Out-Null + $providerResultsDir = "Microsoft.PowerShell.Core\FileSystem::$rd" + $stubPath = Join-Path $_repo 'tests/stubs/Invoke-LVCompare.stub.ps1' + + & pwsh -NoLogo -NoProfile -File (Join-Path $_repo 'tools/Compare-RefsToTemp.ps1') ` + -Path $_target ` + -RefA $pair.A ` + -RefB $pair.B ` + -ResultsDir $providerResultsDir ` + -OutName 'vi2-provider' ` + -Detailed ` + -RenderReport ` + -InvokeScriptPath $stubPath ` + -FailOnDiff:$false | Out-Null + + $sum = Join-Path $rd 'vi2-provider-summary.json' + $sum | Should -Exist + $summary = Get-Content -LiteralPath $sum -Raw | ConvertFrom-Json -Depth 10 + foreach ($pathValue in @( + [string]$summary.out.execJson, + [string]$summary.out.captureJson, + [string]$summary.out.stdout, + [string]$summary.out.stderr, + [string]$summary.out.reportHtml, + [string]$summary.out.reportPath, + [string]$summary.out.artifactDir + )) { + if (-not [string]::IsNullOrWhiteSpace($pathValue)) { + $pathValue | Should -Not -Match 'Microsoft\.PowerShell\.Core\\FileSystem::' + } + } + } } diff --git a/tests/TestFileExistsAtRef.Tests.ps1 b/tests/TestFileExistsAtRef.Tests.ps1 index e7d99ce7f..cedea9875 100644 --- a/tests/TestFileExistsAtRef.Tests.ps1 +++ b/tests/TestFileExistsAtRef.Tests.ps1 @@ -235,4 +235,204 @@ exit 0 $manifest.stats.missing | Should -Be 0 $manifest.comparisons | Should -Not -BeNullOrEmpty } + + It 'emits native history paths when results dir is provider-qualified' { + $repoRoot = Split-Path -Parent $PSScriptRoot + $compareHistoryScript = Join-Path $repoRoot 'tools' 'Compare-VIHistory.ps1' + + $tempRepo = Join-Path $TestDrive 'crossrepo-history-provider' + New-Item -ItemType Directory -Path $tempRepo -Force | Out-Null + + Push-Location $tempRepo + try { + & git init | Out-Null + & git config user.name 'CompareVI Tests' | Out-Null + & git config user.email 'comparevi.tests@example.com' | Out-Null + + $targetRelPath = 'Tooling/deployment/VIP_Post-Install Custom Action.vi' + New-Item -ItemType Directory -Path (Split-Path -Parent $targetRelPath) -Force | Out-Null + + 'base version' | Set-Content -LiteralPath $targetRelPath -Encoding utf8 + & git add . | Out-Null + & git commit -m 'feat: add VIP post-install action' | Out-Null + + 'updated version' | Set-Content -LiteralPath $targetRelPath -Encoding utf8 + & git add . | Out-Null + & git commit -m 'fix: adjust VIP post-install action' | Out-Null + $headCommit = (& git rev-parse HEAD).Trim() + } + finally { + Pop-Location + } + + $stubPath = Join-Path $TestDrive 'Invoke-LVCompare.crossrepo.provider.stub.ps1' + @' +param( + [Parameter(Mandatory = $true)][string]$BaseVi, + [Parameter(Mandatory = $true)][string]$HeadVi, + [string]$OutputDir, + [string]$LabVIEWExePath, + [string]$LabVIEWBitness = "64", + [string]$LVComparePath, + [string[]]$Flags, + [switch]$ReplaceFlags, + [switch]$AllowSameLeaf, + [switch]$RenderReport, + [ValidateSet("html","xml","text")][string[]]$ReportFormat = "html", + [string]$JsonLogPath, + [switch]$Quiet, + [switch]$LeakCheck, + [double]$LeakGraceSeconds = 0, + [string]$LeakJsonPath, + [string]$CaptureScriptPath, + [switch]$Summary, + [Nullable[int]]$TimeoutSeconds, + [Parameter(ValueFromRemainingArguments = $true)][string[]]$PassThru +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +if (-not $OutputDir) { + $OutputDir = Join-Path $env:TEMP ("history-stub-" + [guid]::NewGuid().ToString("N")) +} +New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null + +$reportPath = Join-Path $OutputDir 'compare-report.html' +$capturePath = Join-Path $OutputDir 'lvcompare-capture.json' +$stdoutPath = Join-Path $OutputDir 'lvcompare-stdout.txt' +$stderrPath = Join-Path $OutputDir 'lvcompare-stderr.txt' + +"

    Stub Compare Report

    " | Set-Content -LiteralPath $reportPath -Encoding utf8 +"" | Set-Content -LiteralPath $stdoutPath -Encoding utf8 +"" | Set-Content -LiteralPath $stderrPath -Encoding utf8 + +[ordered]@{ + schema = 'lvcompare-capture-v1' + timestamp = (Get-Date).ToString('o') + base = $BaseVi + head = $HeadVi + cliPath = 'stub' + args = $Flags + exitCode = 0 + seconds = 0.05 + command = 'stub' +} | ConvertTo-Json -Depth 4 | Set-Content -LiteralPath $capturePath -Encoding utf8 + +$parentDir = Split-Path -Parent $OutputDir +$outLeaf = Split-Path -Leaf $OutputDir +if ($outLeaf -like '*-artifacts') { + $outName = $outLeaf.Substring(0, $outLeaf.Length - '-artifacts'.Length) +} else { + $outName = $outLeaf +} +$summaryPath = Join-Path $parentDir ("{0}-summary.json" -f $outName) +$execPath = Join-Path $parentDir ("{0}-exec.json" -f $outName) + +[ordered]@{ + schema = 'ref-compare-summary/v1' + generatedAt = (Get-Date).ToString('o') + name = Split-Path -Leaf $HeadVi + path = $HeadVi + refA = $BaseVi + refB = $HeadVi + temp = $OutputDir + reportFormat= 'html' + out = [pscustomobject]@{ + captureJson = $capturePath + reportPath = $reportPath + stdout = $stdoutPath + stderr = $stderrPath + } + computed = [ordered]@{ + baseBytes = 0 + headBytes = 0 + baseSha = 'stub-base' + headSha = 'stub-head' + expectDiff = $false + } + cli = [pscustomobject]@{ + exitCode = 0 + diff = $false + duration_s = 0.05 + command = 'stub' + cliPath = 'stub' + } +} | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $summaryPath -Encoding utf8 + +[ordered]@{ + schema = 'compare-exec/v1' + generatedAt = (Get-Date).ToString('o') + cliPath = 'stub' + command = 'stub' + args = @() + exitCode = 0 + diff = $false + cwd = (Get-Location).Path + duration_s = 0.05 + base = $BaseVi + head = $HeadVi +} | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $execPath -Encoding utf8 + +exit 0 +'@ | Set-Content -LiteralPath $stubPath -Encoding utf8 + + $resultsDir = Join-Path $TestDrive 'crossrepo-history-provider-results' + New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null + $providerResultsDir = "Microsoft.PowerShell.Core\FileSystem::$resultsDir" + $previousScriptsRoot = $env:COMPAREVI_SCRIPTS_ROOT + $env:COMPAREVI_SCRIPTS_ROOT = $repoRoot + try { + Push-Location $tempRepo + try { + $args = @( + '-NoLogo','-NoProfile','-File', $compareHistoryScript, + '-TargetPath', 'Tooling/deployment/VIP_Post-Install Custom Action.vi', + '-StartRef', $headCommit, + '-MaxPairs', '1', + '-RenderReport', + '-FailOnDiff:$false', + '-ResultsDir', $providerResultsDir, + '-InvokeScriptPath', $stubPath, + '-Quiet' + ) + & pwsh @args | Out-Null + } + finally { + Pop-Location + } + } + finally { + if ($null -ne $previousScriptsRoot) { + $env:COMPAREVI_SCRIPTS_ROOT = $previousScriptsRoot + } else { + Remove-Item Env:COMPAREVI_SCRIPTS_ROOT -ErrorAction SilentlyContinue + } + } + + $aggregateManifestPath = Join-Path $resultsDir 'manifest.json' + $modeManifestPath = Join-Path $resultsDir 'default' 'manifest.json' + $historySummaryPath = Join-Path $resultsDir 'history-summary.json' + + $aggregateManifestPath | Should -Exist + $modeManifestPath | Should -Exist + $historySummaryPath | Should -Exist + + $aggregateManifest = Get-Content -LiteralPath $aggregateManifestPath -Raw | ConvertFrom-Json -Depth 8 + $modeManifest = Get-Content -LiteralPath $modeManifestPath -Raw | ConvertFrom-Json -Depth 8 + $historySummary = Get-Content -LiteralPath $historySummaryPath -Raw | ConvertFrom-Json -Depth 8 + + foreach ($pathValue in @( + [string]$aggregateManifest.modes[0].manifestPath, + [string]$aggregateManifest.modes[0].resultsDir, + [string]$modeManifest.resultsDir, + [string]$historySummary.execution.resultsDir, + [string]$historySummary.execution.manifestPath, + [string]$historySummary.reports.markdownPath, + [string]$historySummary.reports.htmlPath + )) { + if (-not [string]::IsNullOrWhiteSpace($pathValue)) { + $pathValue | Should -Not -Match 'Microsoft\.PowerShell\.Core\\FileSystem::' + } + } + } } diff --git a/tools/Compare-RefsToTemp.ps1 b/tools/Compare-RefsToTemp.ps1 index 8fa85531c..be2e876e7 100644 --- a/tools/Compare-RefsToTemp.ps1 +++ b/tools/Compare-RefsToTemp.ps1 @@ -29,6 +29,36 @@ try { git --version | Out-Null } catch { throw 'git is required on PATH to fetch $repoRoot = (Get-Location).Path +function Convert-ToNativeFileSystemPath { + param([AllowNull()][string]$PathValue) + if ([string]::IsNullOrWhiteSpace($PathValue)) { return $PathValue } + + $candidate = [string]$PathValue + $lastProviderSeparator = $candidate.LastIndexOf('::', [System.StringComparison]::Ordinal) + if ($lastProviderSeparator -ge 0) { + $candidate = $candidate.Substring($lastProviderSeparator + 2) + } + $candidate = ($candidate -replace '^[A-Za-z][A-Za-z0-9.+-]*::', '') + if ($candidate -match '^[\\/](wsl\.localhost|wsl\$)[\\/]') { + $candidate = [System.IO.Path]::DirectorySeparatorChar + $candidate + } + try { + $resolved = Resolve-Path -LiteralPath $candidate -ErrorAction Stop | Select-Object -First 1 + $providerPath = [string]$resolved.ProviderPath + if (-not [string]::IsNullOrWhiteSpace($providerPath)) { + return [System.IO.Path]::GetFullPath($providerPath) + } + } catch {} + + try { + return [System.IO.Path]::GetFullPath($candidate) + } catch { + return $candidate + } +} + +$repoRoot = Convert-ToNativeFileSystemPath -PathValue $repoRoot + function Resolve-CompareVIScriptsRoot { param([string]$PrimaryRoot) @@ -80,7 +110,13 @@ function Split-ArgString { function Normalize-ExistingPath { param([string]$Candidate) if ([string]::IsNullOrWhiteSpace($Candidate)) { return $null } - try { return (Resolve-Path -LiteralPath $Candidate -ErrorAction Stop).Path } catch { return $Candidate } + try { return Convert-ToNativeFileSystemPath -PathValue $Candidate } catch { return $Candidate } +} + +function Resolve-NativeExistingPath { + param([string]$Candidate) + if ([string]::IsNullOrWhiteSpace($Candidate)) { return $null } + try { return Convert-ToNativeFileSystemPath -PathValue $Candidate } catch { return $Candidate } } function Resolve-TempRoot { @@ -636,6 +672,7 @@ Get-FileAtRef -ref $RefA -relPath $Path -dest $base Get-FileAtRef -ref $RefB -relPath $Path -dest $head $rd = if ([System.IO.Path]::IsPathRooted($ResultsDir)) { $ResultsDir } else { Join-Path $repoRoot $ResultsDir } +$rd = Convert-ToNativeFileSystemPath -PathValue $rd New-Item -ItemType Directory -Path $rd -Force | Out-Null $execPath = Join-Path $rd ("$OutName-exec.json") $sumPath = Join-Path $rd ("$OutName-summary.json") @@ -838,7 +875,7 @@ if ($detailRequested) { if (-not $candidatePath) { continue } if (Test-Path -LiteralPath $candidatePath -PathType Leaf) { try { - $leakResolvedPath = (Resolve-Path -LiteralPath $candidatePath).Path + $leakResolvedPath = Resolve-NativeExistingPath -Candidate $candidatePath } catch { $leakResolvedPath = $candidatePath } @@ -875,18 +912,18 @@ if ($detailRequested) { } } - if (Test-Path -LiteralPath $capturePath) { $detailPaths.captureJson = (Resolve-Path -LiteralPath $capturePath).Path } - if (Test-Path -LiteralPath $stdoutPath) { $detailPaths.stdout = (Resolve-Path -LiteralPath $stdoutPath).Path } - if (Test-Path -LiteralPath $stderrPath) { $detailPaths.stderr = (Resolve-Path -LiteralPath $stderrPath).Path } + if (Test-Path -LiteralPath $capturePath) { $detailPaths.captureJson = Resolve-NativeExistingPath -Candidate $capturePath } + if (Test-Path -LiteralPath $stdoutPath) { $detailPaths.stdout = Resolve-NativeExistingPath -Candidate $stdoutPath } + if (Test-Path -LiteralPath $stderrPath) { $detailPaths.stderr = Resolve-NativeExistingPath -Candidate $stderrPath } $reportResolved = $null if ($reportFile -and (Test-Path -LiteralPath $reportFile)) { - $reportResolved = (Resolve-Path -LiteralPath $reportFile).Path + $reportResolved = Resolve-NativeExistingPath -Candidate $reportFile $detailPaths.reportPath = $reportResolved if ($reportFormatEffective -eq 'html') { $detailPaths.reportHtml = $reportResolved } } - if (Test-Path -LiteralPath $imagesDir) { $detailPaths.imagesDir = (Resolve-Path -LiteralPath $imagesDir).Path } + if (Test-Path -LiteralPath $imagesDir) { $detailPaths.imagesDir = Resolve-NativeExistingPath -Candidate $imagesDir } if ($reportFormatEffective -eq 'html' -and $reportResolved) { $includedAttributes = Get-IncludedAttributesFromReport -ReportPath $reportResolved $reportMetadata = Get-ReportCategoryMetadata -ReportPath $reportResolved @@ -933,14 +970,14 @@ if (-not $cliDiff -and $cliExit -eq $null) { $cliExit = 0 } $exec = Get-Content -LiteralPath $execPath -Raw | ConvertFrom-Json -Depth 6 -$outPaths = [ordered]@{ execJson = (Resolve-Path -LiteralPath $execPath).Path } +$outPaths = [ordered]@{ execJson = (Resolve-NativeExistingPath -Candidate $execPath) } foreach ($k in @('captureJson','stdout','stderr','reportHtml','reportPath','imagesDir')) { if ($detailPaths.Contains($k) -and $detailPaths[$k]) { $outPaths[$k] = $detailPaths[$k] } } if ($detailPaths.Contains('leakJson') -and $detailPaths['leakJson']) { $outPaths.leakJson = $detailPaths['leakJson'] } -if ($artifactDir) { $outPaths.artifactDir = (Resolve-Path -LiteralPath $artifactDir).Path } +if ($artifactDir) { $outPaths.artifactDir = Resolve-NativeExistingPath -Candidate $artifactDir } $cliSummary = [ordered]@{ exitCode = $cliExit diff --git a/tools/Compare-VIHistory.ps1 b/tools/Compare-VIHistory.ps1 index 694f9fe89..c0147eba3 100644 --- a/tools/Compare-VIHistory.ps1 +++ b/tools/Compare-VIHistory.ps1 @@ -722,6 +722,47 @@ function Invoke-ProcessCapture { } } +function Convert-ToNativeFileSystemPath { + param([AllowNull()][string]$PathValue) + + if ([string]::IsNullOrWhiteSpace($PathValue)) { + return $PathValue + } + + $candidate = [string]$PathValue + $lastProviderSeparator = $candidate.LastIndexOf('::', [System.StringComparison]::Ordinal) + if ($lastProviderSeparator -ge 0) { + $candidate = $candidate.Substring($lastProviderSeparator + 2) + } + $candidate = ($candidate -replace '^[A-Za-z][A-Za-z0-9.+-]*::', '') + if ($candidate -match '^[\\/](wsl\.localhost|wsl\$)[\\/]') { + $candidate = [System.IO.Path]::DirectorySeparatorChar + $candidate + } + try { + $resolved = Resolve-Path -LiteralPath $candidate -ErrorAction Stop | Select-Object -First 1 + $providerPath = [string]$resolved.ProviderPath + if (-not [string]::IsNullOrWhiteSpace($providerPath)) { + return [System.IO.Path]::GetFullPath($providerPath) + } + } catch {} + + try { + return [System.IO.Path]::GetFullPath($candidate) + } catch { + return $candidate + } +} + +function Resolve-NativeExistingPath { + param([AllowNull()][string]$PathValue) + + if ([string]::IsNullOrWhiteSpace($PathValue)) { + return $PathValue + } + + return Convert-ToNativeFileSystemPath -PathValue $PathValue +} + function Invoke-Git { param( [Parameter(Mandatory = $true)][string[]]$Arguments, @@ -1448,9 +1489,7 @@ if ($labVIEWIniPath) { } $repoRoot = Resolve-RepoRoot -try { - $repoRoot = [System.IO.Path]::GetFullPath($repoRoot) -} catch {} +$repoRoot = Convert-ToNativeFileSystemPath -PathValue $repoRoot if ([string]::IsNullOrWhiteSpace($TargetPath)) { throw 'TargetPath cannot be empty.' @@ -1461,7 +1500,7 @@ try { if (-not [System.IO.Path]::IsPathRooted($targetFullPath)) { $targetFullPath = Join-Path $repoRoot $targetFullPath } - $targetFullPath = [System.IO.Path]::GetFullPath($targetFullPath) + $targetFullPath = Convert-ToNativeFileSystemPath -PathValue $targetFullPath } catch { throw ("Unable to resolve TargetPath '{0}': {1}" -f $TargetPath, $_.Exception.Message) } @@ -1470,25 +1509,40 @@ if (-not (Test-Path -LiteralPath $targetFullPath -PathType Leaf)) { Write-Verbose ("TargetPath '{0}' not found on disk; continuing with git history refs." -f $targetFullPath) } -$targetRel = $targetFullPath -try { - if ($repoRoot) { - $rootNormalized = [System.IO.Path]::GetFullPath($repoRoot) - $rootPrefix = $rootNormalized.TrimEnd('\','/') - if ($rootPrefix.Length -gt 0) { - $rootPrefix = $rootPrefix + [System.IO.Path]::DirectorySeparatorChar - } - if ($targetFullPath.StartsWith($rootPrefix, [System.StringComparison]::OrdinalIgnoreCase)) { - $targetRel = $targetFullPath.Substring($rootPrefix.Length).TrimStart('\','/') +$targetRel = [string]$TargetPath +$targetRelProviderStripped = $targetRel -replace '^[A-Za-z][A-Za-z0-9.+-]*::', '' +$preferOriginalRelativeTargetPath = ( + -not [string]::IsNullOrWhiteSpace($targetRelProviderStripped) -and + -not [System.IO.Path]::IsPathRooted($targetRelProviderStripped) -and + -not $targetRelProviderStripped.StartsWith('\\') -and + -not $targetRelProviderStripped.StartsWith('//') +) + +if ($preferOriginalRelativeTargetPath) { + $targetRel = $targetRelProviderStripped +} else { + $targetRel = $targetFullPath + try { + if ($repoRoot) { + $rootNormalized = [System.IO.Path]::GetFullPath($repoRoot) + $rootPrefix = $rootNormalized.TrimEnd('\','/') + if ($rootPrefix.Length -gt 0) { + $rootPrefix = $rootPrefix + [System.IO.Path]::DirectorySeparatorChar + } + if ($targetFullPath.StartsWith($rootPrefix, [System.StringComparison]::OrdinalIgnoreCase)) { + $targetRel = $targetFullPath.Substring($rootPrefix.Length).TrimStart('\','/') + } } - } -} catch {} + } catch {} +} + +$targetRel = ($targetRel -replace '^[A-Za-z][A-Za-z0-9.+-]*::', '') $targetRel = ($targetRel -replace '\\','/').Trim('/') if ([string]::IsNullOrWhiteSpace($targetRel)) { throw ("TargetPath '{0}' could not be normalized relative to repository root '{1}'." -f $TargetPath, $repoRoot) } Write-Verbose ("Normalized target path: {0}" -f $targetRel) -$targetLeaf = Split-Path $targetRel -Leaf +$targetLeaf = @($targetRel -split '/+' | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | Select-Object -Last 1) if ([string]::IsNullOrWhiteSpace($targetLeaf)) { $targetLeaf = 'vi' } $startRef = if ([string]::IsNullOrWhiteSpace($StartRef)) { 'HEAD' } else { $StartRef.Trim() } @@ -1544,8 +1598,9 @@ if ($resolvedStartRef -ne $startRef) { } $resultsRoot = if ([System.IO.Path]::IsPathRooted($ResultsDir)) { $ResultsDir } else { Join-Path $repoRoot $ResultsDir } +$resultsRoot = Convert-ToNativeFileSystemPath -PathValue $resultsRoot New-Item -ItemType Directory -Path $resultsRoot -Force | Out-Null -$resultsRootResolved = (Resolve-Path -LiteralPath $resultsRoot).Path +$resultsRootResolved = Resolve-NativeExistingPath -PathValue $resultsRoot $aggregateManifestPath = if ($ManifestPath) { if ([System.IO.Path]::IsPathRooted($ManifestPath)) { $ManifestPath } else { Join-Path $repoRoot $ManifestPath } @@ -1807,8 +1862,9 @@ foreach ($modeSpec in $modeSpecs) { Write-Verbose ("Mode {0} flags: {1}" -f $modeName, $mf) $modeResultsRoot = Join-Path $resultsRoot $modeSlug + $modeResultsRoot = Convert-ToNativeFileSystemPath -PathValue $modeResultsRoot New-Item -ItemType Directory -Path $modeResultsRoot -Force | Out-Null - $modeResultsResolved = (Resolve-Path -LiteralPath $modeResultsRoot).Path + $modeResultsResolved = Resolve-NativeExistingPath -PathValue $modeResultsRoot $modeResultsRelative = $null if ($modeResultsResolved) { try { @@ -2076,7 +2132,7 @@ foreach ($modeSpec in $modeSpecs) { if (-not (Test-Path -LiteralPath $summaryPath -PathType Leaf)) { Write-Warning ("Summary not found at {0}; generating fallback summary from exec data." -f $summaryPath) $fallbackOut = [ordered]@{ - execJson = if (Test-Path -LiteralPath $execPath -PathType Leaf) { (Resolve-Path -LiteralPath $execPath).Path } else { $null } + execJson = if (Test-Path -LiteralPath $execPath -PathType Leaf) { (Resolve-NativeExistingPath -PathValue $execPath) } else { $null } } $fallbackCli = [ordered]@{ exitCode = $null @@ -2127,8 +2183,8 @@ foreach ($modeSpec in $modeSpecs) { $diff = [bool]$summaryJson.cli.diff $comparisonRecord.result = [ordered]@{ - summaryPath = (Resolve-Path -LiteralPath $summaryPath).Path - execPath = if (Test-Path -LiteralPath $execPath) { (Resolve-Path -LiteralPath $execPath).Path } else { $null } + summaryPath = Resolve-NativeExistingPath -PathValue $summaryPath + execPath = if (Test-Path -LiteralPath $execPath) { (Resolve-NativeExistingPath -PathValue $execPath) } else { $null } diff = $diff exitCode = $summaryJson.cli.exitCode duration_s = $summaryJson.cli.duration_s @@ -2302,7 +2358,7 @@ foreach ($modeSpec in $modeSpecs) { Remove-Item -LiteralPath $artifactDir -Recurse -Force -ErrorAction SilentlyContinue } } elseif (Test-Path -LiteralPath $artifactDir) { - $comparisonRecord.result.artifactDir = (Resolve-Path -LiteralPath $artifactDir).Path + $comparisonRecord.result.artifactDir = Resolve-NativeExistingPath -PathValue $artifactDir } } $categoryDetails = $null @@ -2450,7 +2506,7 @@ foreach ($modeSpec in $modeSpecs) { $collapsedNoiseStats.count = $noiseCollapsedCount $modeManifest | ConvertTo-Json -Depth 8 | Out-File -FilePath $modeManifestPath -Encoding utf8 - $modeManifestResolved = (Resolve-Path -LiteralPath $modeManifestPath).Path + $modeManifestResolved = Resolve-NativeExistingPath -PathValue $modeManifestPath if (-not $hasStepSummary) { $summaryLines += '' @@ -2724,7 +2780,7 @@ if ($executedModeNames.Count -gt 0) { } $aggregate | ConvertTo-Json -Depth 8 | Out-File -FilePath $aggregateManifestPath -Encoding utf8 -$aggregateManifestResolved = (Resolve-Path -LiteralPath $aggregateManifestPath).Path +$aggregateManifestResolved = Resolve-NativeExistingPath -PathValue $aggregateManifestPath Write-GitHubOutput -Key 'manifest-path' -Value $aggregateManifestResolved -DestPath $GitHubOutputPath Write-GitHubOutput -Key 'results-dir' -Value $resultsRootResolved -DestPath $GitHubOutputPath Write-GitHubOutput -Key 'mode-count' -Value $aggregate.modes.Count -DestPath $GitHubOutputPath @@ -2886,11 +2942,11 @@ if (-not $renderSucceeded) { } if (Test-Path -LiteralPath $historyReportMarkdownPath -PathType Leaf) { - $historyReportMarkdownResolved = (Resolve-Path -LiteralPath $historyReportMarkdownPath).Path + $historyReportMarkdownResolved = Resolve-NativeExistingPath -PathValue $historyReportMarkdownPath Write-GitHubOutput -Key 'history-report-md' -Value $historyReportMarkdownResolved -DestPath $GitHubOutputPath } if ($historyReportHtmlPath -and (Test-Path -LiteralPath $historyReportHtmlPath -PathType Leaf)) { - $historyReportHtmlResolved = (Resolve-Path -LiteralPath $historyReportHtmlPath).Path + $historyReportHtmlResolved = Resolve-NativeExistingPath -PathValue $historyReportHtmlPath Write-GitHubOutput -Key 'history-report-html' -Value $historyReportHtmlResolved -DestPath $GitHubOutputPath } diff --git a/tools/Render-VIHistoryReport.ps1 b/tools/Render-VIHistoryReport.ps1 index 8dcacd2f9..3c7284a4f 100644 --- a/tools/Render-VIHistoryReport.ps1 +++ b/tools/Render-VIHistoryReport.ps1 @@ -22,6 +22,34 @@ try { } } catch {} +function Convert-ToNativeFileSystemPath { + param([AllowNull()][string]$PathValue) + if ([string]::IsNullOrWhiteSpace($PathValue)) { return $PathValue } + + $candidate = [string]$PathValue + $lastProviderSeparator = $candidate.LastIndexOf('::', [System.StringComparison]::Ordinal) + if ($lastProviderSeparator -ge 0) { + $candidate = $candidate.Substring($lastProviderSeparator + 2) + } + $candidate = ($candidate -replace '^[A-Za-z][A-Za-z0-9.+-]*::', '') + if ($candidate -match '^[\\/](wsl\.localhost|wsl\$)[\\/]') { + $candidate = [System.IO.Path]::DirectorySeparatorChar + $candidate + } + try { + $resolved = Resolve-Path -LiteralPath $candidate -ErrorAction Stop | Select-Object -First 1 + $providerPath = [string]$resolved.ProviderPath + if (-not [string]::IsNullOrWhiteSpace($providerPath)) { + return [System.IO.Path]::GetFullPath($providerPath) + } + } catch {} + + try { + return [System.IO.Path]::GetFullPath($candidate) + } catch { + return $candidate + } +} + function Resolve-ExistingPath { param( [string]$Path, @@ -36,7 +64,7 @@ function Resolve-ExistingPath { if ($Optional.IsPresent) { return $null } throw ("{0} file not found: {1}" -f $Description, $Path) } - return (Resolve-Path -LiteralPath $Path).Path + return Convert-ToNativeFileSystemPath -PathValue $Path } function Ensure-Directory { @@ -45,14 +73,14 @@ function Ensure-Directory { if (-not (Test-Path -LiteralPath $Path -PathType Container)) { New-Item -ItemType Directory -Force -Path $Path | Out-Null } - return (Resolve-Path -LiteralPath $Path).Path + return Convert-ToNativeFileSystemPath -PathValue $Path } function Resolve-FullPath { param([string]$Path) if ([string]::IsNullOrWhiteSpace($Path)) { return $null } try { - return (Resolve-Path -LiteralPath $Path -ErrorAction Stop).Path + return Convert-ToNativeFileSystemPath -PathValue $Path } catch { if ([System.IO.Path]::IsPathRooted($Path)) { return [System.IO.Path]::GetFullPath($Path) @@ -1639,7 +1667,7 @@ if ($contextResolved) { $markdownContent = $summaryLines -join [Environment]::NewLine [System.IO.File]::WriteAllText($MarkdownPath, $markdownContent, [System.Text.Encoding]::UTF8) -$markdownOutPath = (Resolve-Path -LiteralPath $MarkdownPath).Path +$markdownOutPath = Convert-ToNativeFileSystemPath -PathValue $MarkdownPath $htmlOutPath = $null if ($emitHtml -and $HtmlPath) { @@ -2041,7 +2069,7 @@ if ($emitHtml -and $HtmlPath) { $htmlContent = $htmlBuilder.ToString() [System.IO.File]::WriteAllText($HtmlPath, $htmlContent, [System.Text.Encoding]::UTF8) - $htmlOutPath = (Resolve-Path -LiteralPath $HtmlPath).Path + $htmlOutPath = Convert-ToNativeFileSystemPath -PathValue $HtmlPath Write-GitHubOutput -Key 'history-report-html' -Value $htmlOutPath -DestPath $GitHubOutputPath } @@ -2162,7 +2190,7 @@ $historySummary = [ordered]@{ modes = @($modeFacadeEntries) } $historySummary | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $SummaryJsonPath -Encoding utf8 -$summaryJsonOutPath = (Resolve-Path -LiteralPath $SummaryJsonPath).Path +$summaryJsonOutPath = Convert-ToNativeFileSystemPath -PathValue $SummaryJsonPath Write-GitHubOutput -Key 'history-report-md' -Value $markdownOutPath -DestPath $GitHubOutputPath Write-GitHubOutput -Key 'history-summary-json' -Value $summaryJsonOutPath -DestPath $GitHubOutputPath From 953ccf14916d5672bf2730eb2d4fbb1f7a313c35 Mon Sep 17 00:00:00 2001 From: Sergio Velderrain Date: Wed, 1 Apr 2026 09:38:58 -0700 Subject: [PATCH 39/44] Fix Windows preflight process capture deadlock (#2091) Co-authored-by: svelderrainruiz --- tests/Invoke-DockerRuntimeManager.Tests.ps1 | 35 +++++ ...est-WindowsNI2026q1HostPreflight.Tests.ps1 | 56 +++++++ tools/Assert-DockerRuntimeDeterminism.ps1 | 44 ++---- tools/Invoke-DockerRuntimeManager.ps1 | 44 ++---- tools/ProcessTimeoutHelper.ps1 | 147 ++++++++++++++++++ tools/Test-WindowsNI2026q1HostPreflight.ps1 | 44 ++---- 6 files changed, 265 insertions(+), 105 deletions(-) create mode 100644 tools/ProcessTimeoutHelper.ps1 diff --git a/tests/Invoke-DockerRuntimeManager.Tests.ps1 b/tests/Invoke-DockerRuntimeManager.Tests.ps1 index 79753c7b9..f85067d63 100644 --- a/tests/Invoke-DockerRuntimeManager.Tests.ps1 +++ b/tests/Invoke-DockerRuntimeManager.Tests.ps1 @@ -109,8 +109,16 @@ if ($Args[0] -eq 'info') { } if ($Args[0] -eq 'manifest' -and $Args.Count -ge 3 -and $Args[1] -eq 'inspect') { + $paddingBytes = [Environment]::GetEnvironmentVariable('DOCKER_STUB_MANIFEST_PADDING_BYTES') + $padding = '' + if (-not [string]::IsNullOrWhiteSpace($paddingBytes)) { + $padding = ('x' * [int]$paddingBytes) + } $manifest = [ordered]@{ schemaVersion = 2 + annotations = [ordered]@{ + padding = $padding + } manifests = @( [ordered]@{ digest = 'sha256:1111111111111111111111111111111111111111111111111111111111111111' @@ -242,6 +250,7 @@ exit 0 DOCKER_STUB_RUN_FAIL_LINUX = $env:DOCKER_STUB_RUN_FAIL_LINUX DOCKER_STUB_INFO_SLEEP_SECONDS = $env:DOCKER_STUB_INFO_SLEEP_SECONDS DOCKER_STUB_INSPECT_SLEEP_SECONDS = $env:DOCKER_STUB_INSPECT_SLEEP_SECONDS + DOCKER_STUB_MANIFEST_PADDING_BYTES = $env:DOCKER_STUB_MANIFEST_PADDING_BYTES DOCKER_STUB_PULL_SLEEP_WINDOWS = $env:DOCKER_STUB_PULL_SLEEP_WINDOWS DOCKER_STUB_PULL_SLEEP_LINUX = $env:DOCKER_STUB_PULL_SLEEP_LINUX DOCKER_STUB_RUN_SLEEP_WINDOWS = $env:DOCKER_STUB_RUN_SLEEP_WINDOWS @@ -456,6 +465,32 @@ exit 0 $json.probes.windows.probe.status | Should -Be 'timeout' } + It 'handles large manifest output without deadlocking the timeout helper' { + $work = Join-Path $TestDrive 'large-manifest-output' + New-Item -ItemType Directory -Path $work -Force | Out-Null + & $script:CreateDockerStub -WorkRoot $work + + Set-Item Env:DOCKER_STUB_STATE_PATH (Join-Path $work 'docker-state.json') + Set-Item Env:DOCKER_STUB_INITIAL_CONTEXT 'desktop-windows' + Set-Item Env:DOCKER_STUB_MANIFEST_PADDING_BYTES '20000' + Set-Item Env:RUNNER_TEMP (Join-Path $work 'runner-temp') + + $jsonPath = Join-Path $work 'docker-runtime-manager.json' + $output = @(& pwsh -NoLogo -NoProfile -File $script:ManagerScript ` + -ProbeScope windows ` + -OutputJsonPath $jsonPath ` + -CommandTimeoutSeconds 5 ` + -SwitchRetryCount 1 ` + -SwitchTimeoutSeconds 30 2>&1) + + $LASTEXITCODE | Should -Be 0 -Because ($output -join "`n") + + $json = Get-Content -LiteralPath $jsonPath -Raw | ConvertFrom-Json -Depth 30 + $json.status | Should -Be 'success' + $json.probes.windows.status | Should -Be 'success' + $json.probes.windows.digest | Should -Be 'sha256:1111111111111111111111111111111111111111111111111111111111111111' + } + It 'fails with lock timeout when the runtime manager lock is held by another process' { $work = Join-Path $TestDrive 'lock-timeout' New-Item -ItemType Directory -Path $work -Force | Out-Null diff --git a/tests/Test-WindowsNI2026q1HostPreflight.Tests.ps1 b/tests/Test-WindowsNI2026q1HostPreflight.Tests.ps1 index 09e2dc340..b7510739a 100644 --- a/tests/Test-WindowsNI2026q1HostPreflight.Tests.ps1 +++ b/tests/Test-WindowsNI2026q1HostPreflight.Tests.ps1 @@ -76,6 +76,31 @@ if ($Args[0] -eq 'info') { exit 0 } +if ($Args[0] -eq 'manifest' -and $Args.Count -ge 3 -and $Args[1] -eq 'inspect') { + $paddingBytes = [Environment]::GetEnvironmentVariable('DOCKER_STUB_MANIFEST_PADDING_BYTES') + $padding = '' + if (-not [string]::IsNullOrWhiteSpace($paddingBytes)) { + $padding = ('x' * [int]$paddingBytes) + } + $manifest = [ordered]@{ + schemaVersion = 2 + annotations = [ordered]@{ + padding = $padding + } + manifests = @( + [ordered]@{ + digest = 'sha256:1111111111111111111111111111111111111111111111111111111111111111' + platform = [ordered]@{ + os = 'windows' + architecture = 'amd64' + } + } + ) + } + ($manifest | ConvertTo-Json -Depth 10) | Write-Output + exit 0 +} + if ($Args[0] -eq 'image' -and $Args.Count -ge 2 -and $Args[1] -eq 'inspect') { $inspectSleep = [Environment]::GetEnvironmentVariable('DOCKER_STUB_INSPECT_SLEEP_SECONDS') if (-not [string]::IsNullOrWhiteSpace($inspectSleep)) { @@ -165,6 +190,7 @@ exit 0 DOCKER_STUB_INFO_EXITCODE = $env:DOCKER_STUB_INFO_EXITCODE DOCKER_STUB_RUN_STDERR = $env:DOCKER_STUB_RUN_STDERR DOCKER_STUB_RUN_EXITCODE = $env:DOCKER_STUB_RUN_EXITCODE + DOCKER_STUB_MANIFEST_PADDING_BYTES = $env:DOCKER_STUB_MANIFEST_PADDING_BYTES DOCKER_STUB_INSPECT_SLEEP_SECONDS = $env:DOCKER_STUB_INSPECT_SLEEP_SECONDS DOCKER_STUB_PULL_SLEEP_SECONDS = $env:DOCKER_STUB_PULL_SLEEP_SECONDS DOCKER_STUB_RUN_SLEEP_SECONDS = $env:DOCKER_STUB_RUN_SLEEP_SECONDS @@ -351,4 +377,34 @@ exit 0 $json.bootstrap.attempted | Should -BeFalse $json.probe.attempted | Should -BeFalse } + + It 'keeps desktop-local preflight ready when manifest output is large' { + $work = Join-Path $TestDrive 'desktop-local-large-manifest' + New-Item -ItemType Directory -Path $work -Force | Out-Null + & $script:CreateDockerHostedStubs -WorkRoot $work + + Set-Item Env:DOCKER_STUB_CONTEXT 'desktop-windows' + Set-Item Env:DOCKER_STUB_OSTYPE 'windows' + Set-Item Env:DOCKER_STUB_IMAGE_EXISTS '1' + Set-Item Env:DOCKER_STUB_MANIFEST_PADDING_BYTES '20000' + Set-Item Env:RUNNER_TEMP (Join-Path $work 'runner-temp') + + $resultsRoot = Join-Path $work 'results' + $outputJsonPath = Join-Path $resultsRoot 'windows-ni-2026q1-host-preflight.json' + + $output = @(& pwsh -NoLogo -NoProfile -File $script:ToolPath ` + -Image 'nationalinstruments/labview:2026q1-windows' ` + -ResultsDir $resultsRoot ` + -ExecutionSurface 'desktop-local' ` + -CommandTimeoutSeconds 5 ` + -OutputJsonPath $outputJsonPath ` + -GitHubOutputPath '' ` + -StepSummaryPath '' 2>&1) + $LASTEXITCODE | Should -Be 0 -Because ($output -join "`n") + + $json = Get-Content -LiteralPath $outputJsonPath -Raw | ConvertFrom-Json -Depth 20 + $json.status | Should -Be 'ready' + $json.bootstrap.imagePresent | Should -BeTrue + $json.probe.status | Should -Be 'success' + } } diff --git a/tools/Assert-DockerRuntimeDeterminism.ps1 b/tools/Assert-DockerRuntimeDeterminism.ps1 index fa6462244..ce668f496 100644 --- a/tools/Assert-DockerRuntimeDeterminism.ps1 +++ b/tools/Assert-DockerRuntimeDeterminism.ps1 @@ -90,6 +90,8 @@ param( Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' +. (Join-Path $PSScriptRoot 'ProcessTimeoutHelper.ps1') + function Write-GitHubOutput { param( [Parameter(Mandatory = $true)][string]$Key, @@ -193,41 +195,13 @@ function Invoke-ProcessWithTimeout { exception = '' } - $psi = [System.Diagnostics.ProcessStartInfo]::new() - $psi.FileName = $resolvedFilePath - $psi.UseShellExecute = $false - $psi.RedirectStandardOutput = $true - $psi.RedirectStandardError = $true - $psi.CreateNoWindow = $true - foreach ($arg in @($effectiveArguments)) { - [void]$psi.ArgumentList.Add([string]$arg) - } - - $proc = [System.Diagnostics.Process]::new() - $proc.StartInfo = $psi - - try { - [void]$proc.Start() - $completed = $proc.WaitForExit($safeTimeout * 1000) - if (-not $completed) { - $result.timedOut = $true - try { $proc.Kill($true) } catch {} - return [pscustomobject]$result - } - - $result.exitCode = [int]$proc.ExitCode - $result.stdout = @(Split-OutputLines -Text $proc.StandardOutput.ReadToEnd()) - $result.stderr = @(Split-OutputLines -Text $proc.StandardError.ReadToEnd()) - } catch { - $result.exception = [string]$_.Exception.Message - try { - if (-not $proc.HasExited) { - $proc.Kill($true) - } - } catch {} - } finally { - $proc.Dispose() - } + $invoke = Invoke-ProcessWithTimeoutCore -FilePath $resolvedFilePath -Arguments @($effectiveArguments) -TimeoutSeconds $safeTimeout + $result.timedOut = [bool]$invoke.TimedOut + $result.exitCode = $invoke.ExitCode + $result.stdout = @($invoke.Stdout | ForEach-Object { [string]$_ }) + $result.stderr = @($invoke.Stderr | ForEach-Object { [string]$_ }) + $result.command = [string]$invoke.Command + $result.exception = [string]$invoke.Exception return [pscustomobject]$result } diff --git a/tools/Invoke-DockerRuntimeManager.ps1 b/tools/Invoke-DockerRuntimeManager.ps1 index fa59acfd2..c08920e12 100644 --- a/tools/Invoke-DockerRuntimeManager.ps1 +++ b/tools/Invoke-DockerRuntimeManager.ps1 @@ -44,6 +44,8 @@ param( Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' +. (Join-Path $PSScriptRoot 'ProcessTimeoutHelper.ps1') + function Resolve-AbsolutePath { param([Parameter(Mandatory)][string]$Path) if ([System.IO.Path]::IsPathRooted($Path)) { @@ -146,41 +148,13 @@ function Invoke-ProcessWithTimeout { exception = '' } - $psi = [System.Diagnostics.ProcessStartInfo]::new() - $psi.FileName = $resolvedFilePath - $psi.UseShellExecute = $false - $psi.RedirectStandardOutput = $true - $psi.RedirectStandardError = $true - $psi.CreateNoWindow = $true - foreach ($arg in @($effectiveArguments)) { - [void]$psi.ArgumentList.Add([string]$arg) - } - - $proc = [System.Diagnostics.Process]::new() - $proc.StartInfo = $psi - - try { - [void]$proc.Start() - $completed = $proc.WaitForExit($safeTimeout * 1000) - if (-not $completed) { - $result.timedOut = $true - try { $proc.Kill($true) } catch {} - return [pscustomobject]$result - } - - $result.exitCode = [int]$proc.ExitCode - $result.stdout = @(Split-OutputLines -Text $proc.StandardOutput.ReadToEnd()) - $result.stderr = @(Split-OutputLines -Text $proc.StandardError.ReadToEnd()) - } catch { - $result.exception = [string]$_.Exception.Message - try { - if (-not $proc.HasExited) { - $proc.Kill($true) - } - } catch {} - } finally { - $proc.Dispose() - } + $invoke = Invoke-ProcessWithTimeoutCore -FilePath $resolvedFilePath -Arguments @($effectiveArguments) -TimeoutSeconds $safeTimeout + $result.timedOut = [bool]$invoke.TimedOut + $result.exitCode = $invoke.ExitCode + $result.stdout = @($invoke.Stdout | ForEach-Object { [string]$_ }) + $result.stderr = @($invoke.Stderr | ForEach-Object { [string]$_ }) + $result.command = [string]$invoke.Command + $result.exception = [string]$invoke.Exception return [pscustomobject]$result } diff --git a/tools/ProcessTimeoutHelper.ps1 b/tools/ProcessTimeoutHelper.ps1 new file mode 100644 index 000000000..1b62d9a52 --- /dev/null +++ b/tools/ProcessTimeoutHelper.ps1 @@ -0,0 +1,147 @@ +#Requires -Version 7.0 + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +if (-not ('CompareVI.ProcessInvokeHelper' -as [type])) { + Add-Type -TypeDefinition @" +using System; +using System.Diagnostics; +using System.Text; +using System.Threading; + +namespace CompareVI { + public sealed class ProcessInvokeResult { + public bool TimedOut { get; set; } + public int? ExitCode { get; set; } + public string[] Stdout { get; set; } = Array.Empty(); + public string[] Stderr { get; set; } = Array.Empty(); + public string Command { get; set; } = ""; + public string Exception { get; set; } = ""; + } + + public static class ProcessInvokeHelper { + private static string[] SplitLines(string value) { + if (string.IsNullOrEmpty(value)) { + return Array.Empty(); + } + + return value + .Replace("\r\n", "\n") + .Replace('\r', '\n') + .Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries); + } + + public static ProcessInvokeResult Run(string filePath, string[] arguments, int timeoutSeconds) { + var safeTimeoutSeconds = Math.Max(5, timeoutSeconds); + var result = new ProcessInvokeResult(); + + using var stdoutClosed = new ManualResetEventSlim(false); + using var stderrClosed = new ManualResetEventSlim(false); + var stdout = new StringBuilder(); + var stderr = new StringBuilder(); + + var psi = new ProcessStartInfo { + FileName = filePath, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true + }; + + if (arguments != null) { + foreach (var argument in arguments) { + psi.ArgumentList.Add(argument ?? string.Empty); + } + } + + result.Command = psi.FileName + (psi.ArgumentList.Count > 0 ? " " + string.Join(" ", psi.ArgumentList) : string.Empty); + + using var process = new Process { + StartInfo = psi, + EnableRaisingEvents = true + }; + + process.OutputDataReceived += (_, eventArgs) => { + if (eventArgs.Data == null) { + stdoutClosed.Set(); + return; + } + + lock (stdout) { + stdout.AppendLine(eventArgs.Data); + } + }; + + process.ErrorDataReceived += (_, eventArgs) => { + if (eventArgs.Data == null) { + stderrClosed.Set(); + return; + } + + lock (stderr) { + stderr.AppendLine(eventArgs.Data); + } + }; + + try { + if (!process.Start()) { + result.Exception = "Process failed to start."; + return result; + } + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + if (!process.WaitForExit(safeTimeoutSeconds * 1000)) { + result.TimedOut = true; + try { + process.Kill(true); + } catch { + } + } + + try { + process.WaitForExit(); + } catch { + } + + stdoutClosed.Wait(2000); + stderrClosed.Wait(2000); + + if (!result.TimedOut) { + result.ExitCode = process.ExitCode; + } + } catch (Exception ex) { + result.Exception = ex.Message; + try { + if (!process.HasExited) { + process.Kill(true); + } + } catch { + } + } + + result.Stdout = SplitLines(stdout.ToString()); + result.Stderr = SplitLines(stderr.ToString()); + return result; + } + } +} +"@ -Language CSharp +} + +function Invoke-ProcessWithTimeoutCore { + [CmdletBinding()] + param( + [Parameter(Mandatory)][string]$FilePath, + [string[]]$Arguments = @(), + [int]$TimeoutSeconds = 45 + ) + + return [CompareVI.ProcessInvokeHelper]::Run( + [string]$FilePath, + [string[]]@($Arguments), + [Math]::Max(5, [int]$TimeoutSeconds) + ) +} diff --git a/tools/Test-WindowsNI2026q1HostPreflight.ps1 b/tools/Test-WindowsNI2026q1HostPreflight.ps1 index 81a863e47..4ed56c33b 100644 --- a/tools/Test-WindowsNI2026q1HostPreflight.ps1 +++ b/tools/Test-WindowsNI2026q1HostPreflight.ps1 @@ -39,6 +39,8 @@ param( Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' +. (Join-Path $PSScriptRoot 'ProcessTimeoutHelper.ps1') + function Resolve-AbsolutePath { param([Parameter(Mandatory)][string]$Path) return $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($Path) @@ -158,41 +160,13 @@ function Invoke-ProcessWithTimeout { exception = '' } - $psi = [System.Diagnostics.ProcessStartInfo]::new() - $psi.FileName = $resolvedFilePath - $psi.UseShellExecute = $false - $psi.RedirectStandardOutput = $true - $psi.RedirectStandardError = $true - $psi.CreateNoWindow = $true - foreach ($arg in @($effectiveArguments)) { - [void]$psi.ArgumentList.Add([string]$arg) - } - - $proc = [System.Diagnostics.Process]::new() - $proc.StartInfo = $psi - - try { - [void]$proc.Start() - $completed = $proc.WaitForExit($safeTimeout * 1000) - if (-not $completed) { - $result.timedOut = $true - try { $proc.Kill($true) } catch {} - return [pscustomobject]$result - } - - $result.exitCode = [int]$proc.ExitCode - $result.stdout = @(Split-OutputLines -Text $proc.StandardOutput.ReadToEnd()) - $result.stderr = @(Split-OutputLines -Text $proc.StandardError.ReadToEnd()) - } catch { - $result.exception = [string]$_.Exception.Message - try { - if (-not $proc.HasExited) { - $proc.Kill($true) - } - } catch {} - } finally { - $proc.Dispose() - } + $invoke = Invoke-ProcessWithTimeoutCore -FilePath $resolvedFilePath -Arguments @($effectiveArguments) -TimeoutSeconds $safeTimeout + $result.timedOut = [bool]$invoke.TimedOut + $result.exitCode = $invoke.ExitCode + $result.stdout = @($invoke.Stdout | ForEach-Object { [string]$_ }) + $result.stderr = @($invoke.Stderr | ForEach-Object { [string]$_ }) + $result.command = [string]$invoke.Command + $result.exception = [string]$invoke.Exception return [pscustomobject]$result } From 1086a80412c34d7923dff0491a3d432525dafa01 Mon Sep 17 00:00:00 2001 From: svelderrainruiz Date: Wed, 1 Apr 2026 09:45:31 -0700 Subject: [PATCH 40/44] chore(release): prepare v0.6.12 --- Directory.Build.props | 6 +++--- package-lock.json | 4 ++-- package.json | 2 +- tools/CompareVI.Tools/CompareVI.Tools.psd1 | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index eb5b0923f..936fc2e2e 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -7,9 +7,9 @@ false true true - 0.6.9 - 0.6.9.0 - 0.6.9.0 + 0.6.12 + 0.6.12.0 + 0.6.12.0 $(Version)+local package-first project diff --git a/package-lock.json b/package-lock.json index cceb48561..c73cc2f1a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "compare-vi-cli-action", - "version": "0.6.9", + "version": "0.6.12", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "compare-vi-cli-action", - "version": "0.6.9", + "version": "0.6.12", "license": "BSD-3-Clause", "dependencies": { "argparse": "^2.0.1", diff --git a/package.json b/package.json index 9395da7f5..7516971f1 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "compare-vi-cli-action", "private": true, - "version": "0.6.9", + "version": "0.6.12", "license": "BSD-3-Clause", "type": "module", "scripts": { diff --git a/tools/CompareVI.Tools/CompareVI.Tools.psd1 b/tools/CompareVI.Tools/CompareVI.Tools.psd1 index 39a3ba310..926dfc0bf 100644 --- a/tools/CompareVI.Tools/CompareVI.Tools.psd1 +++ b/tools/CompareVI.Tools/CompareVI.Tools.psd1 @@ -1,6 +1,6 @@ @{ RootModule = 'CompareVI.Tools.psm1' - ModuleVersion = '0.6.9' + ModuleVersion = '0.6.12' GUID = '1f9b5f7f-1ab6-4db9-8e36-6b7a6d5e9c8f' Author = 'LabVIEW Community CI' CompanyName = 'LabVIEW Community' From 7f0fc5a2f16adc324169505c5590c49a6836966a Mon Sep 17 00:00:00 2001 From: svelderrainruiz Date: Wed, 1 Apr 2026 09:57:58 -0700 Subject: [PATCH 41/44] fix(release): align v0.6.12 release packet --- CHANGELOG.md | 22 ++++++ .../archive/releases/RELEASE_NOTES_v0.6.12.md | 38 ++++++++++ docs/release/PR_NOTES.md | 70 ++++++++++--------- docs/release/README.md | 4 +- docs/release/TAG_PREP_CHECKLIST.md | 52 +++++++------- 5 files changed, 129 insertions(+), 57 deletions(-) create mode 100644 docs/archive/releases/RELEASE_NOTES_v0.6.12.md diff --git a/CHANGELOG.md b/CHANGELOG.md index fdcb605ad..2d5950308 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,28 @@ The format is based on Keep a Changelog, and this project adheres to Semantic Ve ## [Unreleased] +## [v0.6.12] - 2026-04-01 + +### Fixed + +- Reworked Windows preflight and Docker runtime-manager process capture so + large `docker manifest inspect` payloads no longer deadlock the bounded + timeout path, preserving truthful Windows NI preflight and runtime + determinism outcomes instead of hanging the release-proof surface. + +### Added + +- Regression coverage in `tests/Invoke-DockerRuntimeManager.Tests.ps1` and + `tests/Test-WindowsNI2026q1HostPreflight.Tests.ps1` proving large manifest + output still completes cleanly through the shared timeout helper. + +### Changed + +- Built the `v0.6.12` release line on top of the published `v0.6.11` stable + baseline, preserving the Windows NI proof authority and VI-history + native-path repair while adding the preflight deadlock fix required for a + clean release rail. + ## [v0.6.11] - 2026-04-01 ### Fixed diff --git a/docs/archive/releases/RELEASE_NOTES_v0.6.12.md b/docs/archive/releases/RELEASE_NOTES_v0.6.12.md new file mode 100644 index 000000000..29a575938 --- /dev/null +++ b/docs/archive/releases/RELEASE_NOTES_v0.6.12.md @@ -0,0 +1,38 @@ +# Release Notes v0.6.12 + +`v0.6.12` is a maintenance release that carries the published `v0.6.11` stable +line forward and fixes the Windows preflight process-capture deadlock exposed +when Docker returns large manifest output. + +## Highlights + +- The released backend no longer deadlocks while collecting large Docker + manifest payloads during Windows preflight and runtime-manager checks. +- A shared timeout helper now owns bounded process execution and concurrent + stdout/stderr draining for the Windows preflight seam. +- The stable release keeps the `v0.6.11` Windows NI proof authority and + VI-history native-path repair instead of regressing to an older integration + surface. + +## Included maintenance slice + +- `#2091` fix: Windows preflight process capture deadlock +- carry forward the published `v0.6.11` release-line policy and proof surfaces + +## Validation highlights + +- Focused Pester coverage proves large-manifest replay no longer blocks either + `tools/Invoke-DockerRuntimeManager.ps1` or + `tools/Test-WindowsNI2026q1HostPreflight.ps1`. +- The release branch is prepared from the published `v0.6.11` line and merged + with the post-`#2091` `develop` tip, preserving the already-published stable + fixes while adding the Windows preflight deadlock repair. +- Release helper docs, changelog, and version surfaces align on `0.6.12`. + +## Consumer impact + +- Stable consumers should move from `@v0.6.11` to `@v0.6.12` to pick up the + Windows preflight/runtime-manager deadlock repair. +- `comparevi-history` should repin `comparevi-backend-ref.txt` to `v0.6.12` + before rerunning the clone-backed `ni/labview-icon-editor` proof on the + released backend. diff --git a/docs/release/PR_NOTES.md b/docs/release/PR_NOTES.md index 27026a95f..dc0f0b8f4 100644 --- a/docs/release/PR_NOTES.md +++ b/docs/release/PR_NOTES.md @@ -1,37 +1,41 @@ -# Release v0.6.11 - PR Notes Helper +# Release v0.6.12 - PR Notes Helper -Reference sheet for the `v0.6.11` maintenance release. This cut carries the -`v0.6.10` stable line forward and adds the VI-history native-path repair proven -on the Windows NI Docker-backed proof surface. +Reference sheet for the `v0.6.12` maintenance release. This cut carries the +published `v0.6.11` stable line forward and adds the Windows preflight +process-capture deadlock fix proven on the Windows NI Docker-backed proof +surface. ## 1. Summary -Release `v0.6.11` focuses on three themes: +Release `v0.6.12` focuses on three themes: -- **VI-history native-path correctness**: the released backend now resolves - Windows NI proof-surface file paths without losing the repository-relative VI - target, so hosted and local replay lanes can reach the real history target - instead of a synthetic temp-root mismatch. +- **Deadlock-free Windows preflight capture**: the released backend now drains + large Docker manifest output without stalling the bounded timeout path, so + Windows NI preflight and runtime-manager checks can report real readiness + instead of hanging. - **Windows NI proof continuity**: the LabVIEW Docker-backed Windows proof path - remains the authoritative hosted execution surface, and `v0.6.11` is cut - directly from the `v0.6.10` stable line so the released proof authority is - preserved rather than reintroduced from stale `develop`. + remains the authoritative hosted execution surface, and `v0.6.12` is cut + directly from the published `v0.6.11` stable line so the released proof + authority is preserved rather than reintroduced from stale `develop`. - **Consumer-ready repin path**: the release packet is aligned for the next `comparevi-history` pin bump and the clone-backed `ni/labview-icon-editor` history proof rerun on the newly published backend. ## 2. Maintenance Highlights -- `tools/Compare-VIHistory.ps1`, `tools/Compare-RefsToTemp.ps1`, and - `tools/Render-VIHistoryReport.ps1` now preserve the intended VI-history - target path across the Windows NI proof surface instead of collapsing to a - host-native path that the replay layer cannot certify. -- `tests/TestFileExistsAtRef.Tests.ps1` and - `tests/CompareVI.GitRefs.VI2.Tests.ps1` now cover the backend-side path and - git-ref seams that caused the Windows proof regression. -- Stable release surfaces now pin `0.6.11`, while the helper docs still point - consumers at `v0.6.10` until publication completes. +- `tools/ProcessTimeoutHelper.ps1` now centralizes bounded process execution + and concurrent stdout/stderr capture for the Docker-backed Windows proof + seam. +- `tools/Invoke-DockerRuntimeManager.ps1`, + `tools/Assert-DockerRuntimeDeterminism.ps1`, and + `tools/Test-WindowsNI2026q1HostPreflight.ps1` now share that helper so large + manifest responses cannot deadlock the release-proof path. +- `tests/Invoke-DockerRuntimeManager.Tests.ps1` and + `tests/Test-WindowsNI2026q1HostPreflight.Tests.ps1` now prove large manifest + output stays non-blocking on the authoritative Windows surface. +- Stable release surfaces now pin `0.6.12`, while the helper docs still point + consumers at `v0.6.11` until publication completes. ## 3. Validation Snapshot @@ -44,12 +48,12 @@ Release `v0.6.11` focuses on three themes: - [ ] Hosted Windows NI Docker proof is green on the release branch. - [ ] Local-proof autonomy selector still emits the truthful next proof surface instead of looping or hanging. -- [ ] `node tools/npm/run-script.mjs release:finalize -- 0.6.11` completes from +- [ ] `node tools/npm/run-script.mjs release:finalize -- 0.6.12` completes from a clean helper lane and writes fresh finalize metadata under `tests/results/_agent/release/` -- [ ] Published release `v0.6.11` includes the signed distribution assets, +- [ ] Published release `v0.6.12` includes the signed distribution assets, `SHA256SUMS.txt`, `sbom.spdx.json`, and `provenance.json` -- [ ] `comparevi-history` repins `comparevi-backend-ref.txt` to `v0.6.11` +- [ ] `comparevi-history` repins `comparevi-backend-ref.txt` to `v0.6.12` before the clone-backed `ni/labview-icon-editor` proof is rerun ## 4. Reviewer Focus @@ -58,23 +62,25 @@ Release `v0.6.11` focuses on three themes: - `package.json` - `Directory.Build.props` - `tools/CompareVI.Tools/CompareVI.Tools.psd1` -- Review the released Windows NI Docker proof and VI-history backend surfaces for correctness: - - `.github/workflows/windows-ni-proof-reusable.yml` - - `tools/Compare-VIHistory.ps1` - - `tools/Compare-RefsToTemp.ps1` - - `tools/Render-VIHistoryReport.ps1` +- Review the released Windows preflight/runtime-manager surfaces for + correctness: + - `tools/ProcessTimeoutHelper.ps1` + - `tools/Invoke-DockerRuntimeManager.ps1` + - `tools/Test-WindowsNI2026q1HostPreflight.ps1` + - `tests/Invoke-DockerRuntimeManager.Tests.ps1` + - `tests/Test-WindowsNI2026q1HostPreflight.Tests.ps1` - Review the release helper packet for consistency: - `CHANGELOG.md` - `docs/release/TAG_PREP_CHECKLIST.md` - - `docs/archive/releases/RELEASE_NOTES_v0.6.11.md` + - `docs/archive/releases/RELEASE_NOTES_v0.6.12.md` ## 5. Follow-Up After Stable -1. Re-pin `comparevi-history` from `v0.6.10` to `v0.6.11` and rerun the +1. Re-pin `comparevi-history` from `v0.6.11` to `v0.6.12` and rerun the clone-backed `ni/labview-icon-editor` proof on the released backend. 2. Confirm the Windows NI proof artifacts and the published benchmark packet still agree on the certified backend version after the repin. 3. Re-evaluate the current emitted history surface against the real developer question before treating any mode as decision-ready. ---- Updated: 2026-04-01 (prepared for the `v0.6.11` maintenance cut). +--- Updated: 2026-04-01 (prepared for the `v0.6.12` maintenance cut). diff --git a/docs/release/README.md b/docs/release/README.md index ab9ed6058..0aa8ed8a7 100644 --- a/docs/release/README.md +++ b/docs/release/README.md @@ -7,7 +7,9 @@ work. ## Contents - `PR_NOTES.md` - release PR summary helper -- `RELEASE_EVIDENCE_v0.6.6.md` - latest immutable release and certified-consumer evidence packet retained in-repo while the `v0.6.11` cut supersedes the published `v0.6.10` baseline +- `RELEASE_EVIDENCE_v0.6.6.md` - latest immutable release and + certified-consumer evidence packet retained in-repo while the `v0.6.12` cut + supersedes the published `v0.6.11` baseline - `TAG_PREP_CHECKLIST.md` - tag preparation checklist - `POST_RELEASE_FOLLOWUPS.md` - post-release backlog tracker - `ROLLBACK_PLAN.md` - rollback procedure reference diff --git a/docs/release/TAG_PREP_CHECKLIST.md b/docs/release/TAG_PREP_CHECKLIST.md index a99859896..2e473222b 100644 --- a/docs/release/TAG_PREP_CHECKLIST.md +++ b/docs/release/TAG_PREP_CHECKLIST.md @@ -1,14 +1,14 @@ -# v0.6.11 Tag Preparation Checklist +# v0.6.12 Tag Preparation Checklist -Helper reference for cutting or replaying the `v0.6.11` maintenance release. +Helper reference for cutting or replaying the `v0.6.12` maintenance release. Aligns with the archived release notes -(`../archive/releases/RELEASE_NOTES_v0.6.11.md`) and the checked-in stable +(`../archive/releases/RELEASE_NOTES_v0.6.12.md`) and the checked-in stable release surfaces. ## 1. Pre-flight Verification -- [ ] Work from `release/v0.6.11` and ensure it contains the final maintenance +- [ ] Work from `release/v0.6.12` and ensure it contains the final maintenance changes. - [ ] CI is green on the release branch (`lint`, `pester / normalize`, `smoke-gate`, `Policy Guard (Upstream) / policy-guard`, @@ -22,26 +22,30 @@ release surfaces. ## 2. Version & Metadata Consistency - [ ] `CHANGELOG.md` contains a finalized - `## [v0.6.11] - 2026-04-01` section. -- [ ] Stable docs reference `v0.6.10` consistently until `v0.6.11` publication - completes, and the release helper packet references `v0.6.11` + `## [v0.6.12] - 2026-04-01` section. +- [ ] Stable docs reference `v0.6.11` consistently until `v0.6.12` publication + completes, and the release helper packet references `v0.6.12` consistently. - [ ] `package.json`, `Directory.Build.props`, and - `tools/CompareVI.Tools/CompareVI.Tools.psd1` all report `0.6.11`. + `tools/CompareVI.Tools/CompareVI.Tools.psd1` all report `0.6.12`. - [ ] `docs/action-outputs.md` still matches `action.yml`. - [ ] Update `docs/documentation-manifest.json` if release-doc coverage changed. -## 3. Bundle Contract Regression Validation +## 3. Windows Preflight Regression Validation -- [ ] Focused bundle regression tests pass locally: +- [ ] Focused Windows preflight/runtime-manager regression tests pass locally: ```bash -pwsh -NoLogo -NoProfile -File tools/Test-CompareVIHistoryBundleCertification.ps1 -BundleArchivePath tests/results/_agent/bundle-fix/artifacts/CompareVI.Tools-v0.6.11.zip -ResultsDir tests/results/_agent/bundle-fix/certification -SummaryJsonPath tests/results/_agent/bundle-fix/certification/summary.json +pwsh -NoLogo -NoProfile -File Invoke-PesterTests.ps1 -TestsPath tests/Invoke-DockerRuntimeManager.Tests.ps1 +pwsh -NoLogo -NoProfile -File Invoke-PesterTests.ps1 -TestsPath tests/Test-WindowsNI2026q1HostPreflight.Tests.ps1 ``` - [ ] Confirm the released Windows NI / LabVIEW Docker image proof surface is intact: - [ ] The released surface still includes: + `tools/ProcessTimeoutHelper.ps1` + `tools/Invoke-DockerRuntimeManager.ps1` + and `tools/Run-NIWindowsContainerCompare.ps1` and `tools/Test-WindowsNI2026q1HostPreflight.ps1` @@ -56,9 +60,9 @@ pwsh -NoLogo -NoProfile -File tools/Test-CompareVIHistoryBundleCertification.ps1 ## 4. Release Materials Review - [ ] `PR_NOTES.md`, this checklist, and - `../archive/releases/RELEASE_NOTES_v0.6.11.md` are consistent. -- [ ] `README.md` and `docs/USAGE_GUIDE.md` still treat `v0.6.10` as the - previously released stable pin until `v0.6.11` publication completes. + `../archive/releases/RELEASE_NOTES_v0.6.12.md` are consistent. +- [ ] `README.md` and `docs/USAGE_GUIDE.md` still treat `v0.6.11` as the + previously released stable pin until `v0.6.12` publication completes. ## 5. Tag Creation @@ -66,7 +70,7 @@ pwsh -NoLogo -NoProfile -File tools/Test-CompareVIHistoryBundleCertification.ps1 ```pwsh node tools/npm/run-script.mjs priority:release:signing:readiness -node tools/npm/run-script.mjs priority:release:conductor -- --apply --channel stable --version 0.6.11 +node tools/npm/run-script.mjs priority:release:conductor -- --apply --channel stable --version 0.6.12 ``` - [ ] Confirm `tests/results/_agent/release/release-signing-readiness.json` @@ -77,35 +81,35 @@ node tools/npm/run-script.mjs priority:release:conductor -- --apply --channel st - [ ] Create an annotated stable tag: ```pwsh -git tag -a v0.6.11 -m "v0.6.11: repair VI history native paths on Windows proof surfaces" +git tag -a v0.6.12 -m "v0.6.12: fix Windows preflight process capture deadlock" ``` - [ ] Push the tag: ```pwsh -git push origin v0.6.11 +git push origin v0.6.12 ``` ## 6. Validation After Publish -- [ ] Run `node tools/npm/run-script.mjs release:finalize -- 0.6.11` from a +- [ ] Run `node tools/npm/run-script.mjs release:finalize -- 0.6.12` from a clean helper lane to fast-forward `main` and `develop`, then record the finalize metadata. -- [ ] Install the bundle via `@v0.6.11` in a sample workflow and confirm the +- [ ] Install the bundle via `@v0.6.12` in a sample workflow and confirm the released Windows NI and hosted VI-history contracts execute without a local source-tree override. - [ ] Optional maintainer fast loop: run the local Windows Docker replay lane for `vi-history-scenarios-windows` and confirm it still mirrors the hosted contract without replacing the hosted proof requirement. -- [ ] Re-pin `comparevi-history` to `v0.6.11` and confirm the clone-backed +- [ ] Re-pin `comparevi-history` to `v0.6.12` and confirm the clone-backed `ni/labview-icon-editor` proof reaches real comparisons on the released backend. ## 7. Communication -- [ ] Announce the maintenance cut, calling out the promoted Windows NI Docker - proof surface and the required `comparevi-history` repin. -- [ ] Notify consumers that `v0.6.11` supersedes `v0.6.10` as the supported +- [ ] Announce the maintenance cut, calling out the Windows preflight deadlock + fix and the required `comparevi-history` repin. +- [ ] Notify consumers that `v0.6.12` supersedes `v0.6.11` as the supported stable pin. ---- Updated: 2026-04-01 (prepared for the `v0.6.11` maintenance cut). +--- Updated: 2026-04-01 (prepared for the `v0.6.12` maintenance cut). From a33b169fa1ceb1cec80b24ca8a946e8430cce724 Mon Sep 17 00:00:00 2001 From: svelderrainruiz Date: Wed, 1 Apr 2026 12:15:40 -0700 Subject: [PATCH 42/44] fix: unblock windows release guard regressions --- Invoke-PesterTests.ps1 | 39 ++++++++++--------- .../Assert-DockerRuntimeDeterminism.Tests.ps1 | 23 ++++++++++- tools/Assert-DockerRuntimeDeterminism.ps1 | 15 +++++-- 3 files changed, 55 insertions(+), 22 deletions(-) diff --git a/Invoke-PesterTests.ps1 b/Invoke-PesterTests.ps1 index e4a7ee1c4..c8423bc62 100644 --- a/Invoke-PesterTests.ps1 +++ b/Invoke-PesterTests.ps1 @@ -1204,11 +1204,6 @@ Write-Host " Script Root: $root" Write-Host " Tests Directory: $testsDir"; if ($limitToSingle) { Write-Host " Single Test File: $singleTestFile" } Write-Host " Results Directory: $resultsDir" Write-Host "" -$script:dispatcherEventsPath = Join-Path $resultsDir 'dispatcher-events.ndjson' -Write-DispatcherEventLine -Level info -Phase 'lifecycle' -Message 'Dispatcher session initialized.' -Data @{ - testsDir = $testsDir - resultsDir = $resultsDir -} # Validate tests directory exists if (-not (Test-Path -LiteralPath $testsDir -PathType Container)) { @@ -1306,6 +1301,27 @@ function _Build-Snapshot { return $index } +# Artifact tracking pre-snapshot (optional). Capture before any dispatcher-owned +# result files are written so isolated runs report those files as created rather +# than modified. +$script:artifactTrail = $null +$script:executionFinalizeContextPath = $null +$preIndex = $null +$artifactRoots = @() +if ($TrackArtifacts) { + if (-not $ArtifactGlobs -or $ArtifactGlobs.Count -eq 0) { + $ArtifactGlobs = @('tests/results','results','tmp-agg/results','scratch-schema-test/results') + } + $artifactRoots = _Resolve-ArtifactRoots -Roots $ArtifactGlobs -Base $root + try { $preIndex = _Build-Snapshot -Roots $artifactRoots -Base $root } catch { $preIndex = @{} } +} + +$script:dispatcherEventsPath = Join-Path $resultsDir 'dispatcher-events.ndjson' +Write-DispatcherEventLine -Level info -Phase 'lifecycle' -Message 'Dispatcher session initialized.' -Data @{ + testsDir = $testsDir + resultsDir = $resultsDir +} + # Hard gate: never start tests while LabVIEW.exe is running $labviewOpen = @(_Find-ProcsByPattern -Patterns @('LabVIEW') ) if ($labviewOpen.Count -gt 0) { @@ -1765,19 +1781,6 @@ if ($CleanLabVIEW) { _Report-Procs -Names @('LabVIEW') } -# Artifact tracking pre-snapshot (optional) -$script:artifactTrail = $null -$script:executionFinalizeContextPath = $null -$preIndex = $null -$artifactRoots = @() -if ($TrackArtifacts) { - if (-not $ArtifactGlobs -or $ArtifactGlobs.Count -eq 0) { - $ArtifactGlobs = @('tests/results','results','tmp-agg/results','scratch-schema-test/results') - } - $artifactRoots = _Resolve-ArtifactRoots -Roots $ArtifactGlobs -Base $root - try { $preIndex = _Build-Snapshot -Roots $artifactRoots -Base $root } catch { $preIndex = @{} } -} - # Count test files (respect single file mode) if ($limitToSingle) { $testFiles = @([IO.FileInfo]::new($singleTestFile)) diff --git a/tests/Assert-DockerRuntimeDeterminism.Tests.ps1 b/tests/Assert-DockerRuntimeDeterminism.Tests.ps1 index 498c949ff..bffbd1067 100644 --- a/tests/Assert-DockerRuntimeDeterminism.Tests.ps1 +++ b/tests/Assert-DockerRuntimeDeterminism.Tests.ps1 @@ -157,7 +157,7 @@ exit 0 "@ Set-Content -LiteralPath (Join-Path $binDir 'wsl.cmd') -Value $wslCmd -Encoding ascii - $env:PATH = "{0};{1}" -f $binDir, $env:PATH + $env:PATH = "{0}{1}{2}" -f $binDir, [System.IO.Path]::PathSeparator, $env:PATH } } @@ -173,6 +173,7 @@ exit 0 DOCKER_STUB_CONTEXT_USE_FAIL_TARGET = $env:DOCKER_STUB_CONTEXT_USE_FAIL_TARGET DOCKER_COMMAND_OVERRIDE = $env:DOCKER_COMMAND_OVERRIDE DOCKER_HOST = $env:DOCKER_HOST + RUNNER_OS = $env:RUNNER_OS } } @@ -209,11 +210,13 @@ exit 0 Set-Item Env:DOCKER_STUB_INFO_MODE 'daemon-unavailable' Set-Item Env:DOCKER_STUB_CONTEXT 'desktop-windows' + Set-Item Env:RUNNER_OS 'Windows' $snapshotPath = Join-Path $work 'runtime.json' $output = & pwsh -NoLogo -NoProfile -File $script:GuardScript ` -ExpectedOsType windows ` -ExpectedContext desktop-windows ` + -HostPlatformOverride Windows ` -AutoRepair:$false ` -SnapshotPath $snapshotPath ` -GitHubOutputPath '' 2>&1 @@ -259,11 +262,13 @@ exit 9 Set-Item Env:DOCKER_COMMAND_OVERRIDE (Join-Path $work 'bin' 'docker.ps1') Set-Item Env:DOCKER_STUB_INFO_MODE 'parsed-linux' Set-Item Env:DOCKER_STUB_CONTEXT 'desktop-linux' + Set-Item Env:RUNNER_OS 'Windows' $snapshotPath = Join-Path $work 'runtime.json' $output = & pwsh -NoLogo -NoProfile -File $script:GuardScript ` -ExpectedOsType linux ` -ExpectedContext desktop-linux ` + -HostPlatformOverride Windows ` -AutoRepair:$false ` -SnapshotPath $snapshotPath ` -GitHubOutputPath '' 2>&1 @@ -281,12 +286,14 @@ exit 9 Set-Item Env:DOCKER_STUB_INFO_MODE 'unparseable-success' Set-Item Env:DOCKER_STUB_CONTEXT 'desktop-windows' + Set-Item Env:RUNNER_OS 'Windows' $snapshotPath = Join-Path $work 'runtime.json' $githubOutput = Join-Path $work 'github-output.txt' $output = & pwsh -NoLogo -NoProfile -File $script:GuardScript ` -ExpectedOsType windows ` -ExpectedContext desktop-windows ` + -HostPlatformOverride Windows ` -AutoRepair:$false ` -SnapshotPath $snapshotPath ` -GitHubOutputPath $githubOutput 2>&1 @@ -308,11 +315,13 @@ exit 9 Set-Item Env:DOCKER_STUB_INFO_MODE 'daemon-unavailable' Set-Item Env:DOCKER_STUB_CONTEXT 'desktop-windows' + Set-Item Env:RUNNER_OS 'Windows' $snapshotPath = Join-Path $work 'runtime.json' $output = & pwsh -NoLogo -NoProfile -File $script:GuardScript ` -ExpectedOsType windows ` -ExpectedContext desktop-windows ` + -HostPlatformOverride Windows ` -AutoRepair:$true ` -ManageDockerEngine:$true ` -SnapshotPath $snapshotPath ` @@ -333,11 +342,13 @@ exit 9 Set-Item Env:DOCKER_STUB_INFO_MODE 'parsed-linux' Set-Item Env:DOCKER_STUB_CONTEXT 'desktop-linux' + Set-Item Env:RUNNER_OS 'Windows' $snapshotPath = Join-Path $work 'runtime.json' $output = & pwsh -NoLogo -NoProfile -File $script:GuardScript ` -ExpectedOsType windows ` -ExpectedContext desktop-windows ` + -HostPlatformOverride Windows ` -AutoRepair:$true ` -ManageDockerEngine:$true ` -AllowHostEngineMutation:$false ` @@ -365,6 +376,7 @@ exit 9 Set-Item Env:DOCKER_STUB_INFO_MODE 'parsed-windows' Set-Item Env:DOCKER_STUB_CONTEXT 'desktop-windows' + Set-Item Env:RUNNER_OS 'Windows' $snapshotPath = Join-Path $work 'runtime.json' $githubOutput = Join-Path $work 'github-output.txt' @@ -372,6 +384,7 @@ exit 9 $output = & pwsh -NoLogo -NoProfile -File $script:GuardScript ` -ExpectedOsType windows ` -ExpectedContext desktop-windows ` + -HostPlatformOverride Windows ` -AutoRepair:$true ` -SnapshotPath $snapshotPath ` -GitHubOutputPath $githubOutput 2>&1 @@ -399,11 +412,13 @@ exit 9 Set-Item Env:DOCKER_STUB_INFO_MODE 'parsed-windows' Set-Item Env:DOCKER_STUB_CONTEXT 'default' + Set-Item Env:RUNNER_OS 'Windows' $snapshotPath = Join-Path $work 'runtime.json' $output = & pwsh -NoLogo -NoProfile -File $script:GuardScript ` -ExpectedOsType windows ` -ExpectedContext desktop-windows ` + -HostPlatformOverride Windows ` -AutoRepair:$true ` -SnapshotPath $snapshotPath ` -GitHubOutputPath '' 2>&1 @@ -424,11 +439,13 @@ exit 9 Set-Item Env:DOCKER_STUB_INFO_MODE 'parsed-windows' Set-Item Env:DOCKER_STUB_CONTEXT 'default' Set-Item Env:DOCKER_STUB_INFO_FAIL_CONTEXT 'desktop-windows' + Set-Item Env:RUNNER_OS 'Windows' $snapshotPath = Join-Path $work 'runtime.json' $output = & pwsh -NoLogo -NoProfile -File $script:GuardScript ` -ExpectedOsType windows ` -ExpectedContext desktop-windows ` + -HostPlatformOverride Windows ` -AutoRepair:$true ` -SnapshotPath $snapshotPath ` -GitHubOutputPath '' 2>&1 @@ -448,12 +465,14 @@ exit 9 Set-Item Env:DOCKER_STUB_INFO_MODE 'parsed-linux' Set-Item Env:DOCKER_STUB_INFO_JSON '{"OSType":"linux","OperatingSystem":"Ubuntu 24.04.1 LTS","Name":"ubuntu-native","Platform":{"Name":"Docker Engine - Community"},"Labels":["maintainer=comparevi"]}' Set-Item Env:DOCKER_HOST 'unix:///var/run/docker.sock' + Set-Item Env:RUNNER_OS 'Windows' $snapshotPath = Join-Path $work 'runtime.json' $output = & pwsh -NoLogo -NoProfile -File $script:GuardScript ` -ExpectedOsType linux ` -RuntimeProvider native-wsl ` -ExpectedDockerHost 'unix:///var/run/docker.sock' ` + -HostPlatformOverride Windows ` -AutoRepair:$true ` -SnapshotPath $snapshotPath ` -GitHubOutputPath '' 2>&1 @@ -478,12 +497,14 @@ exit 9 Set-Item Env:DOCKER_STUB_INFO_MODE 'parsed-linux' Set-Item Env:DOCKER_STUB_INFO_JSON '{"OSType":"linux","OperatingSystem":"Docker Desktop","Name":"docker-desktop","Platform":{"Name":"Docker Desktop 4.41.0"},"Labels":["com.docker.desktop.address=npipe://"]}' Set-Item Env:DOCKER_HOST 'unix:///var/run/docker.sock' + Set-Item Env:RUNNER_OS 'Windows' $snapshotPath = Join-Path $work 'runtime.json' $output = & pwsh -NoLogo -NoProfile -File $script:GuardScript ` -ExpectedOsType linux ` -RuntimeProvider native-wsl ` -ExpectedDockerHost 'unix:///var/run/docker.sock' ` + -HostPlatformOverride Windows ` -AutoRepair:$true ` -SnapshotPath $snapshotPath ` -GitHubOutputPath '' 2>&1 diff --git a/tools/Assert-DockerRuntimeDeterminism.ps1 b/tools/Assert-DockerRuntimeDeterminism.ps1 index ce668f496..b406bed6c 100644 --- a/tools/Assert-DockerRuntimeDeterminism.ps1 +++ b/tools/Assert-DockerRuntimeDeterminism.ps1 @@ -974,9 +974,18 @@ if ($ExpectedOsType -eq 'windows') { $hostAlignmentOk = $false $reason = "RUNNER_OS is '$runnerOsRaw', expected Windows." } -} elseif (-not [string]::IsNullOrWhiteSpace($runnerOsNormalized) -and $runnerOsNormalized -ne 'linux') { - $hostAlignmentOk = $false - $reason = "RUNNER_OS is '$runnerOsRaw', expected Linux for linux lane." +} elseif (-not [string]::IsNullOrWhiteSpace($runnerOsNormalized)) { + # Linux Docker lanes are valid on Windows Docker Desktop / native-wsl hosts, + # but non-Windows hosts still need a Linux runner identity. + if ($hostIsWindows) { + if ($runnerOsNormalized -notin @('windows', 'linux')) { + $hostAlignmentOk = $false + $reason = "RUNNER_OS is '$runnerOsRaw', expected Windows or Linux for linux lane on a Windows host." + } + } elseif ($runnerOsNormalized -ne 'linux') { + $hostAlignmentOk = $false + $reason = "RUNNER_OS is '$runnerOsRaw', expected Linux for linux lane." + } } $observedDockerHost = if ([string]::IsNullOrWhiteSpace($env:DOCKER_HOST)) { $null } else { $env:DOCKER_HOST.Trim() } From 5e2e9a31e93527ba9af4f907595bd57869d59a81 Mon Sep 17 00:00:00 2001 From: svelderrainruiz Date: Wed, 1 Apr 2026 13:01:16 -0700 Subject: [PATCH 43/44] fix: tolerate missing collapsed comparisons in history report fallback --- tests/Render-VIHistoryReport.Tests.ps1 | 26 ++++++++++++++++++++++++++ tools/Render-VIHistoryReport.ps1 | 12 +++++++----- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/tests/Render-VIHistoryReport.Tests.ps1 b/tests/Render-VIHistoryReport.Tests.ps1 index d05351cd2..b216080da 100644 --- a/tests/Render-VIHistoryReport.Tests.ps1 +++ b/tests/Render-VIHistoryReport.Tests.ps1 @@ -380,4 +380,30 @@ Describe 'Render-VIHistoryReport.ps1' -Tag 'Unit' { $facade.status | Should -Be 'ok' $facade.reason | Should -Be 'within-limit' } + + It 'renders the checked-in offline corpus suite without an explicit history context' { + $manifestPath = Join-Path $script:repoRoot 'fixtures' 'cross-repo' 'labview-icon-editor' 'settings-init' 'manifest.json' + $resultsRoot = Join-Path $TestDrive 'history-results-offline-corpus' + $markdownPath = Join-Path $resultsRoot 'history-report.md' + $htmlPath = Join-Path $resultsRoot 'history-report.html' + $stepSummaryPath = Join-Path $resultsRoot 'history-summary.md' + + & $script:scriptPath ` + -ManifestPath $manifestPath ` + -OutputDir $resultsRoot ` + -MarkdownPath $markdownPath ` + -EmitHtml ` + -HtmlPath $htmlPath ` + -StepSummaryPath $stepSummaryPath | Out-Null + + Test-Path -LiteralPath $markdownPath | Should -BeTrue + Test-Path -LiteralPath $htmlPath | Should -BeTrue + Test-Path -LiteralPath $stepSummaryPath | Should -BeTrue + + $markdown = Get-Content -LiteralPath $markdownPath -Raw + $markdown | Should -Match '## Observed interpretation' + $markdown | Should -Match '\| Coverage Class \| `[^`]+` \|' + $markdown | Should -Match '\| Mode Sensitivity \| `single-mode-observed` \|' + $markdown | Should -Match '\| Outcome Labels \| `clean`, `signal-diff` \|' + } } diff --git a/tools/Render-VIHistoryReport.ps1 b/tools/Render-VIHistoryReport.ps1 index 3c7284a4f..6ddf973e9 100644 --- a/tools/Render-VIHistoryReport.ps1 +++ b/tools/Render-VIHistoryReport.ps1 @@ -952,12 +952,14 @@ function Build-FallbackHistoryContext { } $modeComparisons = New-Object System.Collections.Generic.List[object] - foreach ($comparisonEntry in @($modeManifest.comparisons)) { - if ($comparisonEntry) { - $modeComparisons.Add($comparisonEntry) | Out-Null - } + $comparisonEntries = @() + if ($modeManifest.PSObject.Properties['comparisons']) { + $comparisonEntries += @($modeManifest.comparisons) + } + if ($modeManifest.PSObject.Properties['collapsedComparisons']) { + $comparisonEntries += @($modeManifest.collapsedComparisons) } - foreach ($comparisonEntry in @($modeManifest.collapsedComparisons)) { + foreach ($comparisonEntry in @($comparisonEntries)) { if ($comparisonEntry) { $modeComparisons.Add($comparisonEntry) | Out-Null } From 034d9b670fa5b8ea057944599fcfa119cd525d60 Mon Sep 17 00:00:00 2001 From: svelderrainruiz Date: Wed, 1 Apr 2026 14:13:35 -0700 Subject: [PATCH 44/44] test: stabilize nested dispatcher fixtures --- Invoke-PesterTests.ps1 | 3 +++ tests/FunctionShadowing.Nested.Tests.ps1 | 23 ++++++++++++++---- ...tcher.DiscoveryFailureRegression.Tests.ps1 | 24 +++++++++++++++---- tests/_helpers/DispatcherTestHelper.psm1 | 1 - 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/Invoke-PesterTests.ps1 b/Invoke-PesterTests.ps1 index c8423bc62..4ae31a763 100644 --- a/Invoke-PesterTests.ps1 +++ b/Invoke-PesterTests.ps1 @@ -567,6 +567,9 @@ $ErrorActionPreference = 'Stop' if (-not (Get-Variable -Name includeIntegrationBool -Scope Script -ErrorAction SilentlyContinue)) { $script:includeIntegrationBool = $false } +$script:stuckGuardEnabled = $false +$script:hbPath = $null +$script:partialLogPath = $null $script:executionPackResolved = 'full' $script:executionPackReason = 'default' $script:executionPackBaseIncludePatterns = @() diff --git a/tests/FunctionShadowing.Nested.Tests.ps1 b/tests/FunctionShadowing.Nested.Tests.ps1 index de638d85e..f9bc87ad3 100644 --- a/tests/FunctionShadowing.Nested.Tests.ps1 +++ b/tests/FunctionShadowing.Nested.Tests.ps1 @@ -41,9 +41,19 @@ Describe "Inner Smoke" { $toolsDir = Join-Path $workspace 'tools' New-Item -ItemType Directory -Path $toolsDir -Force | Out-Null - $trackerModule = Join-Path $repoRoot 'tools' 'LabVIEWPidTracker.psm1' - if (Test-Path -LiteralPath $trackerModule -PathType Leaf) { - Copy-Item -Path $trackerModule -Destination $toolsDir -Force + # Keep the nested workspace aligned with dispatcher-side tool growth. + $supportFiles = @( + 'LabVIEWPidTracker.psm1', + 'PesterExecutionPacks.ps1', + 'Invoke-PesterExecutionPostprocess.ps1', + 'PesterServiceModelSchema.ps1', + 'Get-PesterResultXmlSummary.ps1' + ) + foreach ($supportFile in $supportFiles) { + $supportPath = Join-Path $repoRoot 'tools' $supportFile + if (Test-Path -LiteralPath $supportPath -PathType Leaf) { + Copy-Item -Path $supportPath -Destination $toolsDir -Force + } } $dispatcherTools = Join-Path $repoRoot 'tools' 'Dispatcher' @@ -68,7 +78,12 @@ Describe "Inner Smoke" { $nestedExit | Should -Be 0 $json.failed | Should -Be 0 $json.errors | Should -Be 0 - $json.discoveryFailures | Should -Be 0 -Because 'Shadowing smoke test should not introduce discovery failures' + $discoveryFailures = if ($json.PSObject.Properties.Name -contains 'discoveryFailures') { + [int]$json.discoveryFailures + } else { + 0 + } + $discoveryFailures | Should -Be 0 -Because 'Shadowing smoke test should not introduce discovery failures' if ($json.failed -gt 0 -or $json.errors -gt 0 -or $json.discoveryFailures -gt 0) { Write-Host '[nested-shadow] DEBUG OUTPUT START' -ForegroundColor Yellow ($innerOutput | Out-String) | Write-Host diff --git a/tests/NestedDispatcher.DiscoveryFailureRegression.Tests.ps1 b/tests/NestedDispatcher.DiscoveryFailureRegression.Tests.ps1 index c80f4da4d..e95c05cb1 100644 --- a/tests/NestedDispatcher.DiscoveryFailureRegression.Tests.ps1 +++ b/tests/NestedDispatcher.DiscoveryFailureRegression.Tests.ps1 @@ -23,12 +23,26 @@ Describe 'Nested Dispatcher Discovery Failure Regression' -Tag 'Unit' { Set-Content -Path (Join-Path $testsDir 'Broken.Tests.ps1') -Value $badLines -Encoding UTF8 $dispatcherCopy = Join-Path $workspace 'Invoke-PesterTests.ps1' - Copy-Item -Path (Join-Path (Split-Path $PSScriptRoot -Parent) 'Invoke-PesterTests.ps1') -Destination $dispatcherCopy + $repoRoot = Split-Path $PSScriptRoot -Parent + Copy-Item -Path (Join-Path $repoRoot 'Invoke-PesterTests.ps1') -Destination $dispatcherCopy $toolsDir = Join-Path $workspace 'tools' - $trackerModule = Join-Path (Split-Path $PSScriptRoot -Parent) 'tools' 'LabVIEWPidTracker.psm1' - if (Test-Path -LiteralPath $trackerModule -PathType Leaf) { - New-Item -ItemType Directory -Path $toolsDir -Force | Out-Null - Copy-Item -Path $trackerModule -Destination $toolsDir -Force + New-Item -ItemType Directory -Path $toolsDir -Force | Out-Null + $supportFiles = @( + 'LabVIEWPidTracker.psm1', + 'PesterExecutionPacks.ps1', + 'Invoke-PesterExecutionPostprocess.ps1', + 'PesterServiceModelSchema.ps1', + 'Get-PesterResultXmlSummary.ps1' + ) + foreach ($supportFile in $supportFiles) { + $supportPath = Join-Path $repoRoot 'tools' $supportFile + if (Test-Path -LiteralPath $supportPath -PathType Leaf) { + Copy-Item -Path $supportPath -Destination $toolsDir -Force + } + } + $dispatcherTools = Join-Path $repoRoot 'tools' 'Dispatcher' + if (Test-Path -LiteralPath $dispatcherTools -PathType Container) { + Copy-Item -Path $dispatcherTools -Destination (Join-Path $toolsDir 'Dispatcher') -Recurse -Force } Push-Location $workspace diff --git a/tests/_helpers/DispatcherTestHelper.psm1 b/tests/_helpers/DispatcherTestHelper.psm1 index d73e50ae2..02d0a8cad 100644 --- a/tests/_helpers/DispatcherTestHelper.psm1 +++ b/tests/_helpers/DispatcherTestHelper.psm1 @@ -86,7 +86,6 @@ function Invoke-DispatcherSafe { $psi.EnvironmentVariables['SINGLE_INVOKER'] = '1' $psi.EnvironmentVariables['SUPPRESS_NESTED_DISCOVERY'] = '1' $psi.EnvironmentVariables['STUCK_GUARD'] = '0' - $psi.EnvironmentVariables['DISABLE_SINGLE_INVOKER'] = '1' $psi.EnvironmentVariables['SUPPRESS_PATTERN_SELFTEST'] = '1' $baseline = Get-PwshProcessIds