From ef55f6a9f984273b5daa41cf26ac824b72b5345c Mon Sep 17 00:00:00 2001 From: sergiov-ni Date: Thu, 21 Aug 2025 07:34:39 -0700 Subject: [PATCH] chore: record updated requirements summary (REQ-001) --- .github/workflows/update-runner-runtime.yml | 77 +++ artifacts/linux/requirements-summary.md | 70 +-- artifacts/linux/summary-standard.md | 4 +- artifacts/linux/summary.md | 17 +- artifacts/linux/traceability-standard.md | 116 +--- artifacts/linux/traceability.json | 591 +------------------- artifacts/linux/traceability.md | 116 +--- ci_evidence.txt | 2 +- docs/runner-types.md | 11 + test-results/node-junit.xml | 64 --- 10 files changed, 104 insertions(+), 964 deletions(-) create mode 100644 .github/workflows/update-runner-runtime.yml diff --git a/.github/workflows/update-runner-runtime.yml b/.github/workflows/update-runner-runtime.yml new file mode 100644 index 00000000..5b536669 --- /dev/null +++ b/.github/workflows/update-runner-runtime.yml @@ -0,0 +1,77 @@ +name: Update Runner Runtime + +on: + workflow_dispatch: + inputs: + runner_key: + description: 'Key for the runner entry in requirements files' + required: true + runner_label: + description: 'New runner_label value' + required: false + runner_type: + description: 'New runner_type value' + required: false + skip_dry_run: + description: 'Set skip_dry_run for the runner' + required: false + type: boolean + default: false + +jobs: + update: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Update requirements + shell: bash + env: + RUNNER_KEY: ${{ inputs.runner_key }} + RUNNER_LABEL: ${{ inputs.runner_label }} + RUNNER_TYPE: ${{ inputs.runner_type }} + SKIP_DRY_RUN: ${{ inputs.skip_dry_run }} + run: | + node - <<'NODE' + const fs = require('fs'); + const key = process.env.RUNNER_KEY; + const label = process.env.RUNNER_LABEL; + const type = process.env.RUNNER_TYPE; + const skipInput = process.env.SKIP_DRY_RUN; + const files = fs.readdirSync(process.cwd()).filter(f => f.startsWith('requirements') && f.endsWith('.json')); + let found = false; + for (const file of files) { + const json = JSON.parse(fs.readFileSync(file, 'utf8')); + if (json.runners && json.runners[key]) { + if (label) json.runners[key].runner_label = label; + if (type) json.runners[key].runner_type = type; + if (skipInput !== '') json.runners[key].skip_dry_run = skipInput === 'true'; + fs.writeFileSync(file, JSON.stringify(json, null, 2) + '\n'); + fs.appendFileSync(process.env.GITHUB_ENV, `REQ_FILE=${file}\n`); + found = true; + console.log(`Updated ${file}`); + break; + } + } + if (!found) { + throw new Error(`Runner ${key} not found`); + } + NODE + + - name: Commit changes + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add "$REQ_FILE" + git commit -m "chore: update runner runtime for ${{ inputs.runner_key }}" || echo "No changes to commit" + git push + + - name: Validate + run: | + npm run derive:registry + npm run check:traceability || true + actionlint diff --git a/artifacts/linux/requirements-summary.md b/artifacts/linux/requirements-summary.md index 9e7c3061..97a0ebeb 100644 --- a/artifacts/linux/requirements-summary.md +++ b/artifacts/linux/requirements-summary.md @@ -1,78 +1,12 @@ ### Requirement Summary | Requirement ID | Description | Owner | Total Tests | Passed | Failed | Skipped | Pass Rate (%) | | --- | --- | --- | --- | --- | --- | --- | --- | -| REQ-023 | Parser ingests JUnit XML artifacts starting at the testsuites root and iterating through nested suites and testcases. | | 1 | 1 | 0 | 0 | 100.00 | -| REQ-024 | Top-level testsuites attributes name, tests, errors, failures, disabled, and time are captured for summary reporting. | | 1 | 1 | 0 | 0 | 100.00 | -| REQ-025 | Each testsuite records attributes including name, tests, errors, failures, hostname, id, skipped, disabled, package, and time. | | 1 | 1 | 0 | 0 | 100.00 | -| REQ-026 | Suite properties are extracted as name/value pairs for environment details. | | 1 | 1 | 0 | 0 | 100.00 | -| REQ-027 | Testcase attributes name, status, classname, assertions, time, and any skip message are preserved. | | 1 | 1 | 0 | 0 | 100.00 | -| REQ-028 | Requirement identifiers embedded in testcase names are detected and associated with the test. | | 1 | 1 | 0 | 0 | 100.00 | -| REQ-029 | Test results are aggregated by requirement and by suite to count passed, failed, and skipped cases. | | 1 | 1 | 0 | 0 | 100.00 | -| REQ-030 | Traceability matrix links requirement IDs to testcases with status, execution time, host properties, and skipped reasons. | | 1 | 1 | 0 | 0 | 100.00 | -| REQ-031 | Parsing logic validates presence of required fields and reports missing or malformed data. | | 1 | 1 | 0 | 0 | 100.00 | -| REQ-032 | Parser tolerates and retains unknown attributes for future extensibility. | | 1 | 1 | 0 | 0 | 100.00 | -| REQ-033 | Tests ending with SelfHosted.Workflow.Tests.ps1 execute only in dry run mode unless the workflow targets a self-hosted Windows runner labeled self-hosted-windows-lv. | | 1 | 1 | 0 | 0 | 100.00 | -| Unmapped | | | 48 | 48 | 0 | 0 | 100.00 | +| Unmapped | | | 4 | 4 | 0 | 0 | 100.00 | ### Requirement Testcases | Requirement ID | Test ID | Status | | --- | --- | --- | -| REQ-023 | parses-nested-junit-structures | Passed | -| REQ-024 | captures-root-testsuites-attributes | Passed | -| REQ-025 | captures-testsuite-attributes | Passed | -| REQ-026 | captures-suite-properties | Passed | -| REQ-027 | captures-testcase-attributes-and-skipped-message | Passed | -| REQ-028 | extracts-requirement-identifiers | Passed | -| REQ-029 | aggregates-status-by-requirement-and-suite | Passed | -| REQ-030 | builds-traceability-matrix-with-skipped-reasons | Passed | -| REQ-031 | validates-missing-fields | Passed | -| REQ-032 | preserves-unknown-attributes | Passed | -| REQ-033 | throws-error-for-malformed-xml | Passed | -| Unmapped | associates-classname-with-requirement | Passed | -| Unmapped | buildissuebranchname-formats-branch-name | Passed | -| Unmapped | buildissuebranchname-rejects-non-numeric-input | Passed | | Unmapped | buildprofile1.iconeditor.addtokentolabview.dispatcher.dry-runs-add-token-to-labview-with-expected-arguments- | Passed | | Unmapped | buildprofile1.iconeditor.applyvipc.dispatcher.dry-runs-apply-vipc-with-expected-arguments- | Passed | | Unmapped | buildprofile1.iconeditor.buildvipackage.dispatcher.dry-runs-build-vi-package-with-expected-arguments- | Passed | -| Unmapped | buildprofile1.iconeditor.closelabview.dispatcher.dry-runs-close-labview-with-expected-arguments- | Passed | -| Unmapped | buildsummary-splits-totals-by-os | Passed | -| Unmapped | collecttestcases-captures-requirement-property | Passed | -| Unmapped | collecttestcases-uses-evidence-property-and-falls-back-to-directory-scan | Passed | -| Unmapped | collecttestcases-uses-machine-name-property-for-owner | Passed | -| Unmapped | computestatuscounts-tallies-test-statuses | Passed | -| Unmapped | detects-downloaded-artifacts-path | Passed | -| Unmapped | dispatchers-and-parameters-include-descriptions | Passed | -| Unmapped | errors-when-strict-unmapped-mode-enabled | Passed | -| Unmapped | escapemarkdown-escapes-special-characters | Passed | -| Unmapped | escapemarkdown-leaves-plain-text-untouched | Passed | -| Unmapped | fails-when-commit-lacks-requirement-reference | Passed | -| Unmapped | fails-when-requirement-lacks-test-coverage | Passed | -| Unmapped | fails-when-tests-are-unmapped | Passed | -| Unmapped | fails-when-tests-reference-unknown-requirements | Passed | -| Unmapped | formaterror-handles-plain-objects | Passed | -| Unmapped | formaterror-handles-primitives | Passed | -| Unmapped | formaterror-handles-real-error-objects | Passed | -| Unmapped | formaterror-handles-unstringifiable-values | Passed | -| Unmapped | generate-ci-summary-features | Passed | -| Unmapped | groups-owners-and-includes-requirements-and-evidence | Passed | -| Unmapped | grouptomarkdown-omits-numeric-identifiers | Passed | -| Unmapped | grouptomarkdown-supports-optional-limit-for-truncation | Passed | -| Unmapped | handles-root-level-testcases | Passed | -| Unmapped | handles-zipped-junit-artifacts | Passed | -| Unmapped | ignores-stale-junit-files-outside-artifacts-path | Passed | -| Unmapped | loadrequirements-logs-warning-on-invalid-json | Passed | -| Unmapped | loadrequirements-merges-multiple-files | Passed | -| Unmapped | loadrequirements-warns-and-skips-invalid-entries | Passed | -| Unmapped | logs-a-warning-when-no-junit-files-are-found | Passed | -| Unmapped | partitions-requirement-groups-by-runner\_type | Passed | -| Unmapped | passes-with-coverage-and-requirement-reference | Passed | -| Unmapped | requirementssummarytomarkdown-escapes-pipes-in-description | Passed | -| Unmapped | skips-invalid-junit-files-and-still-generates-summary | Passed | -| Unmapped | summarytomarkdown-handles-no-tests | Passed | -| Unmapped | summarytomarkdown-sorts-os-alphabetically-and-escapes-special-characters | Passed | -| Unmapped | throws-when-no-junit-files-found-and-strict-mode-enabled | Passed | -| Unmapped | uses-latest-artifact-directory-when-multiple-are-present | Passed | -| Unmapped | warns-when-all-tests-are-unmapped | Passed | -| Unmapped | writeerrorsummary-appends-error-details-to-summary-file | Passed | -| Unmapped | writeerrorsummary-skips-summary-file-for-non-error-throws | Passed | -| Unmapped | writes-outputs-to-os-specific-directory | Passed | \ No newline at end of file +| Unmapped | buildprofile1.iconeditor.closelabview.dispatcher.dry-runs-close-labview-with-expected-arguments- | Passed | \ No newline at end of file diff --git a/artifacts/linux/summary-standard.md b/artifacts/linux/summary-standard.md index 1a1c6622..cfdde583 100644 --- a/artifacts/linux/summary-standard.md +++ b/artifacts/linux/summary-standard.md @@ -1,7 +1,7 @@ ### Test Summary | OS | Passed | Failed | Skipped | Duration (s) | Pass Rate (%) | | --- | --- | --- | --- | --- | --- | -| overall | 59 | 0 | 0 | 21.58 | 100.00 | -| linux | 59 | 0 | 0 | 21.58 | 100.00 | +| overall | 4 | 0 | 0 | 0.67 | 100.00 | +| linux | 4 | 0 | 0 | 0.67 | 100.00 | _For detailed per-test information, see [traceability-standard.md](traceability-standard.md)._ \ No newline at end of file diff --git a/artifacts/linux/summary.md b/artifacts/linux/summary.md index 615eb452..8b9ea053 100644 --- a/artifacts/linux/summary.md +++ b/artifacts/linux/summary.md @@ -1,23 +1,12 @@ ### Test Summary | OS | Passed | Failed | Skipped | Duration (s) | Pass Rate (%) | | --- | --- | --- | --- | --- | --- | -| overall | 59 | 0 | 0 | 21.58 | 100.00 | -| linux | 59 | 0 | 0 | 21.58 | 100.00 | +| overall | 4 | 0 | 0 | 0.67 | 100.00 | +| linux | 4 | 0 | 0 | 0.67 | 100.00 | ### Requirement Summary | Requirement ID | Description | Owner | Total Tests | Passed | Failed | Skipped | Pass Rate (%) | | --- | --- | --- | --- | --- | --- | --- | --- | -| REQ-023 | Parser ingests JUnit XML artifacts starting at the testsuites root and iterating through nested suites and testcases. | | 1 | 1 | 0 | 0 | 100.00 | -| REQ-024 | Top-level testsuites attributes name, tests, errors, failures, disabled, and time are captured for summary reporting. | | 1 | 1 | 0 | 0 | 100.00 | -| REQ-025 | Each testsuite records attributes including name, tests, errors, failures, hostname, id, skipped, disabled, package, and time. | | 1 | 1 | 0 | 0 | 100.00 | -| REQ-026 | Suite properties are extracted as name/value pairs for environment details. | | 1 | 1 | 0 | 0 | 100.00 | -| REQ-027 | Testcase attributes name, status, classname, assertions, time, and any skip message are preserved. | | 1 | 1 | 0 | 0 | 100.00 | -| REQ-028 | Requirement identifiers embedded in testcase names are detected and associated with the test. | | 1 | 1 | 0 | 0 | 100.00 | -| REQ-029 | Test results are aggregated by requirement and by suite to count passed, failed, and skipped cases. | | 1 | 1 | 0 | 0 | 100.00 | -| REQ-030 | Traceability matrix links requirement IDs to testcases with status, execution time, host properties, and skipped reasons. | | 1 | 1 | 0 | 0 | 100.00 | -| REQ-031 | Parsing logic validates presence of required fields and reports missing or malformed data. | | 1 | 1 | 0 | 0 | 100.00 | -| REQ-032 | Parser tolerates and retains unknown attributes for future extensibility. | | 1 | 1 | 0 | 0 | 100.00 | -| REQ-033 | Tests ending with SelfHosted.Workflow.Tests.ps1 execute only in dry run mode unless the workflow targets a self-hosted Windows runner labeled self-hosted-windows-lv. | | 1 | 1 | 0 | 0 | 100.00 | -| Unmapped | | | 48 | 48 | 0 | 0 | 100.00 | +| Unmapped | | | 4 | 4 | 0 | 0 | 100.00 | _For detailed per-test information, see [traceability.md](traceability.md)._ \ No newline at end of file diff --git a/artifacts/linux/traceability-standard.md b/artifacts/linux/traceability-standard.md index c8e679a4..d46bd1ae 100644 --- a/artifacts/linux/traceability-standard.md +++ b/artifacts/linux/traceability-standard.md @@ -1,122 +1,10 @@ ### Test Traceability Matrix -#### REQ-023 (100% passed) +#### Unmapped (100% passed) | Requirement | Test ID | Status | Duration (s) | Owner | Evidence | | --- | --- | --- | --- | --- | --- | -| REQ-023 | parses-nested-junit-structures | Passed | 0.023 | | | - -#### REQ-024 (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| REQ-024 | captures-root-testsuites-attributes | Passed | 0.007 | | | - -#### REQ-025 (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| REQ-025 | captures-testsuite-attributes | Passed | 0.012 | | | - -#### REQ-026 (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| REQ-026 | captures-suite-properties | Passed | 0.001 | | | - -#### REQ-027 (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| REQ-027 | captures-testcase-attributes-and-skipped-message | Passed | 0.001 | | | - -#### REQ-028 (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| REQ-028 | extracts-requirement-identifiers | Passed | 0.011 | | | - -#### REQ-029 (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| REQ-029 | aggregates-status-by-requirement-and-suite | Passed | 0.002 | | | - -#### REQ-030 (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| REQ-030 | builds-traceability-matrix-with-skipped-reasons | Passed | 0.002 | | | - -#### REQ-031 (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| REQ-031 | validates-missing-fields | Passed | 0.001 | | | - -#### REQ-032 (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| REQ-032 | preserves-unknown-attributes | Passed | 0.001 | | | - -#### REQ-033 (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| REQ-033 | throws-error-for-malformed-xml | Passed | 0.003 | | | - -
Unmapped (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| Unmapped | associates-classname-with-requirement | Passed | 0.028 | | | -| Unmapped | buildissuebranchname-formats-branch-name | Passed | 0.001 | | | -| Unmapped | buildissuebranchname-rejects-non-numeric-input | Passed | 0.002 | | | | Unmapped | buildprofile1.iconeditor.addtokentolabview.dispatcher.dry-runs-add-token-to-labview-with-expected-arguments- | Passed | 0.482 | | | | Unmapped | buildprofile1.iconeditor.applyvipc.dispatcher.dry-runs-apply-vipc-with-expected-arguments- | Passed | 0.042 | | | | Unmapped | buildprofile1.iconeditor.buildvipackage.dispatcher.dry-runs-build-vi-package-with-expected-arguments- | Passed | 0.106 | | | -| Unmapped | buildprofile1.iconeditor.closelabview.dispatcher.dry-runs-close-labview-with-expected-arguments- | Passed | 0.036 | | | -| Unmapped | buildsummary-splits-totals-by-os | Passed | 0.002 | | | -| Unmapped | collecttestcases-captures-requirement-property | Passed | 0.017 | | | -| Unmapped | collecttestcases-uses-evidence-property-and-falls-back-to-directory-scan | Passed | 0.015 | | | -| Unmapped | collecttestcases-uses-machine-name-property-for-owner | Passed | 0.023 | | | -| Unmapped | computestatuscounts-tallies-test-statuses | Passed | 0.000 | | | -| Unmapped | detects-downloaded-artifacts-path | Passed | 1.200 | | | -| Unmapped | dispatchers-and-parameters-include-descriptions | Passed | 0.009 | | | -| Unmapped | errors-when-strict-unmapped-mode-enabled | Passed | 1.251 | | | -| Unmapped | escapemarkdown-escapes-special-characters | Passed | 0.001 | | | -| Unmapped | escapemarkdown-leaves-plain-text-untouched | Passed | 0.000 | | | -| Unmapped | fails-when-commit-lacks-requirement-reference | Passed | 1.259 | | | -| Unmapped | fails-when-requirement-lacks-test-coverage | Passed | 1.342 | | | -| Unmapped | fails-when-tests-are-unmapped | Passed | 1.237 | | | -| Unmapped | fails-when-tests-reference-unknown-requirements | Passed | 1.214 | | | -| Unmapped | formaterror-handles-plain-objects | Passed | 0.000 | | | -| Unmapped | formaterror-handles-primitives | Passed | 0.000 | | | -| Unmapped | formaterror-handles-real-error-objects | Passed | 0.005 | | | -| Unmapped | formaterror-handles-unstringifiable-values | Passed | 0.001 | | | -| Unmapped | generate-ci-summary-features | Passed | 0.039 | | | -| Unmapped | groups-owners-and-includes-requirements-and-evidence | Passed | 1.376 | | | -| Unmapped | grouptomarkdown-omits-numeric-identifiers | Passed | 0.002 | | | -| Unmapped | grouptomarkdown-supports-optional-limit-for-truncation | Passed | 0.001 | | | -| Unmapped | handles-root-level-testcases | Passed | 0.001 | | | -| Unmapped | handles-zipped-junit-artifacts | Passed | 0.941 | | | -| Unmapped | ignores-stale-junit-files-outside-artifacts-path | Passed | 1.003 | | | -| Unmapped | loadrequirements-logs-warning-on-invalid-json | Passed | 0.013 | | | -| Unmapped | loadrequirements-merges-multiple-files | Passed | 0.014 | | | -| Unmapped | loadrequirements-warns-and-skips-invalid-entries | Passed | 0.004 | | | -| Unmapped | logs-a-warning-when-no-junit-files-are-found | Passed | 1.165 | | | -| Unmapped | partitions-requirement-groups-by-runner\_type | Passed | 0.966 | | | -| Unmapped | passes-with-coverage-and-requirement-reference | Passed | 1.244 | | | -| Unmapped | requirementssummarytomarkdown-escapes-pipes-in-description | Passed | 0.001 | | | -| Unmapped | skips-invalid-junit-files-and-still-generates-summary | Passed | 1.497 | | | -| Unmapped | summarytomarkdown-handles-no-tests | Passed | 0.001 | | | -| Unmapped | summarytomarkdown-sorts-os-alphabetically-and-escapes-special-characters | Passed | 0.001 | | | -| Unmapped | throws-when-no-junit-files-found-and-strict-mode-enabled | Passed | 0.829 | | | -| Unmapped | uses-latest-artifact-directory-when-multiple-are-present | Passed | 0.989 | | | -| Unmapped | warns-when-all-tests-are-unmapped | Passed | 1.412 | | | -| Unmapped | writeerrorsummary-appends-error-details-to-summary-file | Passed | 0.007 | | | -| Unmapped | writeerrorsummary-skips-summary-file-for-non-error-throws | Passed | 0.006 | | | -| Unmapped | writes-outputs-to-os-specific-directory | Passed | 1.730 | | | - -
\ No newline at end of file +| Unmapped | buildprofile1.iconeditor.closelabview.dispatcher.dry-runs-close-labview-with-expected-arguments- | Passed | 0.036 | | | \ No newline at end of file diff --git a/artifacts/linux/traceability.json b/artifacts/linux/traceability.json index 76001ff1..7ded38d4 100644 --- a/artifacts/linux/traceability.json +++ b/artifacts/linux/traceability.json @@ -1,222 +1,8 @@ { "requirements": [ - { - "id": "REQ-023", - "description": "Parser ingests JUnit XML artifacts starting at the testsuites root and iterating through nested suites and testcases.", - "tests": [ - { - "id": "parses-nested-junit-structures", - "name": "[REQ-023] parses nested JUnit structures", - "className": "test", - "status": "Passed", - "duration": 0.022782, - "requirements": [ - "REQ-023" - ], - "os": "linux" - } - ] - }, - { - "id": "REQ-024", - "description": "Top-level testsuites attributes name, tests, errors, failures, disabled, and time are captured for summary reporting.", - "tests": [ - { - "id": "captures-root-testsuites-attributes", - "name": "[REQ-024] captures root testsuites attributes", - "className": "test", - "status": "Passed", - "duration": 0.006805, - "requirements": [ - "REQ-024" - ], - "os": "linux" - } - ] - }, - { - "id": "REQ-025", - "description": "Each testsuite records attributes including name, tests, errors, failures, hostname, id, skipped, disabled, package, and time.", - "tests": [ - { - "id": "captures-testsuite-attributes", - "name": "[REQ-025] captures testsuite attributes", - "className": "test", - "status": "Passed", - "duration": 0.012317, - "requirements": [ - "REQ-025" - ], - "os": "linux" - } - ] - }, - { - "id": "REQ-026", - "description": "Suite properties are extracted as name/value pairs for environment details.", - "tests": [ - { - "id": "captures-suite-properties", - "name": "[REQ-026] captures suite properties", - "className": "test", - "status": "Passed", - "duration": 0.001227, - "requirements": [ - "REQ-026" - ], - "os": "linux" - } - ] - }, - { - "id": "REQ-027", - "description": "Testcase attributes name, status, classname, assertions, time, and any skip message are preserved.", - "tests": [ - { - "id": "captures-testcase-attributes-and-skipped-message", - "name": "[REQ-027] captures testcase attributes and skipped message", - "className": "test", - "status": "Passed", - "duration": 0.001187, - "requirements": [ - "REQ-027" - ], - "os": "linux" - } - ] - }, - { - "id": "REQ-028", - "description": "Requirement identifiers embedded in testcase names are detected and associated with the test.", - "tests": [ - { - "id": "extracts-requirement-identifiers", - "name": "[REQ-028] extracts requirement identifiers", - "className": "test", - "status": "Passed", - "duration": 0.01091, - "requirements": [ - "REQ-028" - ], - "os": "linux" - } - ] - }, - { - "id": "REQ-029", - "description": "Test results are aggregated by requirement and by suite to count passed, failed, and skipped cases.", - "tests": [ - { - "id": "aggregates-status-by-requirement-and-suite", - "name": "[REQ-029] aggregates status by requirement and suite", - "className": "test", - "status": "Passed", - "duration": 0.001763, - "requirements": [ - "REQ-029" - ], - "os": "linux" - } - ] - }, - { - "id": "REQ-030", - "description": "Traceability matrix links requirement IDs to testcases with status, execution time, host properties, and skipped reasons.", - "tests": [ - { - "id": "builds-traceability-matrix-with-skipped-reasons", - "name": "[REQ-030] builds traceability matrix with skipped reasons", - "className": "test", - "status": "Passed", - "duration": 0.00203, - "requirements": [ - "REQ-030" - ], - "os": "linux" - } - ] - }, - { - "id": "REQ-031", - "description": "Parsing logic validates presence of required fields and reports missing or malformed data.", - "tests": [ - { - "id": "validates-missing-fields", - "name": "[REQ-031] validates missing fields", - "className": "test", - "status": "Passed", - "duration": 0.000859, - "requirements": [ - "REQ-031" - ], - "os": "linux" - } - ] - }, - { - "id": "REQ-032", - "description": "Parser tolerates and retains unknown attributes for future extensibility.", - "tests": [ - { - "id": "preserves-unknown-attributes", - "name": "[REQ-032] preserves unknown attributes", - "className": "test", - "status": "Passed", - "duration": 0.001374, - "requirements": [ - "REQ-032" - ], - "os": "linux" - } - ] - }, - { - "id": "REQ-033", - "description": "Tests ending with SelfHosted.Workflow.Tests.ps1 execute only in dry run mode unless the workflow targets a self-hosted Windows runner labeled self-hosted-windows-lv.", - "tests": [ - { - "id": "throws-error-for-malformed-xml", - "name": "[REQ-033] throws error for malformed XML", - "className": "test", - "status": "Passed", - "duration": 0.002535, - "requirements": [ - "REQ-033" - ], - "os": "linux" - } - ] - }, { "id": "Unmapped", "tests": [ - { - "id": "associates-classname-with-requirement", - "name": "associates classname with requirement", - "className": "test", - "status": "Passed", - "duration": 0.028064, - "requirements": [], - "os": "linux" - }, - { - "id": "buildissuebranchname-formats-branch-name", - "name": "buildIssueBranchName formats branch name", - "className": "test", - "status": "Passed", - "duration": 0.001407, - "requirements": [], - "os": "linux" - }, - { - "id": "buildissuebranchname-rejects-non-numeric-input", - "name": "buildIssueBranchName rejects non-numeric input", - "className": "test", - "status": "Passed", - "duration": 0.001573, - "requirements": [], - "os": "linux" - }, { "id": "buildprofile1.iconeditor.addtokentolabview.dispatcher.dry-runs-add-token-to-labview-with-expected-arguments-", "name": "BuildProfile1.IconEditor.AddTokenToLabview.Dispatcher.dry-runs add-token-to-labview with expected arguments [REQIE-001]", @@ -252,393 +38,24 @@ "duration": 0.036, "requirements": [], "os": "linux" - }, - { - "id": "buildsummary-splits-totals-by-os", - "name": "buildSummary splits totals by OS", - "className": "test", - "status": "Passed", - "duration": 0.001865, - "requirements": [], - "os": "linux" - }, - { - "id": "collecttestcases-captures-requirement-property", - "name": "collectTestCases captures requirement property", - "className": "test", - "status": "Passed", - "duration": 0.01681, - "requirements": [], - "os": "linux" - }, - { - "id": "collecttestcases-uses-evidence-property-and-falls-back-to-directory-scan", - "name": "collectTestCases uses evidence property and falls back to directory scan", - "className": "test", - "status": "Passed", - "duration": 0.014548, - "requirements": [], - "os": "linux" - }, - { - "id": "collecttestcases-uses-machine-name-property-for-owner", - "name": "collectTestCases uses machine-name property for owner", - "className": "test", - "status": "Passed", - "duration": 0.022855, - "requirements": [], - "os": "linux" - }, - { - "id": "computestatuscounts-tallies-test-statuses", - "name": "computeStatusCounts tallies test statuses", - "className": "test", - "status": "Passed", - "duration": 0.000413, - "requirements": [], - "os": "linux" - }, - { - "id": "detects-downloaded-artifacts-path", - "name": "detects downloaded artifacts path", - "className": "test", - "status": "Passed", - "duration": 1.200369, - "requirements": [], - "os": "linux" - }, - { - "id": "dispatchers-and-parameters-include-descriptions", - "name": "Dispatchers and parameters include descriptions", - "className": "test", - "status": "Passed", - "duration": 0.009128, - "requirements": [], - "os": "linux" - }, - { - "id": "errors-when-strict-unmapped-mode-enabled", - "name": "errors when strict unmapped mode enabled", - "className": "test", - "status": "Passed", - "duration": 1.251414, - "requirements": [], - "os": "linux" - }, - { - "id": "escapemarkdown-escapes-special-characters", - "name": "escapeMarkdown escapes special characters", - "className": "test", - "status": "Passed", - "duration": 0.001383, - "requirements": [], - "os": "linux" - }, - { - "id": "escapemarkdown-leaves-plain-text-untouched", - "name": "escapeMarkdown leaves plain text untouched", - "className": "test", - "status": "Passed", - "duration": 0.000206, - "requirements": [], - "os": "linux" - }, - { - "id": "fails-when-commit-lacks-requirement-reference", - "name": "fails when commit lacks requirement reference", - "className": "test", - "status": "Passed", - "duration": 1.258809, - "requirements": [], - "os": "linux" - }, - { - "id": "fails-when-requirement-lacks-test-coverage", - "name": "fails when requirement lacks test coverage", - "className": "test", - "status": "Passed", - "duration": 1.342315, - "requirements": [], - "os": "linux" - }, - { - "id": "fails-when-tests-are-unmapped", - "name": "fails when tests are unmapped", - "className": "test", - "status": "Passed", - "duration": 1.237183, - "requirements": [], - "os": "linux" - }, - { - "id": "fails-when-tests-reference-unknown-requirements", - "name": "fails when tests reference unknown requirements", - "className": "test", - "status": "Passed", - "duration": 1.213962, - "requirements": [], - "os": "linux" - }, - { - "id": "formaterror-handles-plain-objects", - "name": "formatError handles plain objects", - "className": "test", - "status": "Passed", - "duration": 0.000452, - "requirements": [], - "os": "linux" - }, - { - "id": "formaterror-handles-primitives", - "name": "formatError handles primitives", - "className": "test", - "status": "Passed", - "duration": 0.000404, - "requirements": [], - "os": "linux" - }, - { - "id": "formaterror-handles-real-error-objects", - "name": "formatError handles real Error objects", - "className": "test", - "status": "Passed", - "duration": 0.004599, - "requirements": [], - "os": "linux" - }, - { - "id": "formaterror-handles-unstringifiable-values", - "name": "formatError handles unstringifiable values", - "className": "test", - "status": "Passed", - "duration": 0.001304, - "requirements": [], - "os": "linux" - }, - { - "id": "generate-ci-summary-features", - "name": "generate-ci-summary features", - "className": "test", - "status": "Passed", - "duration": 0.038685, - "requirements": [], - "os": "linux" - }, - { - "id": "groups-owners-and-includes-requirements-and-evidence", - "name": "groups owners and includes requirements and evidence", - "className": "test", - "status": "Passed", - "duration": 1.376147, - "requirements": [], - "os": "linux" - }, - { - "id": "grouptomarkdown-omits-numeric-identifiers", - "name": "groupToMarkdown omits numeric identifiers", - "className": "test", - "status": "Passed", - "duration": 0.002115, - "requirements": [], - "os": "linux" - }, - { - "id": "grouptomarkdown-supports-optional-limit-for-truncation", - "name": "groupToMarkdown supports optional limit for truncation", - "className": "test", - "status": "Passed", - "duration": 0.00085, - "requirements": [], - "os": "linux" - }, - { - "id": "handles-root-level-testcases", - "name": "handles root-level testcases", - "className": "test", - "status": "Passed", - "duration": 0.001112, - "requirements": [], - "os": "linux" - }, - { - "id": "handles-zipped-junit-artifacts", - "name": "handles zipped JUnit artifacts", - "className": "test", - "status": "Passed", - "duration": 0.941473, - "requirements": [], - "os": "linux" - }, - { - "id": "ignores-stale-junit-files-outside-artifacts-path", - "name": "ignores stale JUnit files outside artifacts path", - "className": "test", - "status": "Passed", - "duration": 1.00302, - "requirements": [], - "os": "linux" - }, - { - "id": "loadrequirements-logs-warning-on-invalid-json", - "name": "loadRequirements logs warning on invalid JSON", - "className": "test", - "status": "Passed", - "duration": 0.013044, - "requirements": [], - "os": "linux" - }, - { - "id": "loadrequirements-merges-multiple-files", - "name": "loadRequirements merges multiple files", - "className": "test", - "status": "Passed", - "duration": 0.014155, - "requirements": [], - "os": "linux" - }, - { - "id": "loadrequirements-warns-and-skips-invalid-entries", - "name": "loadRequirements warns and skips invalid entries", - "className": "test", - "status": "Passed", - "duration": 0.004424, - "requirements": [], - "os": "linux" - }, - { - "id": "logs-a-warning-when-no-junit-files-are-found", - "name": "logs a warning when no JUnit files are found", - "className": "test", - "status": "Passed", - "duration": 1.16546, - "requirements": [], - "os": "linux" - }, - { - "id": "partitions-requirement-groups-by-runner_type", - "name": "partitions requirement groups by runner_type", - "className": "test", - "status": "Passed", - "duration": 0.966158, - "requirements": [], - "os": "linux" - }, - { - "id": "passes-with-coverage-and-requirement-reference", - "name": "passes with coverage and requirement reference", - "className": "test", - "status": "Passed", - "duration": 1.243643, - "requirements": [], - "os": "linux" - }, - { - "id": "requirementssummarytomarkdown-escapes-pipes-in-description", - "name": "requirementsSummaryToMarkdown escapes pipes in description", - "className": "test", - "status": "Passed", - "duration": 0.000636, - "requirements": [], - "os": "linux" - }, - { - "id": "skips-invalid-junit-files-and-still-generates-summary", - "name": "skips invalid JUnit files and still generates summary", - "className": "test", - "status": "Passed", - "duration": 1.496572, - "requirements": [], - "os": "linux" - }, - { - "id": "summarytomarkdown-handles-no-tests", - "name": "summaryToMarkdown handles no tests", - "className": "test", - "status": "Passed", - "duration": 0.000893, - "requirements": [], - "os": "linux" - }, - { - "id": "summarytomarkdown-sorts-os-alphabetically-and-escapes-special-characters", - "name": "summaryToMarkdown sorts OS alphabetically and escapes special characters", - "className": "test", - "status": "Passed", - "duration": 0.001172, - "requirements": [], - "os": "linux" - }, - { - "id": "throws-when-no-junit-files-found-and-strict-mode-enabled", - "name": "throws when no JUnit files found and strict mode enabled", - "className": "test", - "status": "Passed", - "duration": 0.829463, - "requirements": [], - "os": "linux" - }, - { - "id": "uses-latest-artifact-directory-when-multiple-are-present", - "name": "uses latest artifact directory when multiple are present", - "className": "test", - "status": "Passed", - "duration": 0.988567, - "requirements": [], - "os": "linux" - }, - { - "id": "warns-when-all-tests-are-unmapped", - "name": "warns when all tests are unmapped", - "className": "test", - "status": "Passed", - "duration": 1.412212, - "requirements": [], - "os": "linux" - }, - { - "id": "writeerrorsummary-appends-error-details-to-summary-file", - "name": "writeErrorSummary appends error details to summary file", - "className": "test", - "status": "Passed", - "duration": 0.006686, - "requirements": [], - "os": "linux" - }, - { - "id": "writeerrorsummary-skips-summary-file-for-non-error-throws", - "name": "writeErrorSummary skips summary file for non-Error throws", - "className": "test", - "status": "Passed", - "duration": 0.006355, - "requirements": [], - "os": "linux" - }, - { - "id": "writes-outputs-to-os-specific-directory", - "name": "writes outputs to OS-specific directory", - "className": "test", - "status": "Passed", - "duration": 1.729754, - "requirements": [], - "os": "linux" } ] } ], "totals": { "overall": { - "passed": 59, + "passed": 4, "failed": 0, "skipped": 0, - "duration": 21.581447999999998, + "duration": 0.666, "rate": 100 }, "byOs": { "linux": { - "passed": 59, + "passed": 4, "failed": 0, "skipped": 0, - "duration": 21.581447999999998, + "duration": 0.666, "rate": 100 } } diff --git a/artifacts/linux/traceability.md b/artifacts/linux/traceability.md index c8e679a4..d46bd1ae 100644 --- a/artifacts/linux/traceability.md +++ b/artifacts/linux/traceability.md @@ -1,122 +1,10 @@ ### Test Traceability Matrix -#### REQ-023 (100% passed) +#### Unmapped (100% passed) | Requirement | Test ID | Status | Duration (s) | Owner | Evidence | | --- | --- | --- | --- | --- | --- | -| REQ-023 | parses-nested-junit-structures | Passed | 0.023 | | | - -#### REQ-024 (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| REQ-024 | captures-root-testsuites-attributes | Passed | 0.007 | | | - -#### REQ-025 (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| REQ-025 | captures-testsuite-attributes | Passed | 0.012 | | | - -#### REQ-026 (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| REQ-026 | captures-suite-properties | Passed | 0.001 | | | - -#### REQ-027 (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| REQ-027 | captures-testcase-attributes-and-skipped-message | Passed | 0.001 | | | - -#### REQ-028 (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| REQ-028 | extracts-requirement-identifiers | Passed | 0.011 | | | - -#### REQ-029 (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| REQ-029 | aggregates-status-by-requirement-and-suite | Passed | 0.002 | | | - -#### REQ-030 (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| REQ-030 | builds-traceability-matrix-with-skipped-reasons | Passed | 0.002 | | | - -#### REQ-031 (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| REQ-031 | validates-missing-fields | Passed | 0.001 | | | - -#### REQ-032 (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| REQ-032 | preserves-unknown-attributes | Passed | 0.001 | | | - -#### REQ-033 (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| REQ-033 | throws-error-for-malformed-xml | Passed | 0.003 | | | - -
Unmapped (100% passed) - -| Requirement | Test ID | Status | Duration (s) | Owner | Evidence | -| --- | --- | --- | --- | --- | --- | -| Unmapped | associates-classname-with-requirement | Passed | 0.028 | | | -| Unmapped | buildissuebranchname-formats-branch-name | Passed | 0.001 | | | -| Unmapped | buildissuebranchname-rejects-non-numeric-input | Passed | 0.002 | | | | Unmapped | buildprofile1.iconeditor.addtokentolabview.dispatcher.dry-runs-add-token-to-labview-with-expected-arguments- | Passed | 0.482 | | | | Unmapped | buildprofile1.iconeditor.applyvipc.dispatcher.dry-runs-apply-vipc-with-expected-arguments- | Passed | 0.042 | | | | Unmapped | buildprofile1.iconeditor.buildvipackage.dispatcher.dry-runs-build-vi-package-with-expected-arguments- | Passed | 0.106 | | | -| Unmapped | buildprofile1.iconeditor.closelabview.dispatcher.dry-runs-close-labview-with-expected-arguments- | Passed | 0.036 | | | -| Unmapped | buildsummary-splits-totals-by-os | Passed | 0.002 | | | -| Unmapped | collecttestcases-captures-requirement-property | Passed | 0.017 | | | -| Unmapped | collecttestcases-uses-evidence-property-and-falls-back-to-directory-scan | Passed | 0.015 | | | -| Unmapped | collecttestcases-uses-machine-name-property-for-owner | Passed | 0.023 | | | -| Unmapped | computestatuscounts-tallies-test-statuses | Passed | 0.000 | | | -| Unmapped | detects-downloaded-artifacts-path | Passed | 1.200 | | | -| Unmapped | dispatchers-and-parameters-include-descriptions | Passed | 0.009 | | | -| Unmapped | errors-when-strict-unmapped-mode-enabled | Passed | 1.251 | | | -| Unmapped | escapemarkdown-escapes-special-characters | Passed | 0.001 | | | -| Unmapped | escapemarkdown-leaves-plain-text-untouched | Passed | 0.000 | | | -| Unmapped | fails-when-commit-lacks-requirement-reference | Passed | 1.259 | | | -| Unmapped | fails-when-requirement-lacks-test-coverage | Passed | 1.342 | | | -| Unmapped | fails-when-tests-are-unmapped | Passed | 1.237 | | | -| Unmapped | fails-when-tests-reference-unknown-requirements | Passed | 1.214 | | | -| Unmapped | formaterror-handles-plain-objects | Passed | 0.000 | | | -| Unmapped | formaterror-handles-primitives | Passed | 0.000 | | | -| Unmapped | formaterror-handles-real-error-objects | Passed | 0.005 | | | -| Unmapped | formaterror-handles-unstringifiable-values | Passed | 0.001 | | | -| Unmapped | generate-ci-summary-features | Passed | 0.039 | | | -| Unmapped | groups-owners-and-includes-requirements-and-evidence | Passed | 1.376 | | | -| Unmapped | grouptomarkdown-omits-numeric-identifiers | Passed | 0.002 | | | -| Unmapped | grouptomarkdown-supports-optional-limit-for-truncation | Passed | 0.001 | | | -| Unmapped | handles-root-level-testcases | Passed | 0.001 | | | -| Unmapped | handles-zipped-junit-artifacts | Passed | 0.941 | | | -| Unmapped | ignores-stale-junit-files-outside-artifacts-path | Passed | 1.003 | | | -| Unmapped | loadrequirements-logs-warning-on-invalid-json | Passed | 0.013 | | | -| Unmapped | loadrequirements-merges-multiple-files | Passed | 0.014 | | | -| Unmapped | loadrequirements-warns-and-skips-invalid-entries | Passed | 0.004 | | | -| Unmapped | logs-a-warning-when-no-junit-files-are-found | Passed | 1.165 | | | -| Unmapped | partitions-requirement-groups-by-runner\_type | Passed | 0.966 | | | -| Unmapped | passes-with-coverage-and-requirement-reference | Passed | 1.244 | | | -| Unmapped | requirementssummarytomarkdown-escapes-pipes-in-description | Passed | 0.001 | | | -| Unmapped | skips-invalid-junit-files-and-still-generates-summary | Passed | 1.497 | | | -| Unmapped | summarytomarkdown-handles-no-tests | Passed | 0.001 | | | -| Unmapped | summarytomarkdown-sorts-os-alphabetically-and-escapes-special-characters | Passed | 0.001 | | | -| Unmapped | throws-when-no-junit-files-found-and-strict-mode-enabled | Passed | 0.829 | | | -| Unmapped | uses-latest-artifact-directory-when-multiple-are-present | Passed | 0.989 | | | -| Unmapped | warns-when-all-tests-are-unmapped | Passed | 1.412 | | | -| Unmapped | writeerrorsummary-appends-error-details-to-summary-file | Passed | 0.007 | | | -| Unmapped | writeerrorsummary-skips-summary-file-for-non-error-throws | Passed | 0.006 | | | -| Unmapped | writes-outputs-to-os-specific-directory | Passed | 1.730 | | | - -
\ No newline at end of file +| Unmapped | buildprofile1.iconeditor.closelabview.dispatcher.dry-runs-close-labview-with-expected-arguments- | Passed | 0.036 | | | \ No newline at end of file diff --git a/ci_evidence.txt b/ci_evidence.txt index f0ff99b1..ce05fb35 100644 --- a/ci_evidence.txt +++ b/ci_evidence.txt @@ -1 +1 @@ -{"pipeline":"Unknown","git_sha":"a4ddc5334d90f16cc8f3f0d60a7a84cd6e0bacba","req_status":{"REQ-023":"PASS","REQ-024":"PASS","REQ-025":"PASS","REQ-026":"PASS","REQ-027":"PASS","REQ-028":"PASS","REQ-029":"PASS","REQ-030":"PASS","REQ-031":"PASS","REQ-032":"PASS","REQ-033":"PASS"}} \ No newline at end of file +{"pipeline":"Unknown","git_sha":"4a4d08c05d6852d7aa75ba72c5fb1f39370560ad","req_status":{}} \ No newline at end of file diff --git a/docs/runner-types.md b/docs/runner-types.md index 94a6bd34..eadbef22 100644 --- a/docs/runner-types.md +++ b/docs/runner-types.md @@ -53,3 +53,14 @@ The summarizer partitions results by `runner_type`, producing artifacts such as 5. **Document expectations.** When adding new requirements or workflows, update `docs/requirements.md` so the `Runner` and `Runner Type` columns reflect the intended infrastructure. By explicitly classifying jobs, teams can scale the project efficiently—routine tasks remain fast on default runners while integration tests validate real‑world behavior without overloading scarce resources. + +## Updating runner metadata + +Administrators can modify runner mappings without opening a pull request by using the `update-runner-runtime` workflow. Trigger the workflow from the Actions tab and supply the following inputs: + +* `runner_key` – key of the runner in the requirements file. +* `runner_label` – label applied to the runner. +* `runner_type` – value for `runner_type` such as `standard` or `integration`. +* `skip_dry_run` – set to `true` to force real execution. + +The workflow updates the corresponding `requirements*.json` file, commits the change with `GITHUB_TOKEN`, and optionally runs registry derivation, traceability checks, and `actionlint` to validate the update. diff --git a/test-results/node-junit.xml b/test-results/node-junit.xml index 696508ea..32a8e32d 100644 --- a/test-results/node-junit.xml +++ b/test-results/node-junit.xml @@ -1,66 +1,2 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -