From 505cfa967777ee4b57fa0cc39e0182e079568e35 Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Wed, 5 Aug 2026 20:22:57 +0200 Subject: [PATCH] feat: integrate Kubernetes Launch Kit validation Add a generic Launch Kit provider and six Network Operator east-west networking use cases backed by discover, generate, deploy, validate, and cleanup workflows. Preserve Launch Kit evidence while exposing reusable semantic checks and use-case-level reporting. Consume the latest Launch Kit GPUDirect DMA-BUF result family with endpoint GPU, PCI, bandwidth, and threshold diagnostics. Register K8S42-07 globally and compose it into every use case with output-driven applicability. Extend orchestration and reporting with named phases, validation-aware lifecycle pruning, linked finalizers, structured composite subtests, process-group timeouts, recursive suite discovery, and accurate JUnit failures for command-stage errors. Document the provider contract, prerequisites, catalog metadata, PRD coverage, and remaining integration gaps. Signed-off-by: Alexander Maslennikov --- AGENTS.md | 127 +- docs/README.md | 1 + docs/guides/configuration.md | 213 ++- .../guides/k8s-launch-kit/network-operator.md | 752 ++++++++++ docs/packages/isvctl.md | 21 + docs/packages/isvtest.md | 19 + docs/requirements/README.md | 19 +- ...network-operator-readiness-requirements.md | 51 + ...twork-operator-readiness-requirements.yaml | 83 ++ .../test-requirements-matrix.adoc | 540 ++++++++ .../test-requirements-matrix.yaml | 262 +++- docs/test-plan.adoc | 312 ++++- docs/test-plan.yaml | 330 +++++ .../providers/k8s-launch-kit/README.md | 89 ++ .../config/network-operator.yaml | 992 ++++++++++++++ .../k8s-launch-kit/config/provider.yaml | 219 +++ .../k8s-launch-kit/scripts/adapter.py | 722 ++++++++++ isvctl/configs/suites/README.md | 72 +- .../network-operator-use-cases.yaml | 187 +++ .../k8s-launch-kit/network-operator.yaml | 122 ++ isvctl/src/isvctl/cli/test.py | 79 +- isvctl/src/isvctl/config/output_schemas.py | 44 + isvctl/src/isvctl/config/schema.py | 86 +- isvctl/src/isvctl/config/suite_resolution.py | 7 +- isvctl/src/isvctl/doctor/checks/config.py | 2 +- isvctl/src/isvctl/orchestrator/commands.py | 5 +- isvctl/src/isvctl/orchestrator/loop.py | 250 +++- isvctl/src/isvctl/orchestrator/process.py | 105 ++ .../src/isvctl/orchestrator/step_executor.py | 10 +- .../providers/k8s_launch_kit/__init__.py | 4 + .../fixtures/launch_kit_scenarios.json | 104 ++ .../k8s_launch_kit/fixtures/mock_kubectl.py | 61 + .../k8s_launch_kit/fixtures/mock_l8k.py | 706 ++++++++++ .../providers/k8s_launch_kit/test_provider.py | 1220 +++++++++++++++++ .../k8s_launch_kit/test_timeout_config.py | 33 + isvctl/tests/test_orchestrator_loop.py | 411 ++++++ isvctl/tests/test_orchestrator_process.py | 102 ++ isvctl/tests/test_schema.py | 103 ++ isvctl/tests/test_stub_contracts.py | 2 +- isvctl/tests/test_suite_resolution.py | 15 + isvctl/tests/test_test_cli_labels.py | 103 ++ isvtest/src/isvtest/catalog.py | 4 +- isvtest/src/isvtest/core/composite.py | 20 +- isvtest/src/isvtest/core/resolution.py | 145 +- isvtest/src/isvtest/main.py | 12 +- isvtest/src/isvtest/testing/subtests.py | 26 +- isvtest/src/isvtest/tests/test_validations.py | 12 + .../validations/k8s_launch_kit/__init__.py | 4 + .../validations/k8s_launch_kit/checks.py | 719 ++++++++++ isvtest/tests/k8s_launch_kit/test_checks.py | 521 +++++++ isvtest/tests/test_catalog.py | 3 + isvtest/tests/test_composite.py | 48 + isvtest/tests/test_main.py | 22 + isvtest/tests/test_subtests_junit.py | 22 +- isvtest/tests/test_validation.py | 20 + scripts/requirements_source_to_md.py | 48 +- scripts/test_plan_coverage.py | 6 +- .../tests/test_requirements_source_to_md.py | 67 + scripts/tests/test_validate_suite_wiring.py | 13 + scripts/validate_suite_wiring.py | 4 +- 60 files changed, 10124 insertions(+), 177 deletions(-) create mode 100644 docs/guides/k8s-launch-kit/network-operator.md create mode 100644 docs/requirements/network-operator-readiness-requirements.md create mode 100644 docs/requirements/network-operator-readiness-requirements.yaml create mode 100644 isvctl/configs/providers/k8s-launch-kit/README.md create mode 100644 isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml create mode 100644 isvctl/configs/providers/k8s-launch-kit/config/provider.yaml create mode 100644 isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py create mode 100644 isvctl/configs/suites/k8s-launch-kit/network-operator-use-cases.yaml create mode 100644 isvctl/configs/suites/k8s-launch-kit/network-operator.yaml create mode 100644 isvctl/src/isvctl/orchestrator/process.py create mode 100644 isvctl/tests/providers/k8s_launch_kit/__init__.py create mode 100644 isvctl/tests/providers/k8s_launch_kit/fixtures/launch_kit_scenarios.json create mode 100755 isvctl/tests/providers/k8s_launch_kit/fixtures/mock_kubectl.py create mode 100755 isvctl/tests/providers/k8s_launch_kit/fixtures/mock_l8k.py create mode 100644 isvctl/tests/providers/k8s_launch_kit/test_provider.py create mode 100644 isvctl/tests/providers/k8s_launch_kit/test_timeout_config.py create mode 100644 isvctl/tests/test_orchestrator_process.py create mode 100644 isvtest/src/isvtest/validations/k8s_launch_kit/__init__.py create mode 100644 isvtest/src/isvtest/validations/k8s_launch_kit/checks.py create mode 100644 isvtest/tests/k8s_launch_kit/test_checks.py create mode 100644 scripts/tests/test_requirements_source_to_md.py diff --git a/AGENTS.md b/AGENTS.md index 229e33daa..673108371 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,7 +77,8 @@ Entry point: `isvctl/src/isvctl/main.py` (Typer). - `cli/` - subcommands (`test`, `deploy`, `clean`, `docs`, `report`) - `orchestrator/` - `loop.py` (phase loop), `step_executor.py` (step + validation - execution, supports `best_effort` mode), `commands.py` (timeouts), `context.py` + execution, supports `best_effort` mode), `commands.py` (legacy command model), + `process.py` (shared subprocess and process-group timeout handling), `context.py` (Jinja2 with missing-reference warnings) - `config/` - `schema.py` (Pydantic), `output_schemas.py` (per-step JSON schemas), `merger.py` (multi-file merge) @@ -149,7 +150,9 @@ forwarded env vars → optional isvreporter upload. - Workspace root `pyproject.toml` defines members; each package has its own `pyproject.toml`; all source under `src/`. -- `isvctl/configs/suites/` - provider-agnostic test contracts. +- `isvctl/configs/suites/` - provider-agnostic test contracts. Discovery is + recursive, so related domain suites may be grouped in a subdirectory; YAML + filename stems must remain globally unique. - `isvctl/configs/providers//` - one folder per provider (`aws/`, `my-isv/`, ...): - `config/` - YAML wiring (imports a suite, supplies commands) - `scripts/` - executable scripts (Python/Bash) that do the work, organized by @@ -170,6 +173,126 @@ forwarded env vars → optional isvreporter upload. `aws/scripts/common/` provides `ec2`, `errors` (with `delete_with_retry`), `ssh_utils.wait_for_ssh`, `serial_console`, `vpc`. +### Network Operator / Kubernetes Launch Kit + +- All provider-owned Launch Kit files live under + `isvctl/configs/providers/k8s-launch-kit/`: generic and Network Operator YAML + in `config/`, executable transport in `scripts/`, and provider documentation + in `README.md`. +- `isvctl/configs/providers/k8s-launch-kit/config/provider.yaml` is the generic provider. Its + public API mirrors the CLI: `prepare`, `verify`, Kubernetes preflight, + `discover`, `generate`, `deploy`, `validate`, and `clean`. Workflow settings are raw + argument arrays; do not model or duplicate Launch Kit flags/defaults here. + The single file-level input, `user_config`, points to a complete Launch Kit + configuration. Discovery copies it to `/user-config.yaml` and + writes the resolved result to `/cluster-config.yaml`, preserving + the source file and the default paths used by subsequent commands. + Its `validate` step uses `timeout: null` so l8k owns the automatically + calculated or user-supplied matrix deadline; all other workflow steps retain + finite outer isvctl watchdogs. Any provider may use a null `StepConfig` + timeout when its invoked command owns a bounded deadline. +- Launch Kit-specific transport code belongs under + `isvctl/configs/providers/k8s-launch-kit/`, not `providers/shared/` (which is + reserved for scripts reused by unrelated providers). Production code lives + in `scripts/`. Executable mocks and pinned fixtures are test-only and live in + `isvctl/tests/providers/k8s_launch_kit/fixtures/`; product configuration must + never reference them. +- `prepare` supports `verify` and explicit `install` modes. Install mode + downloads and records the official Launch Kit installer, then delegates + archive selection, checksum verification, and installation to it. Both modes + verify `l8k version --output json` and `l8k schema`. The configured string + environment is shared by install, verification, preflight, and workflows. +- The Kubernetes preflight is mandatory before each normal test use case. It + derives a single explicit kubeconfig from the raw l8k arguments (and rejects + conflicts), verifies API access, requires a non-empty node inventory, and + requires at least one Ready node. A failure stops the remaining steps in that + workflow/use case. An explicit teardown-only recovery intentionally invokes + `l8k clean` directly because cleanup must not depend on a passing test + prerequisite. +- `isvctl/configs/suites/k8s-launch-kit/network-operator.yaml` owns only catalog wiring and + interpretation for the globally selectable PRD checks. Each check binds to + the real step that produced its evidence. +- `isvctl/configs/suites/k8s-launch-kit/network-operator-use-cases.yaml` composes those shared + check classes into six concrete end-to-end tests: RoCE and InfiniBand across + SR-IOV, RDMA Shared, and host-device modes. Include only checks applicable to + a use case; do not run all checks and hide mismatches as interleaved skips. +- `isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml` is the + production six-use-case configuration. It defaults to real `l8k` and + `kubectl`, executes each supported fabric/deployment combination as a named + custom phase, and gives every use case isolated working and evidence + directories. Its fabric/deployment arguments define test identity; Launch + Kit continues to own runtime defaults and users extend raw argv in overlays. + A global `user_config` is staged independently for each selected use case; + when it is set, raw discovery arguments cannot also select user/save config paths. + Keep default config/deployment path flags out of all grouped phase arguments; + overlays may add them only when intentionally overriding Launch Kit's paths. +- Mock-backed coverage loads that same production YAML and injects test-owned + executables only in `isvctl/tests/providers/k8s_launch_kit/test_provider.py`. + Result-check tests live under `isvtest/tests/k8s_launch_kit/`. +- Independent use-case phases are listed in `continue_after_failure` so a failed + case does not suppress later evidence. The failed phase still fails the final + run after its linked teardown succeeds. Never use that option for shared + setup or dependent phases; a failed finalizer always blocks later phases. +- `StepConfig.finalizer_for` links cleanup to a mutating target. Provider + cleanup belongs in `phase: teardown`; the orchestrator runs it immediately + after the target phase validations and reports `-teardown`, + including for `--phase test`. It only activates when the target process + started, while `--phase teardown` runs it unconditionally as recovery. Use + this instead of unconditional cleanup when a preflight failure must not + delete pre-existing state. Schema validation requires matching capability + and validation-selection gates. +- Mutating steps associated with a selectable test declare + `requires_selected_validations`. The gate applies release, capability, label, + and suite exclusions before command execution. It is also the reporting + ownership edge: a failed selected step makes each named validation a + `step_failed` error in structured results and JUnit rather than allowing a + later missing-output skip. Keep + `requires_available_validations` for release-only gating; pytest `-k`/`-m` + selection remains too late to prune lifecycle commands. +- `CompositeCheck` predates the Launch Kit work and is framework machinery for + `compose:` entries. It now forwards member probes as `MemberName/probe-name`. + A member-level `pytest.skip` is reported as a skipped member while the + composite continues; skipped members neither pass nor fail the parent. + Successful validations with subtests are compacted by the shared isvctl + renderer; do not add suite-specific output flags. +- Launch Kit areas are separate validation classes in + `isvtest/validations/k8s_launch_kit/checks.py`; detailed probes use `report_subtest()` so + all manifest and connectivity rows reach JUnit output before the parent fails. +- Do not invent a `selfValidation` field in l8k output. Current `discover`, + `generate`, and `clean` emit one `ui.JSONResult`, successful standalone + `deploy` emits no stdout, and `validate` emits a JSON stream (static state, + connectivity matrix, then report path). The provider wraps these unmodified documents in a transport + envelope and keeps semantic assertions in pytest. The envelope records the + absolute command working directory so validations can resolve Launch Kit's + relative evidence paths without rewriting its output. +- Current l8k base check selection remains ICMP, `rping`, and `ib_write_bw`. + When `validation.gpuDirect.enabled` is true, GPUDirect DMA-BUF follows + `ib_write_bw` and is emitted as the distinct `gpudirect_dmabuf` result family. + Consume that family without adding an AI Cloud Validation default or a fourth + `--validation-checks` value. +- `l8k clean` is the only supported deletion path. Each use case declares it in + the teardown phase, linked to its deploy step; do not reproduce its + CR/finalizer/Helm logic with kubectl. It removes the test deployment but does + not snapshot or restore pre-existing state, so do not claim full ENT-REQ-010 + coverage until Launch Kit owns a transactional restore and verification API. +- Use `--label ethernet` or `--label infiniband` to prune the grouped run to one + fabric's three workflows. Use `--label sriov`, `--label rdma_shared`, or + `--label host_device` for one deployment-mode pair; labels compose to select + one concrete use case. `-k`/marker selection still happens after lifecycle + commands and does not prune Launch Kit workflows. +- Use `--label gpudirect` to select the six GPU-capable use-case definitions. + The semantic member skips when Launch Kit emits no `gpudirect_dmabuf` rows; + emitted failed rows must fail the parent with endpoint GPU evidence. +- Current reporting uploads JUnit/log/catalog only. Files under + `_output/k8s-launch-kit` are local evidence until the reporter gains an + explicit, redacted attachment contract. +- Design, prerequisites, unit-test boundaries, PRD mapping, and production gaps live in + `docs/guides/k8s-launch-kit/network-operator.md`. +- The structured PRD source is + `docs/requirements/network-operator-readiness-requirements.yaml`; keep its + `ENT-REQ-*` edges in `docs/requirements/test-requirements-matrix.yaml` and + regenerate committed views with `make plan`. + ## Environment Variables | Variable | Description | Used by | diff --git a/docs/README.md b/docs/README.md index 052375a85..ed828d15d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,6 +19,7 @@ Welcome to the documentation for NVIDIA AI Cloud Validation suite - a collection - [Configuration](guides/configuration.md) - Configuration file format and options - [External Validation Guide](guides/external-validation-guide.md) - Create custom validations without modifying the repo +- [Network Operator Launch Kit integration](guides/k8s-launch-kit/network-operator.md) - Production provider, Kubernetes preflight, mock-backed unit coverage, evidence, and limitations - [Remote Deployment](guides/remote-deployment.md) - Deploy and run tests on remote machines - [Local Development](guides/local-development.md) - MicroK8s setup for local testing - [Troubleshooting: Test runs stuck in STARTED](guides/troubleshooting-started-tests.md) - Why runs stay STARTED in the portal and how to fix it diff --git a/docs/guides/configuration.md b/docs/guides/configuration.md index 9e5547144..86d9bdbc4 100644 --- a/docs/guides/configuration.md +++ b/docs/guides/configuration.md @@ -56,7 +56,10 @@ Pre-built configs are provided in `isvctl/configs/`: | `providers/aws/config/vm.yaml` | AWS EC2 GPU instance tests | | `providers/aws/config/iam.yaml` | AWS IAM user lifecycle | | `providers/aws/config/eks.yaml` | AWS EKS with GPU nodes | +| `providers/k8s-launch-kit/config/provider.yaml` | Generic Kubernetes Launch Kit workflow | +| `providers/k8s-launch-kit/config/network-operator.yaml` | Six Network Operator Launch Kit use cases | | `suites/k8s.yaml` | Standard Kubernetes cluster | +| `suites/k8s-launch-kit/*.yaml` | Launch Kit-specific Network Operator catalog wiring | | `suites/slurm.yaml` | Slurm HPC cluster | ## Basic Usage @@ -155,17 +158,60 @@ Each platform defines phases and steps: commands: network: phases: ["setup", "test", "teardown"] # Execution order + continue_after_failure: [] # Optional independent test phases steps: [...] # Steps grouped by phase ``` | Field | Required | Description | | ----- | -------- | ----------- | -| `phases` | No | Ordered list of phases (default: `["setup", "test", "teardown"]`) | +| `phases` | No | Ordered list of phases (default: `["setup", "teardown"]`) | +| `continue_after_failure` | No | Phase names whose failure records a failed run but does not prevent later phases from running | | `steps` | Yes | List of step configurations | | `skip` | No | Skip this entire platform | **Important:** If a step's `phase` is not in the `phases` list, an error is raised. +Phase names are not limited to `setup`, `test`, and `teardown`. Any other name +is a custom test phase: it runs in the declared order, appears under its own +name in the orchestration summary, and is selected by `--phase test`. A +validation bound to a step runs after that step's custom phase. + +By default, a failed phase prevents later non-teardown phases from running. Use +`continue_after_failure` only when the named phases are independent test cases +and collecting every result in one invocation is more useful than stopping at +the first failure: + +```yaml +commands: + network_operator: + phases: [setup, roce-sriov, infiniband-sriov, roce-host-device] + continue_after_failure: [roce-sriov, infiniband-sriov, roce-host-device] + steps: + - name: prepare + phase: setup + command: ./prepare.sh + - name: test_roce_sriov + phase: roce-sriov + command: ./run-use-case.sh + args: [roce-sriov] + - name: test_infiniband_sriov + phase: infiniband-sriov + command: ./run-use-case.sh + args: [infiniband-sriov] + - name: test_roce_host_device + phase: roce-host-device + command: ./run-use-case.sh + args: [roce-host-device] +``` + +This setting changes continuation, not the verdict: if `roce-sriov` fails, +later listed use cases still run, but the final orchestration result remains +failed. Every continuation name must also appear once in `phases`; +configuration validation rejects unknown or duplicate names and forbids +`setup` and `teardown`. Do not list prerequisites shared by later phases or +phases that leave state on which later phases depend. Teardown retains its +existing `teardown_on_failure` behavior. + ### Step Configuration Each step defines a command to execute: @@ -180,6 +226,7 @@ Each step defines a command to execute: AWS_PROFILE: "production" skip: false continue_on_failure: false + finalizer_for: null output_schema: vpc ``` @@ -189,12 +236,84 @@ Each step defines a command to execute: | `phase` | No | Phase this step belongs to (default: `setup`) | | `command` | Yes | Script/command to execute | | `args` | No | Arguments (supports Jinja2 templates) | -| `timeout` | No | Timeout in seconds (default: 300) | +| `timeout` | No | Orchestration watchdog in seconds (default: 300); `null` disables it | | `env` | No | Environment variables | | `skip` | No | Skip this step | | `continue_on_failure` | No | Continue even if this step fails | +| `finalizer_for` | No | Run as linked teardown after the named step's phase when that command was attempted | | `output_schema` | No | Schema name for output validation | | `requires` | No | Capability contexts this step runs in (see [Capabilities](#capabilities-and-requires)) | +| `requires_available_validations` | No | Validation names that must be available after release filtering | +| `requires_selected_validations` | No | Configured validation names that must remain selected after release, capability, label, and suite-exclusion filtering; failed steps become errors on these owning validations | + +The timeout is an orchestration watchdog, not a provider-specific setting. Set +it to `null` only when the invoked tool owns a bounded deadline; isvctl will +then wait for the command to exit. On POSIX systems, isvctl starts each step in +a separate process group. When the +watchdog expires, it sends `SIGTERM` to the entire group, waits briefly, then +uses `SIGKILL` if needed. This prevents a wrapper's child CLI from continuing +to modify infrastructure after the wrapper step has been reported as timed +out. On non-POSIX systems, isvctl terminates the direct child process. + +#### Linked teardown finalizers + +Use `finalizer_for` when cleanup must run after the validations for one custom +test phase, including when the mutating step or a validation failed. Declare +cleanup in `phase: teardown`; the orchestrator executes it directly after its +target's test phase instead of waiting until every test case has finished: + +```yaml +commands: + network: + phases: [setup, use-case-one, use-case-two, teardown] + continue_after_failure: [use-case-one] + steps: + - name: deploy_fixture + phase: use-case-one + command: ./deploy.sh + + - name: clean_fixture + phase: teardown + command: ./clean.sh + finalizer_for: deploy_fixture +``` + +The finalizer target must resolve to one unique step, precede the configured +`teardown` phase, and cannot itself be a finalizer. The finalizer must use the +same capability and validation-selection gates as its target. Configuration +validation rejects violations of these rules. + +The orchestrator withholds linked teardown from normal phase execution, runs +the target phase validations, and then executes the eligible cleanup in +best-effort mode. The result is reported separately as +`-teardown`. This interleaving applies even to `--phase test`, so +multiple independent cases cannot leave deployments overlapping until the end +of the suite. A target activates cleanup only when its command process actually +started, whether it passed or failed. If an earlier prerequisite stopped the +phase, a template could not be rendered, or the executable could not be +started, cleanup is reported as skipped; this prevents deletion of pre-existing +state the current run never mutated. + +An explicit `--phase teardown` run executes linked teardown steps without an +in-memory target attempt. This is the standalone recovery path for resources +left by an interrupted earlier run. When target test phases and teardown are +part of the same invocation, already-linked cleanup is not run again in the +final teardown position. + +An ordinary use-case failure may still honor `continue_after_failure` after its +finalizers succeed. A failed finalizer always blocks later non-teardown phases, +because the fixture can no longer be assumed clean. Finalizer command output +and failure details are recorded in the teardown phase result. Keep finalizers +lifecycle-only rather than binding validations to their output, because target +phase validations intentionally run before cleanup. A same-phase finalizer is +still supported for compatibility, but a destructive provider cleanup should +normally be declared in `phase: teardown` so its lifecycle role and reporting +are explicit. + +Finalizers are an orchestration guarantee, not a recovery service. An abrupt +isvctl process termination, host failure, or `SIGKILL` can prevent them from +running. Provider cleanup commands should therefore be idempotent and usable as +standalone recovery commands. #### Gating a step with `requires` @@ -217,6 +336,58 @@ same one, so setup and teardown always move together. A step that survives the gate must not reference a gated-off step's output; use `default(...)` if it legitimately might be absent. +#### Gating Mutating Steps by Test Selection + +Use `requires_selected_validations` when a lifecycle step exists only to serve +specific validation entries. This applies selection before the command runs, +so `--label` and `--exclude-label` do not execute an unrelated deployment and +then discard its result: + +```yaml +commands: + network: + steps: + - name: deploy_ethernet_fixture + phase: ethernet + command: ./deploy-ethernet.sh + requires_selected_validations: [EthernetConnectivityCheck] + +tests: + validations: + network: + checks: + EthernetConnectivityCheck: + step: deploy_ethernet_fixture + labels: [ethernet] +``` + +With no label filter, the validation is selected and the step runs. With +`--label ethernet`, it also runs; with `--label infiniband`, the step is +skipped before execution. Every listed validation must be configured and +selected. The gate also honors the release manifest, capability requirements, +`tests.exclude.tests`, and effective label exclusions. + +The same list is the reporting ownership edge for the lifecycle step. If a +selected step fails before its validation can run, each listed validation is +reported as `error` with reason `step_failed`, including in JUnit. This prevents +an early deploy or setup failure from being misreported as a harmless +`step_no_output` skip merely because a later validation step was never reached. +The error message names the failed step and retains its redacted command +diagnostic. + +`requires_available_validations` is narrower: it only prevents a step from +running when its named checks are absent from the release manifest. Retain it +for providers that only need release gating. + +Pytest `-k` and `-m` expressions are evaluated inside pytest and therefore do +not drive `requires_selected_validations`. Use framework `--label` filtering +for lifecycle pruning in mutating suites. + +Selection-filtered validations remain in the structured result and JUnit +report. With the default `tests.settings.show_skipped_tests: false`, terminal +output omits summary phases containing only those filtered validations. Set it +to `true` when the skipped selection decisions should be visible interactively. + ### Validation Configuration Validations are centralized in `tests.validations`, grouped by category. Each group binds to a step and lists checks as a dict: @@ -351,6 +522,13 @@ Capability names and plain-suite names share one namespace, so a plain suite may not be named after a capability. `catalog_document` and `scripts/validate_suite_wiring.py` both reject the collision. +Suite discovery is recursive under `isvctl/configs/suites/`. A domain with +multiple related suites may therefore use a subdirectory such as +`suites/k8s-launch-kit/`; catalog generation, `--suite` resolution, doctor, +wiring validation, and test-plan coverage all discover the nested YAMLs. Suite +identity is still the YAML filename stem, so stems must remain unique across +the complete suite tree. + ## Import and Override Provider configs can import a canonical test suite and override command definitions while inheriting validations (unless explicitly overridden): @@ -584,6 +762,37 @@ checks: fields: ["network_id"] ``` +`CompositeCheck` is the existing framework runner behind `compose:`; authors do +not register or invoke that class directly. The YAML key creates one catalog +test and runs every listed validation member. Each member is reported as a +subtest. If a member reports its own probes through `report_subtest()`, those +probes are retained with qualified names such as +`LaunchKitRdmaConnectivityCheck/rping/worker-a->worker-b/rail-0->rail-1`. +This avoids collisions between members and keeps the full probe tree in pytest +and JUnit output. + +A member may call `pytest.skip` when it is not applicable to the current +environment. `CompositeCheck` records that member as a skipped subtest and +continues with the remaining members. The skip neither passes nor fails the +member, and the parent composite passes when every non-skipped member passes. +This is different from skipping the step output or the composite itself, both +of which skip the entire parent validation. + +The orchestration summary automatically abbreviates a successful validation +that reported subtests: + +```text +MyUseCase: PASSED - 12 subtests passed +``` + +If optional probes were skipped, the summary includes passed, failed, and +skipped counts. Failed and errored validations keep their original diagnostic +message instead of being abbreviated. There is no YAML presentation flag; +this behavior applies to composites and ordinary validation classes alike. +After subtest testcase nodes are injected into JUnit, the suite's tests, +failures, errors, and skipped counters are recalculated from those serialized +nodes so reports do not double-count pytest's pre-counted subtest events. + `SchemaValidation` remains directly wireable, but is catalog-excluded because the step executor runs schema checks automatically. diff --git a/docs/guides/k8s-launch-kit/network-operator.md b/docs/guides/k8s-launch-kit/network-operator.md new file mode 100644 index 000000000..82c40aa5d --- /dev/null +++ b/docs/guides/k8s-launch-kit/network-operator.md @@ -0,0 +1,752 @@ + + + +# Network Operator validation through Kubernetes Launch Kit + +## Status and safety boundary + +This integration ships a generic Kubernetes Launch Kit provider and a +Network Operator configuration that invoke real `l8k` and `kubectl` binaries. +It also adds catalog checks, requirement traceability, and mock-backed unit +tests. The mocks are not reachable from product configuration: tests load the +production YAML and inject test-owned executables in memory. + +The production workflow reaches a Kubernetes cluster and mutates it during +`l8k deploy`. Each `l8k clean` step is declared in the teardown phase and linked +to its deployment. After every attempted deployment, the orchestrator runs that +cleanup directly after the use case's validations and reports a distinct +`-teardown` result. Launch Kit deletes Network Operator custom +resources, waits for their finalizers, and uninstalls the Helm release last. +Cleanup is destructive and covers the complete resolved operator namespace plus +Launch Kit's known cluster-scoped Network Operator resources. Use a dedicated +qualification cluster. + +`l8k clean` removes the test deployment but does not snapshot and restore +pre-existing state, so the provider still does not fully satisfy ENT-REQ-010. +A green unit test proves the AI Cloud Validation integration and reporting +path; it is not certification evidence. + +The RoCE SR-IOV use case has also been exercised end to end on a two-node +Ubuntu 24.04 single-rail cluster. Launch Kit completed discovery, generation, +deployment, validation, and cleanup. The multi-rail member was reported as +skipped because the discovered topology contained one rail; the remaining +members and the parent use case passed. The other five use cases retain their +mock-backed integration status until they are qualified on applicable live +clusters. + +## Framework architecture + +AI Cloud Validation separates command execution from result interpretation: + +```text +provider configuration + -> ordered isvctl steps + -> setup: l8k installation verification + -> test prerequisite: repeat l8k version/schema verification + -> one named phase per Launch Kit use case + -> kubectl prerequisite probes + -> l8k discover + -> l8k generate + -> l8k deploy + -> l8k validate + -> generic adapter.py transport + -> phase validations + -> linked teardown: l8k clean (when deploy was attempted) + -> raw JSON transport envelopes stored as step outputs + -> suite wiring + -> globally reusable PRD validation classes + -> one CompositeCheck-backed test per concrete use case + -> member and probe-level pytest subtests + -> catalog pass/fail/skip + -> JUnit, run logs, catalog, and optional AI Cloud Labs reporting +``` + +The main files are: + +| Layer | File | +|---|---| +| Generic provider | `isvctl/configs/providers/k8s-launch-kit/config/provider.yaml` | +| Production Network Operator provider | `isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml` | +| CLI transport | `isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py` | +| Individual PRD check wiring | `isvctl/configs/suites/k8s-launch-kit/network-operator.yaml` | +| End-to-end use-case wiring | `isvctl/configs/suites/k8s-launch-kit/network-operator-use-cases.yaml` | +| Result interpretation | `isvtest/src/isvtest/validations/k8s_launch_kit/checks.py` | +| Provider unit tests | `isvctl/tests/providers/k8s_launch_kit/test_provider.py` | +| Executable test doubles and data | `isvctl/tests/providers/k8s_launch_kit/fixtures/` | +| Result-check unit tests | `isvtest/tests/k8s_launch_kit/test_checks.py` | +| PRD source | `docs/requirements/network-operator-readiness-requirements.yaml` | +| Traceability | `docs/requirements/test-requirements-matrix.yaml` | + +Both suite files are plain suites with Kubernetes `requires` metadata. The +provider supplies a `network_operator` command target. This keeps the catalog reusable +while capability selection (`--capability kubernetes`) prevents the steps and +checks from running in VM, bare-metal, or Slurm contexts. Provider and suite +files are intentionally separate: a live overlay imports the provider plus the +individual suite or a program-specific use-case suite. + +## Reusable framework changes + +The productization audit kept only behavior that is useful outside Launch Kit: + +| Change | Abstraction owner | Reuse contract | +|---|---|---| +| Named custom test phases | `isvctl.orchestrator` | Any configured phase other than setup/teardown is selected by `--phase test`, executes in declared order, and retains its configured name in results. | +| `continue_after_failure` | `isvctl.config.PlatformCommands` and orchestrator | Independent test-case phases may continue collecting evidence after a failure without changing the failed final verdict. | +| `finalizer_for` | `isvctl.config.StepConfig` and orchestrator | A cleanup step declared in teardown runs directly after its target phase validations when the mutating process started, gets a separate teardown result, and can also run through explicit teardown-only recovery. | +| Selection-aware step gate and failure ownership | `isvctl.config.StepConfig` and orchestrator | `requires_selected_validations` skips a lifecycle step unless its named validation remains selected after release, capability, label, and suite-exclusion filtering. A failed selected step becomes a `step_failed` JUnit error on the owning validation instead of a later missing-output skip. | +| Nested composite probes and member skips | `isvtest.core.CompositeCheck` | A composed member's own subtests are forwarded as `MemberName/probe-name`. A member-level `pytest.skip` is reported as a skipped member while the composite continues and evaluates its remaining members. `CompositeCheck` itself predates this integration. | +| Structured subtest summary | `isvtest.core.ResolvedEntry` and the `isvctl` renderer | Any validation with subtests gets passed/failed/skipped counts; successful terminal lines are concise while failures keep diagnostics. | +| Filtered-summary visibility | `isvctl` renderer | `tests.settings.show_skipped_tests: false` omits terminal phases containing only selection-filtered validations while retaining them in structured results and JUnit. | +| JUnit counter reconciliation | `isvtest.testing.subtests` | After subtest testcase injection, suite counters are derived from serialized testcase nodes so pytest's pre-counted subtests are not counted twice. | +| Project-PRD rendering | `scripts/requirements_source_to_md.py` | A uniquely named requirement source can set `format: project-prd`; future PRDs reuse the renderer without adding source-specific code. | +| Process-tree timeout enforcement | `isvctl.orchestrator` | On POSIX, timed-out steps terminate their complete process group, so a wrapper cannot leave its provider CLI running after orchestration has moved on. | + +These APIs are documented for other suites in the +[configuration guide](../configuration.md) and +[`isvtest` package guide](../../packages/isvtest.md). Launch Kit's transport, +output schema, checks, and mocks remain provider/domain-specific and are not +framework features. + +## Provider API: the actual Launch Kit workflow + +The provider does not expose a different Network Operator abstraction. Its +steps follow the CLI workflow directly: + +1. `launch_kit_prepare` — optionally install, then verify Launch Kit; +2. `launch_kit_verify` — verify again in the test phase, because a test-only run + can skip setup; +3. `launch_kit_kubernetes_preflight` — prove the Kubernetes prerequisite; +4. `launch_kit_discover` — invoke `l8k discover`; +5. `launch_kit_generate` — invoke `l8k generate`; +6. `launch_kit_deploy` — invoke `l8k deploy`; +7. `launch_kit_validate` — invoke `l8k validate`; +8. `launch_kit_clean` — declare `l8k clean` in teardown, linked to the deploy + step and executed immediately after that use case's phase validations. + +The public context is deliberately small: + +| Key | Meaning | +|---|---| +| `executable` | Existing `l8k` command or path | +| `installation.mode` | `verify` (default) or explicitly requested `install` | +| `installation.version` | Optional release/tag passed to the official installer path | +| `installation.installer_ref` | Full 40-character Git commit SHA for the official installer; required in install mode | +| `installation.installer_sha256` | Trusted SHA-256 of the installer at `installer_ref`; required in install mode | +| `installation.prefix` | Optional install prefix | +| `user_config` | Optional absolute path to a complete Launch Kit configuration outside the retained working directory; staged transiently for every selected use case | +| `kubectl_command` | Optional kubectl-compatible argv prefix, mainly for controlled environments/tests | +| `working_dir` | Directory in which l8k resolves relative inputs and writes outputs | +| `artifact_dir` | Directory for provider command evidence | +| `environment` | String environment entries forwarded to installation, verification, l8k workflows, and the matching Kubernetes preflight | +| `.arguments` | Raw string argv for `discover`, `generate`, `deploy`, `validate`, or `clean` | + +There are no AI Cloud Validation defaults for namespace, node selector, +Network Operator release, driver mode, rails, resource names, IP pools, GPU +count, validation mode/checks, RDMA tuning, or l8k timeouts. A user may pass any +flag supported by the installed Launch Kit release; omitted values are resolved +by Launch Kit. The provider adds only `--output json` to workflow commands so it +can preserve structured evidence. A user-supplied non-JSON output selection is +rejected. + +The `timeout` fields on provider steps are outer isvctl watchdogs. They bound a +hung child process; they are not Launch Kit configuration defaults and are not +forwarded to l8k. On POSIX, timeout handling terminates the adapter and its l8k +child process group before returning the failed step. Launch Kit-specific +timeout flags remain raw user arguments. + +Launch Kit `validate` steps intentionally set this outer watchdog to `null`. +The installed l8k release calculates and logs a bounded timeout from its matrix +plan, or honors the user's explicit `--connectivity-timeout` argument. Other +workflow steps retain finite isvctl watchdogs, and an enclosing CI job may +still impose its own deadline. + +The configured executable is invoked directly. A path ending in `.py` receives +no special treatment; executable test doubles must use a shebang and executable +file mode. This prevents test-only behavior from becoming part of the live +provider API. + +### Complete user configuration + +Set `context.k8s_launch_kit.user_config` when a site needs values that are not +exposed as Launch Kit CLI flags. The file must be a complete Launch Kit +configuration; AI Cloud Validation does not merge partial YAML fragments or +interpret its fields. Use an absolute path available on the machine executing +the suite. + +For each selected use case, the adapter copies the source to +`/user-config.yaml` with mode `0600`, then invokes discovery with: + +```text +l8k discover \ + --user-config /user-config.yaml \ + --save-cluster-config /cluster-config.yaml \ + --output json +``` + +The source is not modified. The staged copy is deleted as soon as discovery +exits, whether it succeeds or fails; retained evidence contains only the input +path, byte size, and SHA-256 provenance. The source must therefore live outside +the retained provider working directory. Discovery refreshes hardware and +applies the use-case's explicit fabric/deployment selectors, while Launch Kit +preserves the settings from the staged input according to its own precedence +rules. The resolved `cluster-config.yaml` is then consumed through Launch Kit's +normal default paths by generate, deploy, validate, and clean. When +`user_config` is set, raw discovery arguments must not also contain +`--user-config` or `--save-cluster-config`. + +### Example live overlay + +Keep site and program choices in a user-owned overlay instead of adding them to +the generic provider: + +```yaml +import: + - isvctl/configs/providers/k8s-launch-kit/config/provider.yaml + - isvctl/configs/suites/k8s-launch-kit/network-operator.yaml + +context: + k8s_launch_kit: + executable: /opt/nvidia/bin/l8k + user_config: /secure/partner-cluster-config.yaml + working_dir: /var/tmp/aicv-launch-kit/work + artifact_dir: /var/tmp/aicv-launch-kit/evidence + discover: + arguments: + - --kubeconfig + - /secure/partner.kubeconfig + - --fabric + - ethernet + - --deployment-type + - sriov + generate: + arguments: [] + deploy: + arguments: [--kubeconfig, /secure/partner.kubeconfig] + validate: + arguments: [--kubeconfig, /secure/partner.kubeconfig] + clean: + arguments: [--kubeconfig, /secure/partner.kubeconfig] +``` + +These values are explicit inputs for that run. The complete user config is +copied into the isolated working directory only for discovery, then removed; +its values are not copied into evidence or treated as AI Cloud Validation +defaults. + +## Download, install, and verification + +`installation.mode: verify` resolves the configured executable from an explicit +path or `PATH`, then requires both commands to succeed and emit one JSON object: + +```text +l8k version --output json +l8k schema +``` + +When `installation.version` is non-empty, verification also requires the +reported `version` field to match it exactly. The test-phase verification +repeats this check, so `--phase test` cannot bypass a configured version pin. +The schema must advertise `discover`, `generate`, `deploy`, `validate`, and +`clean`; a pre-clean Launch Kit binary is rejected before cluster mutation. + +`installation.mode: install` is opt-in. The provider downloads +`scripts/install.sh` from the official NVIDIA Kubernetes Launch Kit repository +at the full Git commit in `installation.installer_ref`. It requires a trusted +`installation.installer_sha256`, records the source and actual digest, and +fails before writing or executing the script unless they match. Mutable refs, +tags, and abbreviated commits are rejected. The upstream script remains +responsible for release/archive selection, checksum verification, and +installing the binary and profiles. The provider then performs the same version +and schema verification against the binary at `/bin/l8k` (or +`/usr/local/bin/l8k` when no prefix is supplied), rather than accepting an older +binary found elsewhere on `PATH`. Network access and permissions required by +the upstream installer remain operator prerequisites. + +The installed schema is captured as evidence. This makes version/capability +drift diagnosable without teaching AI Cloud Validation Launch Kit's flag +defaults. + +Keep `verify` as the default. Production automation that enables `install` +must obtain the immutable installer commit and its SHA-256 through a trusted +release channel; computing a digest from the same download is not trust. + +An overlay opts into installation explicitly: + +```yaml +context: + k8s_launch_kit: + executable: /opt/nvidia/bin/l8k + installation: + mode: install + version: v0.1.0 + installer_ref: "" + installer_sha256: "" + prefix: /opt/nvidia +``` + +With `prefix` omitted, the upstream installer uses its own installation +default. With `version` omitted, it resolves its own latest supported release; +`installer_ref` and `installer_sha256` remain mandatory in install mode. + +## Mandatory Kubernetes prerequisite + +Every normal test use case verifies the cluster before `discover`, `generate`, +`deploy`, or `validate` can run. Its linked `clean` then acts on the same +working directory and cluster selection. The preflight: + +1. scans all five raw argument arrays for `--kubeconfig` and + `--kubeconfig=`; +2. fails if different workflow commands select different kubeconfigs; +3. forwards the same `environment` mapping used for l8k and otherwise uses + kubectl's normal environment/default resolution when no kubeconfig is + explicit; +4. runs `kubectl version -o json` and requires + `serverVersion.gitVersion`; +5. runs `kubectl get nodes -o json` and requires at least one node and at least + one node with `Ready=True`. + +Each probe stores argv, stdout, and stderr and becomes a pytest subtest. Normal +steps within a use case are sequential and stop on failure. Therefore a failed +preflight, discovery, or generation prevents deployment and does not activate +destructive cleanup. Once deploy is attempted, `clean` runs after the phase +validations as a linked teardown regardless of the deploy, validate, or +validation verdict. A missing executable or unresolved command template is not +an attempt and cannot activate destructive cleanup. The grouped production +configuration's use cases are independent custom phases, so +`continue_after_failure` allows the next use case to start only when teardown +succeeded. The final run still records the failed case and a failed overall +verdict; a cleanup failure blocks later cases because cluster state is unknown. +An explicit `--phase teardown` run invokes the selected cleanup steps without a +prior in-memory deployment attempt or a fresh preflight, providing an +idempotent recovery command even when test prerequisites no longer pass. The +cleanup command still performs Launch Kit's own API access and safety checks. + +This is intentionally a minimum prerequisite, not a replacement for Launch +Kit's topology, RBAC, device, fabric, or deployment preflight logic. + +## Transport and Launch Kit output fidelity + +Every provider action emits one JSON transport envelope with fields such as: + +```json +{ + "success": true, + "platform": "kubernetes", + "operation": "deploy", + "executable": "/opt/nvidia/bin/l8k", + "argv": ["/opt/nvidia/bin/l8k", "deploy", "...", "--output", "json"], + "working_directory": "/var/tmp/aicv-launch-kit/work", + "exit_code": 0, + "documents": [], + "artifacts": { + "command": ".../command.json", + "stdout": ".../stdout.txt", + "stderr": ".../stderr.log" + } +} +``` + +`documents` are unmodified JSON objects parsed from Launch Kit stdout. The unit +fixture is pinned to the machine-output shapes and validation profile contract +audited on Launch Kit `main` at commit `db32e4b98170`: + +| Command | Current machine stdout | +|---|---| +| `discover` | One `ui.JSONResult` containing the resolved profile | +| `generate` | One `ui.JSONResult` containing generated file paths | +| successful standalone `deploy` | Empty stdout; progress remains on stderr | +| `validate` | Three concatenated objects: static validation, connectivity matrix, and report path | +| `clean` | One `ui.JSONResult` containing namespace, custom-resource deletion count, Helm removal status, and Helm-retention choice | + +The transport parses concatenated JSON without renaming PascalCase Launch Kit +fields such as `PingResults`, `Family`, `ObservedOK`, and `BandwidthGbps`. +GPUDirect DMA-BUF rows remain in that same matrix with +`Family: gpudirect_dmabuf` and endpoint GPU indices/PCI addresses. Semantic +knowledge stays in the pytest validations, not the provider. The envelope also +records the absolute command working directory so evidence checks can resolve +relative paths emitted by Launch Kit without rewriting the documents. + +## Production use cases and mock-backed unit tests + +The production `config/network-operator.yaml` imports the generic provider and +the use-case suite, then replaces the generic single-workflow command list with +six named phases. It inherits `executable: l8k` and an empty +`kubectl_command`, which means normal `kubectl` resolution from `PATH`. The YAML +contains only the fabric/deployment flags that define each test identity. +Launch Kit owns the default `./cluster-config.yaml` and `./deployment` paths +across `discover -> generate -> deploy -> validate -> clean`; the grouped +`generate`, `deploy`, `validate`, and `clean` argument arrays are therefore +empty. Each use case runs in an isolated working directory, so these defaults +cannot collide. + +Provider unit tests load this same YAML, replace the two executable settings in +memory, and run it against `mock_l8k.py` and `mock_kubectl.py`. The test double +produces a discovered configuration, generated manifests, static resource +results, connectivity results, an HTML report, and the current Launch Kit +cleanup summary. Representative resolved values in mock output belong to the +test fixture's Launch Kit side; they are not provider defaults. + +The fixture supports these explicit profiles: + +| Fabric | Deployment | Secondary network kind | +|---|---|---| +| Ethernet/RoCE | `sriov` | `SriovNetwork` | +| InfiniBand | `sriov` | `SriovIBNetwork` | +| Ethernet/RoCE | `rdma_shared` | `MacvlanNetwork` | +| InfiniBand | `rdma_shared` | `IPoIBNetwork` | +| Ethernet/RoCE | `host_device` | `HostDeviceNetwork` | +| InfiniBand | `host_device` | `HostDeviceNetwork` | + +One production invocation runs all six profiles in the table, in order. Every +profile has explicit raw discover arguments and its own working/evidence +directory, so one case cannot reuse another case's generated output. Its +linked `clean` teardown removes the deployed Network Operator resources before +the next test phase. The ordered result names make that boundary explicit: + +```text +setup +launch-kit-verification +roce-sriov +roce-sriov-teardown +infiniband-sriov +infiniband-sriov-teardown +roce-rdma-shared +roce-rdma-shared-teardown +infiniband-rdma-shared +infiniband-rdma-shared-teardown +roce-host-device +roce-host-device-teardown +infiniband-host-device +infiniband-host-device-teardown +``` + +Run the production workflow against the active Kubernetes context: + +```bash +ISVTEST_INCLUDE_UNRELEASED=1 uv run isvctl test run \ + -f isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml \ + --capability kubernetes --no-upload -- -v +``` + +This command invokes real `l8k`, real `kubectl`, destructive `l8k deploy`, and +destructive `l8k clean`. Confirm the active kubeconfig and use a dedicated +qualification cluster. If an interrupted run left a selected deployment behind, +invoke its teardown without rerunning the test: + +```bash +ISVTEST_INCLUDE_UNRELEASED=1 uv run isvctl test run \ + -f isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml \ + --phase teardown --capability kubernetes \ + --label ethernet --label sriov --no-upload +``` + +This example runs only `launch_kit_roce_sriov_clean`. Omitting labels runs all +six idempotent cleanup entries in best-effort mode. Confirm the target cluster +before using either form. To validate the integration without a cluster, run +the automated tests instead: + +```bash +uv run pytest \ + isvctl/tests/providers/k8s_launch_kit \ + isvtest/tests/k8s_launch_kit -q +``` + +The final orchestration summary has one test result and one teardown result per +use case. The verbose pytest section still contains every member/probe result: + +```text +[PASS] SETUP : launch_kit_prepare: passed +[PASS] LAUNCH-KIT-VERIFICATION : launch_kit_verify: passed +[PASS] ROCE-SRIOV : ... + [network_operator_roce_sriov] EastWestNetworkRoceSriovCheck: PASSED - 122 subtests passed +[PASS] ROCE-SRIOV-TEARDOWN : launch_kit_roce_sriov_clean: passed +[PASS] INFINIBAND-SRIOV : ... + [network_operator_infiniband_sriov] EastWestNetworkInfiniBandSriovCheck: PASSED - 122 subtests passed +[PASS] INFINIBAND-SRIOV-TEARDOWN: launch_kit_infiniband_sriov_clean: passed +[PASS] ROCE-RDMA-SHARED : ... + [network_operator_roce_rdma_shared] EastWestNetworkRoceRdmaSharedCheck: PASSED - 117 subtests passed +[PASS] ROCE-RDMA-SHARED-TEARDOWN: launch_kit_roce_rdma_shared_clean: passed +[PASS] INFINIBAND-RDMA-SHARED : ... + [network_operator_infiniband_rdma_shared] EastWestNetworkInfiniBandRdmaSharedCheck: PASSED - 117 subtests passed +[PASS] INFINIBAND-RDMA-SHARED-TEARDOWN: launch_kit_infiniband_rdma_shared_clean: passed +[PASS] ROCE-HOST-DEVICE : ... + [network_operator_roce_host_device] EastWestNetworkRoceHostDeviceCheck: PASSED - 117 subtests passed +[PASS] ROCE-HOST-DEVICE-TEARDOWN: launch_kit_roce_host_device_clean: passed +[PASS] INFINIBAND-HOST-DEVICE : ... + [network_operator_infiniband_host_device] EastWestNetworkInfiniBandHostDeviceCheck: PASSED - 117 subtests passed +[PASS] INFINIBAND-HOST-DEVICE-TEARDOWN: launch_kit_infiniband_host_device_clean: passed +[PASS] All phases completed successfully +``` + +The launch-kit checks remain unreleased, so the environment variable is +required for development runs. Do not add them directly to +`released_tests.json`; the repository's release process owns that file. + +The six use-case phases are listed under `continue_after_failure`. This is a +generic isvctl feature for independent test cases: a failed case does not hide +later cases after its linked teardown succeeds, but it still makes the final +run fail. A teardown failure blocks later cases. Setup is deliberately not +listed because all cases depend on the prepared and verified executable. + +## Selectable checks, use cases, and error reporting + +The suite contains one prerequisite check and fourteen currently supported +PRD-facing checks: + +| Test ID | Check | Evidence | +|---|---|---| +| N/A | Kubernetes prerequisite | API version, node inventory, Ready nodes | +| K8S42-01 | Deployment health | version match, summary, every manifest row | +| K8S42-02 | SR-IOV readiness | policy plus profile-specific SR-IOV network | +| K8S42-03 | RDMA connectivity | every `rping` matrix row | +| K8S42-04 | RoCE | exact Ethernet profile network kind | +| K8S42-05 | InfiniBand | exact InfiniBand profile network kind | +| K8S42-06 | Host-device | `HostDeviceNetwork` for an applicable profile | +| K8S42-07 | GPUDirect RDMA | every `gpudirect_dmabuf` row, endpoint GPUs, PCI addresses, bandwidth, and Launch Kit threshold | +| K8S42-08 | Topology discovery | successful discover and resolved profile | +| K8S42-09 | Secondary network/IPAM | `IPPool`, exact network kind, test DaemonSet rollout | +| K8S42-10 | RDMA Shared | `MacvlanNetwork` or `IPoIBNetwork`, as selected | +| K8S42-11 | ICMP | every source-bound ICMP matrix row | +| K8S42-12 | RDMA bandwidth | every `ib_write_bw` row and Launch Kit threshold | +| K8S42-13 | Multi-rail | same-rail and cross-rail matrix coverage; skipped when the matrix contains only one distinct rail | +| K8S42-15 | Evidence | command files and Launch Kit HTML report | + +These check classes are registered once and reused by each applicable use case. +The individual suite can still run them directly against one selected Launch +Kit profile; in that mode, profile-specific checks call `pytest.skip` when the +profile is not applicable. `LaunchKitMultirailCheck` also skips when Launch Kit +returns connectivity rows for only one distinct rail. `LaunchKitGpuDirectRdmaCheck` +skips when Launch Kit emits no `gpudirect_dmabuf` family because discovery +disabled `validation.gpuDirect` or `ib_write_bw` was not selected. If GPUDirect +is enabled but topology or execution fails, Launch Kit emits failed rows and +the check fails with their diagnostics. Inside a grouped use-case composite, +an inapplicable member is reported as skipped while the other checks continue; +it does not fail or skip the parent use case. + +Full state restoration (`ENT-REQ-010`) remains deferred and has no test ID. +Launch Kit cleanup removes the test deployment rather than capturing and +restoring pre-existing state; full coverage still requires a snapshot, +restore, and semantic verification workflow. + +The grouped suite exposes six separate catalog tests. Each composite lists only +the global checks applicable to its fabric and deployment type: + +| Test ID | Use-case test | Fabric | Deployment-specific check | +|---|---|---|---| +| K8S42-16 | `EastWestNetworkRoceSriovCheck` | Ethernet/RoCE | SR-IOV readiness | +| K8S42-17 | `EastWestNetworkInfiniBandSriovCheck` | InfiniBand | SR-IOV readiness | +| K8S42-18 | `EastWestNetworkRoceRdmaSharedCheck` | Ethernet/RoCE | RDMA Shared | +| K8S42-19 | `EastWestNetworkInfiniBandRdmaSharedCheck` | InfiniBand/IPoIB | RDMA Shared | +| K8S42-20 | `EastWestNetworkRoceHostDeviceCheck` | Ethernet/RoCE | host-device | +| K8S42-21 | `EastWestNetworkInfiniBandHostDeviceCheck` | InfiniBand | host-device | + +This is why the grouped output has no unrelated fabric/deployment skips in +the middle of a use case. + +Every resource and connectivity row is a named pytest subtest. A connectivity +name includes family, source/destination, and source/destination rails, for +example: + +```text +rping/worker-a->worker-b/rail-0->rail-1 +ib_write_bw/worker-b->worker-a/rail-1->rail-1 +gpudirect_dmabuf/worker-a->worker-b/rail-0->rail-0 +``` + +Messages preserve expectation, observed outcome, stderr, and bandwidth/minimum +values. GPUDirect messages additionally preserve source/destination GPU indices +and PCI addresses when Launch Kit reports them. The validation reports every +row before aggregating failures, so a user does not need repeated runs to +discover the next failed pair. + +`CompositeCheck` was already the framework mechanism behind `compose:`. This +integration extends it to forward a member's own probes as +`MemberName/probe-name`, preserving the detailed pytest and JUnit diagnostics. +It also preserves member-level applicability: when a member calls +`pytest.skip`, that member is emitted as a skipped subtest and the composite +continues. The parent can therefore pass when all of its applicable members +pass. +The shared isvctl renderer now abbreviates any successful parent that reported +subtests, for example `PASSED - 122 subtests passed`. Failed and errored parents +keep the original actionable message. There is no Launch Kit-specific or YAML +`compact_output` option. + +Individual PRD-check selection is available from a run configuration that +imports the generic provider and `network-operator.yaml`, such as the live +overlay shown earlier: + +```bash +ISVTEST_INCLUDE_UNRELEASED=1 uv run isvctl test run \ + -f /path/to/my-launch-kit-overlay.yaml \ + --capability kubernetes --no-upload -- \ + -v -k LaunchKitRdmaConnectivityCheck +``` + +The grouped provider imports the use-case suite, so its selectable pytest identities are the +six `EastWestNetwork*Check` parents. For example, use +`-k EastWestNetworkRoceSriovCheck` to interpret only that parent. Composite member +names are detailed subtests, not separately selectable pytest identities in +that configuration. + +The grouped suite can be selected with `--label network_operator` or the more +specific `--label network_operator_use_cases`. Fabric labels divide it into +three Ethernet/RoCE and three InfiniBand workflows: + +```bash +# All six use cases (default) +ISVTEST_INCLUDE_UNRELEASED=1 uv run isvctl test run \ + -f isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml \ + --capability kubernetes --no-upload -- -v + +# Ethernet/RoCE only +ISVTEST_INCLUDE_UNRELEASED=1 uv run isvctl test run \ + -f isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml \ + --capability kubernetes --label ethernet --no-upload -- -v + +# InfiniBand only +ISVTEST_INCLUDE_UNRELEASED=1 uv run isvctl test run \ + -f isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml \ + --capability kubernetes --label infiniband --no-upload -- -v +``` + +The `roce` label remains available as a more specific alias for the Ethernet +subset. Each use-case step declares its owning composite through +`requires_selected_validations`, so the other fabric's preflight, discover, +generate, deploy, validate, and clean commands do not execute. With no fabric +label, all six validations are selected and all workflows run. + +Deployment-mode labels select the corresponding Ethernet/RoCE and InfiniBand +pair: + +| Selection | Workflows | +|---|---| +| `--label sriov` | RoCE SR-IOV and InfiniBand SR-IOV | +| `--label rdma_shared` | RoCE RDMA Shared and InfiniBand RDMA Shared/IPoIB | +| `--label host_device` | RoCE host-device and InfiniBand host-device | +| `--label gpudirect` | All six workflows; the GPUDirect member runs or skips from Launch Kit output in each | + +Labels compose with all-match semantics. For example, +`--label ethernet --label sriov` runs only +`EastWestNetworkRoceSriovCheck`, while +`--label infiniband --label rdma_shared` runs only +`EastWestNetworkInfiniBandRdmaSharedCheck`. + +The `era` and `ncp` labels are present for future program policy. +Required-versus-optional program profiles are not yet typed metadata. Pytest +`-k`/`-m` selection is still applied after lifecycle steps and therefore does +not prune workflows; use `--label` for mutating suite selection. + +## Evidence and reporting + +The provider records these files per action or command: + +- exact argv, exit code, and duration in `command.json`; +- raw stdout, including Launch Kit JSON, in `stdout.txt`; +- raw stderr in `stderr.log`; +- immutable installer source, expected/actual SHA-256, and verification result + when installation is requested; +- version and schema responses; +- Kubernetes preflight output; +- safe `user_config` provenance (path, size, and SHA-256), when configured, and + Launch Kit's resolved `cluster-config.yaml`; the raw staged input is removed; +- generated manifests, validation report, and other files written beneath the + configured working directory. + +The evidence validation checks command artifact paths and the report path +emitted by Launch Kit. JUnit, run logs, and catalog status flow through existing +AI Cloud Validation reporting. AI Cloud Labs does not currently upload the +arbitrary evidence directory as a binary attachment set; that remains an +explicit gap. + +In the grouped production run, shared installation/version/schema evidence is stored once under +`_output/k8s-launch-kit/network-operator/shared-evidence`. Each use case then stores its command +evidence and Launch Kit files below +`_output/k8s-launch-kit/network-operator/use-cases//`. These files are validated and +referenced locally; they are not yet registered as Labs binary attachments. + +The generic provider does not recursively delete a working or evidence +directory, because those paths are user-controlled and may contain retained +evidence. Use run-scoped directories when historical separation is required. +Current-step command files are overwritten, and a failed prerequisite cannot +reuse stale files as a passing validation because downstream steps have no +current output and are skipped. + +## Cluster prerequisites + +| Area | Required state | +|---|---| +| All live checks | Reachable Kubernetes API, authorization to list nodes, non-empty cluster, at least one Ready node, consistent kubeconfig across workflow commands | +| Discovery/generation | Permissions required by Launch Kit discovery; supported Kubernetes and Launch Kit versions; access to required profiles/config inputs | +| Deployment health | Helm/Kubernetes mutation and read permissions, image registry access, supported Network Operator release | +| SR-IOV | Supported NVIDIA NICs and VFs, SR-IOV components, Multus, secondary-network CRDs, and applicable IPAM | +| RDMA Shared | RDMA Shared device-plugin resource on enough workers; Macvlan for RoCE or IPoIB for InfiniBand | +| RoCE | Configured Ethernet fabric, valid GIDs, and required lossless QoS/PFC/ECN outside Kubernetes | +| InfiniBand | Active fabric and subnet manager, valid P_Keys, IB VFs or shared devices, and IPoIB when selected | +| Host-device in VMs | Supported Ethernet or InfiniBand devices passed through to worker VMs and allocatable through host-device networking | +| Connectivity/bandwidth | At least two applicable schedulable workers, test image availability, secondary addresses, RDMA device mapping, `ping`, `rping`, and `ib_write_bw` support | +| Multi-rail | At least two distinct rails in Launch Kit connectivity output; the multi-rail check is skipped on a single-rail topology | +| Cleanup | Permission to list CRDs and list/get/delete Network Operator custom resources cluster-wide, process their finalizers, and manage the `network-operator` Helm release in the resolved namespace | +| GPUDirect RDMA | At least two GPU workers, allocatable `validation.gpuDirect.gpuResourceType` on every targeted worker, compatible CUDA/DMA-BUF and GPU/NIC topology, unambiguous per-rail `connectedGPU` mappings, the full-runtime DOCA validation image, and its pull Secret in every validation namespace. Discovery enables the check only when every worker can satisfy the topology-derived GPU request. | +| State restoration | Cleanup removes the test deployment; preserving and restoring pre-existing state still requires a future Launch Kit transaction API | + +## PRD coverage and remaining gaps + +| Requirement | Current result | +|---|---| +| ENT-REQ-000/001 | Generic provider boundary and normal isvctl/pytest workflow exist; long-term Network Operator ownership is organizational. | +| ENT-REQ-002 | Reuses Launch Kit discover, manifest readiness, ICMP, `rping`, host-memory `ib_write_bw`, and GPUDirect DMA-BUF output. | +| ENT-REQ-003 | Individual PRD checks and six grouped end-to-end use-case tests exist; independent phases continue to collect all case results. Fabric labels prune unselected lifecycle commands; typed Enterprise/NCP required/optional profiles remain a gap. | +| ENT-REQ-004 | Users pass raw supported l8k arguments; Launch Kit owns flags and defaults. | +| ENT-REQ-005/006 | Production wiring covers SR-IOV and RDMA Shared resources plus ICMP, `rping`, and bandwidth for RoCE and InfiniBand; unit fixtures exercise every path, and RoCE SR-IOV has been exercised on a live single-rail cluster. The other fabric/deployment profiles still require live qualification. Current l8k output has no dedicated pod RDMA-device inventory result, so an explicit device-availability contract remains. | +| ENT-REQ-007 | Production wiring and unit fixtures cover both host-device fabrics; live worker-VM qualification remains. | +| ENT-REQ-008 | `K8S42-07` consumes Launch Kit's `gpudirect_dmabuf` matrix, including endpoint GPU topology, bandwidth, threshold, and errors. The integration and mocks are qualified; representative live GPUDirect hardware qualification remains. | +| ENT-REQ-009 | Version, summary, manifest rows, IPAM/network kinds, and test workload readiness are interpreted. | +| ENT-REQ-010 | Partially addressed: teardown-linked `l8k clean` runs after every attempted deployment, including failure, and waits for CR finalizers before Helm uninstall. Full coverage remains deferred because cleanup does not capture and restore pre-test state. | +| ENT-REQ-011/012 | Catalog YAML, IDs, labels, Kubernetes dependencies, descriptions, traceability, and prerequisite documentation exist; typed owner metadata remains a catalog gap. | +| ENT-REQ-013 | Pass/fail/skip, subtests, logs, JUnit, catalog, and local evidence integrate; Labs binary attachment upload remains a gap. | + +The unit tests are cluster-free because they inject test executables and give +every use case an isolated temporary directory. The grouped product YAML is not +cluster-free: it runs six mutation workflows sequentially on one live cluster, +with `l8k clean` between attempted deployments. Use a dedicated qualification +cluster because cleanup intentionally removes the complete Network Operator +deployment boundary and does not restore any installation that existed before +the run. + +## Recommended framework and Launch Kit improvements + +1. Extend lifecycle pruning to structured selectors beyond labels. Mutating + steps can declare `requires_selected_validations`, but raw pytest `-k`/`-m` + expressions remain intentionally downstream of lifecycle execution. +2. Add typed catalog fields for owner, dependencies, applicability, and + `profiles.{era,ncp}.requirement` rather than encoding all policy in labels. +3. Add a redacted attachment manifest with path, media type, checksum, size, + retention, and upload status. Explicitly exclude kubeconfigs, Secrets, + tokens, and private-registry credentials. +4. Extend Launch Kit cleanup with an idempotent snapshot/restore/verify workflow with a + semantic post-restore diff. AI Cloud Validation should orchestrate that API, + not reimplement Network Operator state handling. +5. Replace the unversioned three-document validate stream with one versioned + envelope containing verdict, checks, warnings, artifacts, and report path, + while keeping a compatibility parser for older releases. +6. Add explicit executed/disabled result-family metadata to the versioned + validate envelope so consumers need not infer disabled GPUDirect from an + absent `gpudirect_dmabuf` family. +7. Add nested progress or resumable command events so Labs can display long + deploy/validate execution and retry validation without repeating deployment. + +## Production exit criteria + +1. qualify the generic provider against a released Launch Kit binary and a real + representative cluster; +2. agree on and version the Launch Kit machine-output compatibility contract; +3. publish supported immutable installer refs and SHA-256 values through a + trusted release channel; +4. implement and failure-inject Launch Kit-owned state restoration; +5. add secure evidence attachment upload; +6. define Enterprise and NCP required/optional subsets; +7. live-qualify GPUDirect DMA-BUF across representative GPU/NIC topologies; +8. qualify every required fabric/deployment profile on representative hardware; +9. qualify sequential use cases and failure-inject the linked `l8k clean` teardown on + representative clusters; +10. release the checks through the normal repository release process. diff --git a/docs/packages/isvctl.md b/docs/packages/isvctl.md index 10ba7f3a4..348082ab6 100644 --- a/docs/packages/isvctl.md +++ b/docs/packages/isvctl.md @@ -133,6 +133,27 @@ isvctl test validate -f isvctl/configs/suites/k8s.yaml See [Configuration Guide](../guides/configuration.md) for full details. +All lifecycle and step commands run with captured stdout/stderr and an outer +watchdog. On POSIX, a timeout terminates the command's complete process group +(`SIGTERM`, then `SIGKILL` after a short grace period), which prevents a child +provider CLI from continuing after its wrapper step has timed out. See +[Step Configuration](../guides/configuration.md#step-configuration). + +Cleanup steps may use `phase: teardown` with +`finalizer_for: `. The linked teardown runs directly after the +target's phase validations whenever that command started, including after +target or validation failure, and is reported as `-teardown`. +An explicit teardown-only run executes it as standalone recovery. Cleanup +failure blocks later non-teardown phases. See +[Linked teardown finalizers](../guides/configuration.md#linked-teardown-finalizers) +for activation, ordering, and process-failure limitations. + +Steps gated with `requires_selected_validations` also declare which validation +owns their lifecycle result. If one of those steps fails, the named validation +is emitted as a `step_failed` error in structured results and JUnit even when a +later validation-producing step never runs. See +[Gating mutating steps by test selection](../guides/configuration.md#gating-mutating-steps-by-test-selection). + ### Unified Config Structure ```yaml diff --git a/docs/packages/isvtest.md b/docs/packages/isvtest.md index 32c65483f..8d93e2914 100644 --- a/docs/packages/isvtest.md +++ b/docs/packages/isvtest.md @@ -67,6 +67,25 @@ Utility checks that work with any step output. `SchemaValidation` remains directly wireable, but is catalog-excluded because the step executor runs schema checks automatically. +### Composite checks and nested results + +`CompositeCheck` is existing internal framework machinery used when a suite +declares `compose:`. It is excluded from discovery and the catalog; the named +YAML entry is the test identity. Every member runs, even after an earlier member +fails, and is reported as a parent subtest. Subtests reported by a member are +also forwarded with `MemberName/probe-name` names, so their detailed messages +and timing remain available in pytest and JUnit. If a member calls +`pytest.skip`, `CompositeCheck` records that member as skipped and continues +with the remaining members. A skipped member neither passes nor fails the +composite; the composite passes when every non-skipped member passes. + +The isvctl orchestration summary uses the structured subtest counts to render a +concise line for successful parents. Failure and error messages are never +replaced by that summary. JUnit suite counters are reconciled with the emitted +parent and subtest testcase nodes after injection. See the +[configuration guide](../guides/configuration.md#available-validations) for +YAML and output examples. + | Validation | Platforms | Description | | ---------- | --------- | ----------- | | `StepSuccessCheck` | all | Compose-only: check step completed successfully | diff --git a/docs/requirements/README.md b/docs/requirements/README.md index 7857fa8c5..068a54d7d 100644 --- a/docs/requirements/README.md +++ b/docs/requirements/README.md @@ -15,6 +15,8 @@ reconciles all of these different goals. | `software-reference-requirements.md` | Generated rendering of the reference YAML (`make plan`); one contributing requirements doc among several. | | `storage-acceptance-requirements.yaml` | **Source of record** for the DGXC Storage Acceptance Test requirements (PRD-ref namespace). | | `storage-acceptance-requirements.md` | Generated rendering of the storage YAML (`make plan`). | +| `network-operator-readiness-requirements.yaml` | **Source of record** for the Enterprise RA Network Operator self-validation integration PRD. | +| `network-operator-readiness-requirements.md` | Generated rendering of the Network Operator PRD YAML (`make plan`). | | `test-requirements-matrix.yaml` | The **traceability matrix (index)**: which requirement(s) each test relates to, across documents (`source`). | | `test-requirements-matrix.adoc` | Generated and committed traceability matrix, viewable in github (or renderable to html) | | `../../scripts/reqtrace.py` | Integrity checks (`reqtrace validate`; `make reqcheck`). | @@ -96,6 +98,7 @@ record. | `BFX` (04+) | reference | break-fix health (continues offtake `BFX`) | | `BENCH` | reference | exemplar benchmarking | | `N-*` | storage | Storage Acceptance test IDs | +| `ENT-REQ-*` | network-operator-prd | Enterprise Network Operator self-validation integration | ## 3. `legacy_ids` @@ -124,16 +127,22 @@ belonging to a given requirements document*. We keep the data flexible for this: When a new team's requirements document is blessed, reconcile it here: -1. **Register prefix(es)** for the new doc in the registry (sec. 2). Resolve +1. **Add a structured requirements source.** Give it a globally unique + top-level `source`, because that value is the matrix join key. For a project + PRD, set `format: project-prd` to reuse the generic section/area renderer; + do not add a source-specific renderer branch. Add the file to + `DEFAULT_SOURCES` in `requirements_source_to_md.py` when `make plan` should + render it by default. +2. **Register prefix(es)** for the new doc in the registry (sec. 2). Resolve any overload before proceeding (see the `CP` lesson). -2. **Assign IDs.** Prefer mirroring the upstream requirement IDs. On collision +3. **Assign IDs.** Prefer mirroring the upstream requirement IDs. On collision with an existing prefix, apply the collision policy (sec. 2): continue the number space, or (selectively) choose another resolution and record why. -3. **Add/adjust tests** in `test-plan.yaml` (the canonical truth). Use +4. **Add/adjust tests** in `test-plan.yaml` (the canonical truth). Use `legacy_ids` for any renames. -4. **Update the matrix** (`test-requirements-matrix.yaml`): add each +5. **Update the matrix** (`test-requirements-matrix.yaml`): add each test->requirement edge with the new `source`, plus `annotations`/`notes`. -5. **Validate**: `make reqcheck` must pass; **regenerate**: `make plan`. +6. **Validate**: `make reqcheck` must pass; **regenerate**: `make plan`. > Kept as a subsection for now; promote to its own `ONBOARDING.md` if it grows. diff --git a/docs/requirements/network-operator-readiness-requirements.md b/docs/requirements/network-operator-readiness-requirements.md new file mode 100644 index 000000000..12e0a8485 --- /dev/null +++ b/docs/requirements/network-operator-readiness-requirements.md @@ -0,0 +1,51 @@ + + +# Enterprise RA Network Operator Self-Validation Integration PRD + +> Structured source of record: `network-operator-readiness-requirements.yaml` (version prd-snapshot-2026-08-04). +> Owner: NVIDIA Network Operator team. +> Edit the YAML, not this file. + +## Ownership + +| Req ID | Requirement Area | Description | Status | +| :----- | :--------------- | :---------- | :----- | +| ENT-REQ-000 | Integration ownership | The Network Operator team owns and maintains the integration solution, including compatibility updates for the underlying tests. | active | + +## Framework Integration + +| Req ID | Requirement Area | Description | Status | +| :----- | :--------------- | :---------- | :----- | +| ENT-REQ-001 | Standard validation workflow | Integrate Network Operator self-validation tests into AI Cloud Validation so Enterprise and AI Cloud Ready users can run them through the standard workflow. | active | +| ENT-REQ-002 | Launch Kit reuse | Reuse applicable Kubernetes Launch Kit validation components, including topology discovery, manifest readiness, RDMA connectivity, and RDMA bandwidth. | active | +| ENT-REQ-003 | Selection and program profiles | Support individual and grouped Network Operator validation, with Enterprise and AI Cloud Ready profiles able to mark checks required or optional. | active | +| ENT-REQ-004 | Runtime parameters | Expose applicable runtime parameters such as namespace, node selector, network and driver modes, rail and network names, resource and IP pool names, GPU count, and timeout. | active | + +## Network Validation + +| Req ID | Requirement Area | Description | Status | +| :----- | :--------------- | :---------- | :----- | +| ENT-REQ-005 | Ethernet and RoCE | Validate SR-IOV Network RDMA and RDMA Shared scenarios, including secondary network attachment, RDMA device availability, pod-to-pod RDMA or RoCE connectivity, and basic bandwidth. | active | +| ENT-REQ-006 | InfiniBand | Validate InfiniBand SR-IOV and RDMA Shared with IPoIB scenarios, including IB device availability, pod network attachment, and pod-to-pod InfiniBand connectivity. | active | +| ENT-REQ-007 | Host-device networking | Validate host-device networking for Kubernetes workers running in virtual machines, covering both Ethernet or RoCE and InfiniBand. | active | +| ENT-REQ-008 | GPUDirect RDMA | Validate GPUDirect RDMA peer-to-peer connectivity between GPU-enabled pods across supported worker nodes. | active | +| ENT-REQ-009 | Deployment health | Validate Network Operator deployment health and required resources for the selected mode, including policies, secondary networks, IP pools, Multus and CNI components, and drivers. | active | + +## Lifecycle Safety + +| Req ID | Requirement Area | Description | Status | +| :----- | :--------------- | :---------- | :----- | +| ENT-REQ-010 | State restoration | For tests that modify Network Operator or cluster state, capture the pre-test configuration and restore the original Network Operator state after success or failure. | active | + +## Catalog and Documentation + +| Req ID | Requirement Area | Description | Status | +| :----- | :--------------- | :---------- | :----- | +| ENT-REQ-011 | Catalog metadata | Add catalog entries for all tests with owner, labels, dependencies, descriptions, and required YAML updates while following repository contribution standards. | active | +| ENT-REQ-012 | Prerequisites | Document the required cluster prerequisites for each validation area. | active | + +## Reporting + +| Req ID | Requirement Area | Description | Status | +| :----- | :--------------- | :---------- | :----- | +| ENT-REQ-013 | Results and evidence | Integrate pass or fail status, logs, Launch Kit reports, generated manifests, Kubernetes state, connectivity results, and bandwidth results into AI Cloud Validation reporting, catalog, and AI Cloud Labs artifacts. | active | diff --git a/docs/requirements/network-operator-readiness-requirements.yaml b/docs/requirements/network-operator-readiness-requirements.yaml new file mode 100644 index 000000000..5dc2aabef --- /dev/null +++ b/docs/requirements/network-operator-readiness-requirements.yaml @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Enterprise RA Network Operator self-validation integration requirements. +# Structured from the PRD supplied for the Kubernetes Launch Kit integration. +# Render the publishable Markdown view with `make plan`. + +source: network-operator-prd +format: project-prd +title: Enterprise RA Network Operator Self-Validation Integration PRD +version: prd-snapshot-2026-08-04 +owner: NVIDIA Network Operator team +requirements: + - req_id: ENT-REQ-000 + section: Ownership + area: Integration ownership + description: The Network Operator team owns and maintains the integration solution, including compatibility updates for the underlying tests. + status: active + - req_id: ENT-REQ-001 + section: Framework Integration + area: Standard validation workflow + description: Integrate Network Operator self-validation tests into AI Cloud Validation so Enterprise and AI Cloud Ready users can run them through the standard workflow. + status: active + - req_id: ENT-REQ-002 + section: Framework Integration + area: Launch Kit reuse + description: Reuse applicable Kubernetes Launch Kit validation components, including topology discovery, manifest readiness, RDMA connectivity, and RDMA bandwidth. + status: active + - req_id: ENT-REQ-003 + section: Framework Integration + area: Selection and program profiles + description: Support individual and grouped Network Operator validation, with Enterprise and AI Cloud Ready profiles able to mark checks required or optional. + status: active + - req_id: ENT-REQ-004 + section: Framework Integration + area: Runtime parameters + description: Expose applicable runtime parameters such as namespace, node selector, network and driver modes, rail and network names, resource and IP pool names, GPU count, and timeout. + status: active + - req_id: ENT-REQ-005 + section: Network Validation + area: Ethernet and RoCE + description: Validate SR-IOV Network RDMA and RDMA Shared scenarios, including secondary network attachment, RDMA device availability, pod-to-pod RDMA or RoCE connectivity, and basic bandwidth. + status: active + - req_id: ENT-REQ-006 + section: Network Validation + area: InfiniBand + description: Validate InfiniBand SR-IOV and RDMA Shared with IPoIB scenarios, including IB device availability, pod network attachment, and pod-to-pod InfiniBand connectivity. + status: active + - req_id: ENT-REQ-007 + section: Network Validation + area: Host-device networking + description: Validate host-device networking for Kubernetes workers running in virtual machines, covering both Ethernet or RoCE and InfiniBand. + status: active + - req_id: ENT-REQ-008 + section: Network Validation + area: GPUDirect RDMA + description: Validate GPUDirect RDMA peer-to-peer connectivity between GPU-enabled pods across supported worker nodes. + status: active + - req_id: ENT-REQ-009 + section: Network Validation + area: Deployment health + description: Validate Network Operator deployment health and required resources for the selected mode, including policies, secondary networks, IP pools, Multus and CNI components, and drivers. + status: active + - req_id: ENT-REQ-010 + section: Lifecycle Safety + area: State restoration + description: For tests that modify Network Operator or cluster state, capture the pre-test configuration and restore the original Network Operator state after success or failure. + status: active + - req_id: ENT-REQ-011 + section: Catalog and Documentation + area: Catalog metadata + description: Add catalog entries for all tests with owner, labels, dependencies, descriptions, and required YAML updates while following repository contribution standards. + status: active + - req_id: ENT-REQ-012 + section: Catalog and Documentation + area: Prerequisites + description: Document the required cluster prerequisites for each validation area. + status: active + - req_id: ENT-REQ-013 + section: Reporting + area: Results and evidence + description: Integrate pass or fail status, logs, Launch Kit reports, generated manifests, Kubernetes state, connectivity results, and bandwidth results into AI Cloud Validation reporting, catalog, and AI Cloud Labs artifacts. + status: active diff --git a/docs/requirements/test-requirements-matrix.adoc b/docs/requirements/test-requirements-matrix.adoc index 8161dd761..66095b3c3 100644 --- a/docs/requirements/test-requirements-matrix.adoc +++ b/docs/requirements/test-requirements-matrix.adoc @@ -3495,4 +3495,544 @@ docs/requirements/test-requirements-matrix.yaml. Run `make plan` to regenerate. | full | +| [[K8S42-01]]K8S42-01 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate Network Operator deployment health, versions, policies, Multus, secondary networks, and IP pools +| ENT-REQ-000 +| network-operator-prd +| partial +| + +| K8S42-01 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate Network Operator deployment health, versions, policies, Multus, secondary networks, and IP pools +| ENT-REQ-001 +| network-operator-prd +| full +| + +| K8S42-01 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate Network Operator deployment health, versions, policies, Multus, secondary networks, and IP pools +| ENT-REQ-003 +| network-operator-prd +| partial +| + +| K8S42-01 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate Network Operator deployment health, versions, policies, Multus, secondary networks, and IP pools +| ENT-REQ-009 +| network-operator-prd +| full +| + +| K8S42-01 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate Network Operator deployment health, versions, policies, Multus, secondary networks, and IP pools +| ENT-REQ-011 +| network-operator-prd +| partial +| + +| [[K8S42-02]]K8S42-02 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate the SR-IOV policy and profile-specific secondary network resources reported by Launch Kit +| ENT-REQ-005 +| network-operator-prd +| partial +| + +| [[K8S42-03]]K8S42-03 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate pod-to-pod RDMA-CM connectivity across selected rails +| ENT-REQ-002 +| network-operator-prd +| full +| + +| K8S42-03 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate pod-to-pod RDMA-CM connectivity across selected rails +| ENT-REQ-005 +| network-operator-prd +| partial +| + +| K8S42-03 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate pod-to-pod RDMA-CM connectivity across selected rails +| ENT-REQ-006 +| network-operator-prd +| partial +| + +| [[K8S42-04]]K8S42-04 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate the exact secondary network resource selected for an Ethernet or RoCE profile +| ENT-REQ-005 +| network-operator-prd +| partial +| + +| [[K8S42-05]]K8S42-05 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate the exact secondary network resource selected for an InfiniBand profile +| ENT-REQ-006 +| network-operator-prd +| partial +| + +| [[K8S42-06]]K8S42-06 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate HostDeviceNetwork readiness for an applicable Ethernet or InfiniBand profile +| ENT-REQ-007 +| network-operator-prd +| partial +| + +| [[K8S42-07]]K8S42-07 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate GPUDirect RDMA DMA-BUF bandwidth between GPU-enabled pod endpoints +| ENT-REQ-008 +| network-operator-prd +| partial +| + +| [[K8S42-08]]K8S42-08 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate Launch Kit discovery completed and resolved a fabric and deployment profile +| ENT-REQ-002 +| network-operator-prd +| partial +| + +| K8S42-08 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate Launch Kit discovery completed and resolved a fabric and deployment profile +| ENT-REQ-004 +| network-operator-prd +| partial +| + +| K8S42-08 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate Launch Kit discovery completed and resolved a fabric and deployment profile +| ENT-REQ-012 +| network-operator-prd +| partial +| + +| [[K8S42-09]]K8S42-09 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate IP pool and profile-specific secondary-network resources plus test DaemonSet readiness +| ENT-REQ-005 +| network-operator-prd +| partial +| + +| K8S42-09 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate IP pool and profile-specific secondary-network resources plus test DaemonSet readiness +| ENT-REQ-006 +| network-operator-prd +| partial +| + +| K8S42-09 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate IP pool and profile-specific secondary-network resources plus test DaemonSet readiness +| ENT-REQ-009 +| network-operator-prd +| partial +| + +| [[K8S42-10]]K8S42-10 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate Macvlan or IPoIB network readiness for an applicable RDMA Shared profile +| ENT-REQ-005 +| network-operator-prd +| partial +| + +| K8S42-10 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate Macvlan or IPoIB network readiness for an applicable RDMA Shared profile +| ENT-REQ-006 +| network-operator-prd +| partial +| + +| [[K8S42-11]]K8S42-11 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate source-bound same-rail ICMP connectivity and strict cross-rail isolation +| ENT-REQ-002 +| network-operator-prd +| partial +| + +| K8S42-11 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate source-bound same-rail ICMP connectivity and strict cross-rail isolation +| ENT-REQ-005 +| network-operator-prd +| partial +| + +| K8S42-11 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate source-bound same-rail ICMP connectivity and strict cross-rail isolation +| ENT-REQ-006 +| network-operator-prd +| partial +| + +| [[K8S42-12]]K8S42-12 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate every ib_write_bw result and report its Launch Kit-resolved minimum bandwidth +| ENT-REQ-002 +| network-operator-prd +| partial +| + +| K8S42-12 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate every ib_write_bw result and report its Launch Kit-resolved minimum bandwidth +| ENT-REQ-005 +| network-operator-prd +| partial +| + +| K8S42-12 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate every ib_write_bw result and report its Launch Kit-resolved minimum bandwidth +| ENT-REQ-006 +| network-operator-prd +| partial +| + +| [[K8S42-13]]K8S42-13 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate same-rail and cross-rail coverage for a multi-rail Launch Kit profile +| ENT-REQ-002 +| network-operator-prd +| partial +| + +| [[K8S42-15]]K8S42-15 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Verify raw phase output, generated files, connectivity results, and Launch Kit reports were preserved +| ENT-REQ-011 +| network-operator-prd +| partial +| + +| K8S42-15 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Verify raw phase output, generated files, connectivity results, and Launch Kit reports were preserved +| ENT-REQ-013 +| network-operator-prd +| partial +| + +| [[K8S42-16]]K8S42-16 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for an Ethernet or RoCE SR-IOV Network RDMA profile +| ENT-REQ-001 +| network-operator-prd +| partial +| + +| K8S42-16 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for an Ethernet or RoCE SR-IOV Network RDMA profile +| ENT-REQ-003 +| network-operator-prd +| partial +| + +| K8S42-16 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for an Ethernet or RoCE SR-IOV Network RDMA profile +| ENT-REQ-005 +| network-operator-prd +| partial +| + +| K8S42-16 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for an Ethernet or RoCE SR-IOV Network RDMA profile +| ENT-REQ-008 +| network-operator-prd +| partial +| + +| K8S42-16 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for an Ethernet or RoCE SR-IOV Network RDMA profile +| ENT-REQ-013 +| network-operator-prd +| partial +| + +| [[K8S42-17]]K8S42-17 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for an InfiniBand SR-IOV Network RDMA profile +| ENT-REQ-001 +| network-operator-prd +| partial +| + +| K8S42-17 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for an InfiniBand SR-IOV Network RDMA profile +| ENT-REQ-003 +| network-operator-prd +| partial +| + +| K8S42-17 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for an InfiniBand SR-IOV Network RDMA profile +| ENT-REQ-006 +| network-operator-prd +| partial +| + +| K8S42-17 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for an InfiniBand SR-IOV Network RDMA profile +| ENT-REQ-008 +| network-operator-prd +| partial +| + +| K8S42-17 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for an InfiniBand SR-IOV Network RDMA profile +| ENT-REQ-013 +| network-operator-prd +| partial +| + +| [[K8S42-18]]K8S42-18 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for an Ethernet or RoCE RDMA Shared profile +| ENT-REQ-001 +| network-operator-prd +| partial +| + +| K8S42-18 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for an Ethernet or RoCE RDMA Shared profile +| ENT-REQ-003 +| network-operator-prd +| partial +| + +| K8S42-18 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for an Ethernet or RoCE RDMA Shared profile +| ENT-REQ-005 +| network-operator-prd +| partial +| + +| K8S42-18 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for an Ethernet or RoCE RDMA Shared profile +| ENT-REQ-008 +| network-operator-prd +| partial +| + +| K8S42-18 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for an Ethernet or RoCE RDMA Shared profile +| ENT-REQ-013 +| network-operator-prd +| partial +| + +| [[K8S42-19]]K8S42-19 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for an InfiniBand or IPoIB RDMA Shared profile +| ENT-REQ-001 +| network-operator-prd +| partial +| + +| K8S42-19 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for an InfiniBand or IPoIB RDMA Shared profile +| ENT-REQ-003 +| network-operator-prd +| partial +| + +| K8S42-19 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for an InfiniBand or IPoIB RDMA Shared profile +| ENT-REQ-006 +| network-operator-prd +| partial +| + +| K8S42-19 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for an InfiniBand or IPoIB RDMA Shared profile +| ENT-REQ-008 +| network-operator-prd +| partial +| + +| K8S42-19 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for an InfiniBand or IPoIB RDMA Shared profile +| ENT-REQ-013 +| network-operator-prd +| partial +| + +| [[K8S42-20]]K8S42-20 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for Ethernet or RoCE host-device networking on worker VMs +| ENT-REQ-001 +| network-operator-prd +| partial +| + +| K8S42-20 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for Ethernet or RoCE host-device networking on worker VMs +| ENT-REQ-003 +| network-operator-prd +| partial +| + +| K8S42-20 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for Ethernet or RoCE host-device networking on worker VMs +| ENT-REQ-007 +| network-operator-prd +| partial +| + +| K8S42-20 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for Ethernet or RoCE host-device networking on worker VMs +| ENT-REQ-008 +| network-operator-prd +| partial +| + +| K8S42-20 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for Ethernet or RoCE host-device networking on worker VMs +| ENT-REQ-013 +| network-operator-prd +| partial +| + +| [[K8S42-21]]K8S42-21 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for InfiniBand host-device networking on worker VMs +| ENT-REQ-001 +| network-operator-prd +| partial +| + +| K8S42-21 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for InfiniBand host-device networking on worker VMs +| ENT-REQ-003 +| network-operator-prd +| partial +| + +| K8S42-21 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for InfiniBand host-device networking on worker VMs +| ENT-REQ-007 +| network-operator-prd +| partial +| + +| K8S42-21 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for InfiniBand host-device networking on worker VMs +| ENT-REQ-008 +| network-operator-prd +| partial +| + +| K8S42-21 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the complete Launch Kit workflow for InfiniBand host-device networking on worker VMs +| ENT-REQ-013 +| network-operator-prd +| partial +| + |=== diff --git a/docs/requirements/test-requirements-matrix.yaml b/docs/requirements/test-requirements-matrix.yaml index 13047e0e1..1aba3bb33 100644 --- a/docs/requirements/test-requirements-matrix.yaml +++ b/docs/requirements/test-requirements-matrix.yaml @@ -8,7 +8,7 @@ # # Per mapping: # test_id - matches docs/test-plan.yaml -# requirements - list of { req_id, source: offtake|reference, coverage: full|partial } +# requirements - list of { req_id, source: , coverage: full|partial } # annotations - free-form (e.g. how this relationship was decided) # notes - free-form scratch # @@ -2561,3 +2561,263 @@ mappings: coverage: full annotations: '' notes: '' + - test_id: K8S42-01 + requirements: + - req_id: ENT-REQ-000 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-001 + source: network-operator-prd + coverage: full + - req_id: ENT-REQ-003 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-009 + source: network-operator-prd + coverage: full + - req_id: ENT-REQ-011 + source: network-operator-prd + coverage: partial + annotations: 'Production provider establishes the integration and catalog boundary; mock-backed unit tests exercise it; long-term ownership and typed program policy metadata are not runtime assertions.' + notes: '' + - test_id: K8S42-02 + requirements: + - req_id: ENT-REQ-005 + source: network-operator-prd + coverage: partial + annotations: 'Covers SR-IOV attachment and device readiness; connectivity and bandwidth are separate selectable checks.' + notes: '' + - test_id: K8S42-03 + requirements: + - req_id: ENT-REQ-002 + source: network-operator-prd + coverage: full + - req_id: ENT-REQ-005 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-006 + source: network-operator-prd + coverage: partial + annotations: 'Reuses the Launch Kit rping matrix for both RoCE and InfiniBand profiles.' + notes: '' + - test_id: K8S42-04 + requirements: + - req_id: ENT-REQ-005 + source: network-operator-prd + coverage: partial + annotations: 'Aggregates the Ethernet and RoCE profile results.' + notes: '' + - test_id: K8S42-05 + requirements: + - req_id: ENT-REQ-006 + source: network-operator-prd + coverage: partial + annotations: 'Checks the profile-specific InfiniBand network kind; device, attachment, and connectivity evidence is split across other checks.' + notes: '' + - test_id: K8S42-06 + requirements: + - req_id: ENT-REQ-007 + source: network-operator-prd + coverage: partial + annotations: 'Production wiring and unit fixtures cover both host-device profile contracts; live VM qualification remains required.' + notes: '' + - test_id: K8S42-07 + requirements: + - req_id: ENT-REQ-008 + source: network-operator-prd + coverage: partial + annotations: 'Consumes Launch Kit gpudirect_dmabuf matrix verdicts with endpoint GPU indices, PCI addresses, bandwidth, threshold, and errors; mock-qualified only and still requires live GPUDirect hardware qualification.' + notes: '' + - test_id: K8S42-08 + requirements: + - req_id: ENT-REQ-002 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-004 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-012 + source: network-operator-prd + coverage: partial + annotations: 'Forwards user-owned discover arguments without copying Launch Kit defaults; live prerequisites are documented separately.' + notes: '' + - test_id: K8S42-09 + requirements: + - req_id: ENT-REQ-005 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-006 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-009 + source: network-operator-prd + coverage: partial + annotations: 'Checks generated secondary-network, IP pool, Multus attachment, and pod-address evidence.' + notes: '' + - test_id: K8S42-10 + requirements: + - req_id: ENT-REQ-005 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-006 + source: network-operator-prd + coverage: partial + annotations: 'Covers Macvlan and IPoIB RDMA Shared profiles.' + notes: '' + - test_id: K8S42-11 + requirements: + - req_id: ENT-REQ-002 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-005 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-006 + source: network-operator-prd + coverage: partial + annotations: 'Reuses Launch Kit strict ICMP same-rail reachability and cross-rail isolation results.' + notes: '' + - test_id: K8S42-12 + requirements: + - req_id: ENT-REQ-002 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-005 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-006 + source: network-operator-prd + coverage: partial + annotations: 'Reuses Launch Kit ib_write_bw verdicts and reports the observed and Launch Kit-resolved minimum bandwidth.' + notes: '' + - test_id: K8S42-13 + requirements: + - req_id: ENT-REQ-002 + source: network-operator-prd + coverage: partial + annotations: 'Checks the complete two-rail strict connectivity matrix.' + notes: '' + - test_id: K8S42-15 + requirements: + - req_id: ENT-REQ-011 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-013 + source: network-operator-prd + coverage: partial + annotations: 'Local evidence and framework reporting are implemented; typed catalog ownership and Labs binary attachment upload remain gaps.' + notes: '' + - test_id: K8S42-16 + requirements: + - req_id: ENT-REQ-001 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-003 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-005 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-008 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-013 + source: network-operator-prd + coverage: partial + annotations: 'Concrete RoCE SR-IOV discover-generate-deploy-validate use case; live-qualified on a two-node Ubuntu 24.04 single-rail cluster, with the multi-rail member skipped as inapplicable.' + notes: '' + - test_id: K8S42-17 + requirements: + - req_id: ENT-REQ-001 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-003 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-006 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-008 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-013 + source: network-operator-prd + coverage: partial + annotations: 'Concrete InfiniBand SR-IOV discover-generate-deploy-validate use case; mock-qualified only.' + notes: '' + - test_id: K8S42-18 + requirements: + - req_id: ENT-REQ-001 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-003 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-005 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-008 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-013 + source: network-operator-prd + coverage: partial + annotations: 'Concrete RoCE RDMA Shared discover-generate-deploy-validate use case; mock-qualified only.' + notes: '' + - test_id: K8S42-19 + requirements: + - req_id: ENT-REQ-001 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-003 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-006 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-008 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-013 + source: network-operator-prd + coverage: partial + annotations: 'Concrete InfiniBand and IPoIB RDMA Shared discover-generate-deploy-validate use case; mock-qualified only.' + notes: '' + - test_id: K8S42-20 + requirements: + - req_id: ENT-REQ-001 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-003 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-007 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-008 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-013 + source: network-operator-prd + coverage: partial + annotations: 'Concrete RoCE host-device workflow; mock-qualified and not yet proven on worker VMs.' + notes: '' + - test_id: K8S42-21 + requirements: + - req_id: ENT-REQ-001 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-003 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-007 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-008 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-013 + source: network-operator-prd + coverage: partial + annotations: 'Concrete InfiniBand host-device workflow; mock-qualified and not yet proven on worker VMs.' + notes: '' diff --git a/docs/test-plan.adoc b/docs/test-plan.adoc index d0631bb83..75d3248bd 100644 --- a/docs/test-plan.adoc +++ b/docs/test-plan.adoc @@ -2189,7 +2189,7 @@ a| | [[BFX04-01]]BFX04-01 | BFX04 | -| +| bare_metal, breakfix | | P1 a| @@ -2203,7 +2203,7 @@ a| | [[BFX05-01]]BFX05-01 | BFX05 | -| +| bare_metal, breakfix | | a| @@ -2217,7 +2217,7 @@ a| | [[BFX06-01]]BFX06-01 | BFX06 | -| +| bare_metal, breakfix | | a| @@ -2233,7 +2233,7 @@ a| | [[BFX01-01]]BFX01-01 | BFX01 | https://github.com/NVIDIA/ai-cloud-validation/issues/206[#206] -| min_req +| breakfix, kubernetes, min_req | | P2 a| https://github.com/NVIDIA/ai-cloud-validation/issues/206[#206] @@ -2247,7 +2247,7 @@ a| https://github.com/NVIDIA/ai-cloud-validation/issues/206[#206] | [[BFX01-02]]BFX01-02 | BFX01 | https://github.com/NVIDIA/ai-cloud-validation/issues/207[#207] -| min_req +| bare_metal, breakfix, min_req | | P1 a| https://github.com/NVIDIA/ai-cloud-validation/issues/207[#207] @@ -2261,7 +2261,7 @@ a| https://github.com/NVIDIA/ai-cloud-validation/issues/207[#207] | [[BFX01-03]]BFX01-03 | BFX01 | https://github.com/NVIDIA/ai-cloud-validation/issues/208[#208] -| min_req +| bare_metal, breakfix, min_req | | P1 a| https://github.com/NVIDIA/ai-cloud-validation/issues/208[#208] @@ -2275,7 +2275,7 @@ a| https://github.com/NVIDIA/ai-cloud-validation/issues/208[#208] | [[BFX01-04]]BFX01-04 | BFX01 | https://github.com/NVIDIA/ai-cloud-validation/issues/209[#209] -| min_req +| breakfix, kubernetes, min_req | | P1 a| https://github.com/NVIDIA/ai-cloud-validation/issues/209[#209] @@ -2289,7 +2289,7 @@ a| https://github.com/NVIDIA/ai-cloud-validation/issues/209[#209] | [[BFX01-05]]BFX01-05 | BFX01 | https://github.com/NVIDIA/ai-cloud-validation/issues/210[#210] -| min_req +| bare_metal, breakfix, min_req | | P1 a| https://github.com/NVIDIA/ai-cloud-validation/issues/210[#210] @@ -2305,7 +2305,7 @@ a| https://github.com/NVIDIA/ai-cloud-validation/issues/210[#210] | [[BFX02-01]]BFX02-01 | BFX02 | https://github.com/NVIDIA/ai-cloud-validation/issues/211[#211] -| min_req +| bare_metal, breakfix, min_req | | P1 a| https://github.com/NVIDIA/ai-cloud-validation/issues/211[#211] @@ -2319,7 +2319,7 @@ a| https://github.com/NVIDIA/ai-cloud-validation/issues/211[#211] | [[BFX02-02]]BFX02-02 | BFX02 | https://github.com/NVIDIA/ai-cloud-validation/issues/212[#212] -| min_req +| bare_metal, breakfix, min_req | | P1 a| https://github.com/NVIDIA/ai-cloud-validation/issues/212[#212] @@ -2333,7 +2333,7 @@ a| https://github.com/NVIDIA/ai-cloud-validation/issues/212[#212] | [[BFX02-03]]BFX02-03 | BFX02 | https://github.com/NVIDIA/ai-cloud-validation/issues/213[#213] -| min_req +| bare_metal, breakfix, min_req | | P1 a| https://github.com/NVIDIA/ai-cloud-validation/issues/213[#213] @@ -2363,7 +2363,7 @@ a| https://github.com/NVIDIA/ai-cloud-validation/issues/320[#320] | [[BFX03-02]]BFX03-02 | BFX03 | https://github.com/NVIDIA/ai-cloud-validation/issues/214[#214] -| min_req +| bare_metal, breakfix, min_req | | P1 a| https://github.com/NVIDIA/ai-cloud-validation/issues/214[#214] @@ -2377,7 +2377,7 @@ a| https://github.com/NVIDIA/ai-cloud-validation/issues/214[#214] | [[BFX03-03]]BFX03-03 | BFX03 | https://github.com/NVIDIA/ai-cloud-validation/issues/215[#215] -| min_req +| bare_metal, breakfix, min_req | | P1 a| https://github.com/NVIDIA/ai-cloud-validation/issues/215[#215] @@ -3222,7 +3222,7 @@ a| | pending | -.58+| Workload Orchestration +.78+| Workload Orchestration | Backup and Recovery | Centralized managed service to automate and govern data backup across services | AWS backup @@ -3400,7 +3400,7 @@ a| | published | -.45+| Managed Kubernetes Control Plane +.65+| Managed Kubernetes Control Plane | Tenant-isolated Kubernetes control planes for managing k8s workloads. does placement, networking, lifecyle mgmt. | AWS EKS, GCP GKE | [[K8S05-01]]K8S05-01 @@ -3763,6 +3763,288 @@ a| https://github.com/NVIDIA/ai-cloud-validation/issues/220[#220] | pending | +.20+| Network Operator self-validation through Kubernetes Launch Kit +.20+| +| [[K8S42-01]]K8S42-01 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Production provider integration and unit coverage for ENT-REQ-001, ENT-REQ-009, and ENT-REQ-011 +| P0 +a| +| LimitedEnv +| +| +| Validate Network Operator deployment health, versions, policies, Multus, secondary networks, and IP pools +| pending +| + +| [[K8S42-02]]K8S42-02 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for ENT-REQ-005 +| P0 +a| +| LimitedEnv +| +| +| Validate the SR-IOV policy and profile-specific secondary network resources reported by Launch Kit +| pending +| + +| [[K8S42-03]]K8S42-03 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for ENT-REQ-002 and ENT-REQ-005 +| P0 +a| +| LimitedEnv +| +| +| Validate pod-to-pod RDMA-CM connectivity across selected rails +| pending +| + +| [[K8S42-04]]K8S42-04 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for ENT-REQ-005 +| P0 +a| +| LimitedEnv +| +| +| Validate the exact secondary network resource selected for an Ethernet or RoCE profile +| pending +| + +| [[K8S42-05]]K8S42-05 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for ENT-REQ-006 +| P0 +a| +| LimitedEnv +| +| +| Validate the exact secondary network resource selected for an InfiniBand profile +| pending +| + +| [[K8S42-06]]K8S42-06 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for ENT-REQ-007 +| P0 +a| +| LimitedEnv +| +| +| Validate HostDeviceNetwork readiness for an applicable Ethernet or InfiniBand profile +| pending +| + +| [[K8S42-07]]K8S42-07 +| K8S42 +| +| era, gpudirect, kubernetes, ncp, network_operator, slow +| Launch Kit gpudirect_dmabuf result-family integration and mock-backed unit coverage for ENT-REQ-008; live GPU qualification remains required +| P0 +a| +| LimitedEnv +| +| +| Validate GPUDirect RDMA DMA-BUF bandwidth between GPU-enabled pod endpoints +| pending +| + +| [[K8S42-08]]K8S42-08 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for ENT-REQ-002, ENT-REQ-004, and ENT-REQ-012 +| P0 +a| +| LimitedEnv +| +| +| Validate Launch Kit discovery completed and resolved a fabric and deployment profile +| pending +| + +| [[K8S42-09]]K8S42-09 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for ENT-REQ-005, ENT-REQ-006, and ENT-REQ-009 +| P0 +a| +| LimitedEnv +| +| +| Validate IP pool and profile-specific secondary-network resources plus test DaemonSet readiness +| pending +| + +| [[K8S42-10]]K8S42-10 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for ENT-REQ-005 and ENT-REQ-006 +| P0 +a| +| LimitedEnv +| +| +| Validate Macvlan or IPoIB network readiness for an applicable RDMA Shared profile +| pending +| + +| [[K8S42-11]]K8S42-11 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for ENT-REQ-002, ENT-REQ-005, and ENT-REQ-006 +| P0 +a| +| LimitedEnv +| +| +| Validate source-bound same-rail ICMP connectivity and strict cross-rail isolation +| pending +| + +| [[K8S42-12]]K8S42-12 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for ENT-REQ-002, ENT-REQ-005, and ENT-REQ-006 +| P0 +a| +| LimitedEnv +| +| +| Validate every ib_write_bw result and report its Launch Kit-resolved minimum bandwidth +| pending +| + +| [[K8S42-13]]K8S42-13 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for Enterprise multi-rail coverage in ENT-REQ-002 +| P0 +a| +| LimitedEnv +| +| +| Validate same-rail and cross-rail coverage for a multi-rail Launch Kit profile +| pending +| + +| [[K8S42-15]]K8S42-15 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Local evidence and framework reporting cover ENT-REQ-013; Labs attachment upload remains required +| P0 +a| +| LimitedEnv +| +| +| Verify raw phase output, generated files, connectivity results, and Launch Kit reports were preserved +| pending +| + +| [[K8S42-16]]K8S42-16 +| K8S42 +| +| era, ethernet, gpudirect, kubernetes, ncp, network_operator, network_operator_use_cases, roce, slow, sriov +| Composite E2E use case: live-qualified on a two-node Ubuntu 24.04 single-rail cluster; the multi-rail member is skipped as inapplicable +| P0 +a| +| LimitedEnv +| +| +| Run the complete Launch Kit workflow for an Ethernet or RoCE SR-IOV Network RDMA profile +| pending +| + +| [[K8S42-17]]K8S42-17 +| K8S42 +| +| era, gpudirect, infiniband, kubernetes, ncp, network_operator, network_operator_use_cases, slow, sriov +| Composite E2E use case: preflight, discover, generate, deploy, and validate +| P0 +a| +| LimitedEnv +| +| +| Run the complete Launch Kit workflow for an InfiniBand SR-IOV Network RDMA profile +| pending +| + +| [[K8S42-18]]K8S42-18 +| K8S42 +| +| era, ethernet, gpudirect, kubernetes, ncp, network_operator, network_operator_use_cases, rdma_shared, roce, slow +| Composite E2E use case: preflight, discover, generate, deploy, and validate +| P0 +a| +| LimitedEnv +| +| +| Run the complete Launch Kit workflow for an Ethernet or RoCE RDMA Shared profile +| pending +| + +| [[K8S42-19]]K8S42-19 +| K8S42 +| +| era, gpudirect, infiniband, kubernetes, ncp, network_operator, network_operator_use_cases, rdma_shared, slow +| Composite E2E use case: preflight, discover, generate, deploy, and validate +| P0 +a| +| LimitedEnv +| +| +| Run the complete Launch Kit workflow for an InfiniBand or IPoIB RDMA Shared profile +| pending +| + +| [[K8S42-20]]K8S42-20 +| K8S42 +| +| era, ethernet, gpudirect, host_device, kubernetes, ncp, network_operator, network_operator_use_cases, roce, slow +| Composite E2E use case: preflight, discover, generate, deploy, and validate; live VM qualification remains required +| P0 +a| +| LimitedEnv +| +| +| Run the complete Launch Kit workflow for Ethernet or RoCE host-device networking on worker VMs +| pending +| + +| [[K8S42-21]]K8S42-21 +| K8S42 +| +| era, gpudirect, host_device, infiniband, kubernetes, ncp, network_operator, network_operator_use_cases, slow +| Composite E2E use case: preflight, discover, generate, deploy, and validate; live VM qualification remains required +| P0 +a| +| LimitedEnv +| +| +| Run the complete Launch Kit workflow for InfiniBand host-device networking on worker VMs +| pending +| + .2+| K8s Versioning & Compliance .2+| | [[K8S02-01]]K8S02-01 diff --git a/docs/test-plan.yaml b/docs/test-plan.yaml index dd14fdbd3..08c52d895 100644 --- a/docs/test-plan.yaml +++ b/docs/test-plan.yaml @@ -3358,6 +3358,336 @@ domains: milestone: M5 github_issues: - "#220" + - description: Network Operator self-validation through Kubernetes Launch Kit + tests: + - summary: Validate Network Operator deployment health, versions, policies, Multus, secondary networks, and IP pools + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-01 + status: pending + notes: "Production provider integration and unit coverage for ENT-REQ-001, ENT-REQ-009, and ENT-REQ-011" + - summary: Validate the SR-IOV policy and profile-specific secondary network resources reported by Launch Kit + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-02 + status: pending + notes: "Provider wiring and unit coverage for ENT-REQ-005" + - summary: Validate pod-to-pod RDMA-CM connectivity across selected rails + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-03 + status: pending + notes: "Provider wiring and unit coverage for ENT-REQ-002 and ENT-REQ-005" + - summary: Validate the exact secondary network resource selected for an Ethernet or RoCE profile + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-04 + status: pending + notes: "Provider wiring and unit coverage for ENT-REQ-005" + - summary: Validate the exact secondary network resource selected for an InfiniBand profile + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-05 + status: pending + notes: "Provider wiring and unit coverage for ENT-REQ-006" + - summary: Validate HostDeviceNetwork readiness for an applicable Ethernet or InfiniBand profile + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-06 + status: pending + notes: "Provider wiring and unit coverage for ENT-REQ-007" + - summary: Validate GPUDirect RDMA DMA-BUF bandwidth between GPU-enabled pod endpoints + labels: + - era + - gpudirect + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-07 + status: pending + notes: "Launch Kit gpudirect_dmabuf result-family integration and mock-backed unit coverage for ENT-REQ-008; live GPU qualification remains required" + - summary: Validate Launch Kit discovery completed and resolved a fabric and deployment profile + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-08 + status: pending + notes: "Provider wiring and unit coverage for ENT-REQ-002, ENT-REQ-004, and ENT-REQ-012" + - summary: Validate IP pool and profile-specific secondary-network resources plus test DaemonSet readiness + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-09 + status: pending + notes: "Provider wiring and unit coverage for ENT-REQ-005, ENT-REQ-006, and ENT-REQ-009" + - summary: Validate Macvlan or IPoIB network readiness for an applicable RDMA Shared profile + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-10 + status: pending + notes: "Provider wiring and unit coverage for ENT-REQ-005 and ENT-REQ-006" + - summary: Validate source-bound same-rail ICMP connectivity and strict cross-rail isolation + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-11 + status: pending + notes: "Provider wiring and unit coverage for ENT-REQ-002, ENT-REQ-005, and ENT-REQ-006" + - summary: Validate every ib_write_bw result and report its Launch Kit-resolved minimum bandwidth + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-12 + status: pending + notes: "Provider wiring and unit coverage for ENT-REQ-002, ENT-REQ-005, and ENT-REQ-006" + - summary: Validate same-rail and cross-rail coverage for a multi-rail Launch Kit profile + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-13 + status: pending + notes: "Provider wiring and unit coverage for Enterprise multi-rail coverage in ENT-REQ-002" + - summary: Verify raw phase output, generated files, connectivity results, and Launch Kit reports were preserved + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-15 + status: pending + notes: "Local evidence and framework reporting cover ENT-REQ-013; Labs attachment upload remains required" + - summary: Run the complete Launch Kit workflow for an Ethernet or RoCE SR-IOV Network RDMA profile + labels: + - era + - ethernet + - gpudirect + - kubernetes + - ncp + - network_operator + - network_operator_use_cases + - roce + - slow + - sriov + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-16 + status: pending + notes: "Composite E2E use case: live-qualified on a two-node Ubuntu 24.04 single-rail cluster; the multi-rail member is skipped as inapplicable" + - summary: Run the complete Launch Kit workflow for an InfiniBand SR-IOV Network RDMA profile + labels: + - era + - gpudirect + - infiniband + - kubernetes + - ncp + - network_operator + - network_operator_use_cases + - slow + - sriov + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-17 + status: pending + notes: "Composite E2E use case: preflight, discover, generate, deploy, and validate" + - summary: Run the complete Launch Kit workflow for an Ethernet or RoCE RDMA Shared profile + labels: + - era + - ethernet + - gpudirect + - kubernetes + - ncp + - network_operator + - network_operator_use_cases + - rdma_shared + - roce + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-18 + status: pending + notes: "Composite E2E use case: preflight, discover, generate, deploy, and validate" + - summary: Run the complete Launch Kit workflow for an InfiniBand or IPoIB RDMA Shared profile + labels: + - era + - gpudirect + - infiniband + - kubernetes + - ncp + - network_operator + - network_operator_use_cases + - rdma_shared + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-19 + status: pending + notes: "Composite E2E use case: preflight, discover, generate, deploy, and validate" + - summary: Run the complete Launch Kit workflow for Ethernet or RoCE host-device networking on worker VMs + labels: + - era + - ethernet + - gpudirect + - host_device + - kubernetes + - ncp + - network_operator + - network_operator_use_cases + - roce + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-20 + status: pending + notes: "Composite E2E use case: preflight, discover, generate, deploy, and validate; live VM qualification remains required" + - summary: Run the complete Launch Kit workflow for InfiniBand host-device networking on worker VMs + labels: + - era + - gpudirect + - host_device + - infiniband + - kubernetes + - ncp + - network_operator + - network_operator_use_cases + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-21 + status: pending + notes: "Composite E2E use case: preflight, discover, generate, deploy, and validate; live VM qualification remains required" - description: "K8s Versioning & Compliance" tests: - summary: Verify support for the three most recent minor releases (N-2) diff --git a/isvctl/configs/providers/k8s-launch-kit/README.md b/isvctl/configs/providers/k8s-launch-kit/README.md new file mode 100644 index 000000000..f23cb3ed0 --- /dev/null +++ b/isvctl/configs/providers/k8s-launch-kit/README.md @@ -0,0 +1,89 @@ + + + +# Kubernetes Launch Kit provider internals + +This directory owns the implementation behind +`config/provider.yaml`. It is provider-specific code, not a cross-provider +helper. + +## Layout + +| Path | Purpose | +|---|---| +| `config/provider.yaml` | Generic single-workflow provider using real `l8k` and `kubectl` by default | +| `config/network-operator.yaml` | Production six-use-case Network Operator workflow | +| `scripts/adapter.py` | Transport for install/verify, Kubernetes preflight, and one `l8k` workflow command | + +Test doubles and pinned scenario data intentionally live outside the shipped +provider under `isvctl/tests/providers/k8s_launch_kit/fixtures/`. The provider +tests load the production YAML and inject those paths in memory. + +The adapter must remain thin. It accepts raw argument arrays for `discover`, +`generate`, `deploy`, `validate`, and `clean`, appends `--output json`, executes the +configured `l8k` executable, and preserves the CLI's JSON documents without +renaming or interpreting fields. The one file-level input is `user_config`, a +path to a complete Launch Kit configuration. Before discovery, the adapter +copies it to the workflow as a mode-`0600` `user-config.yaml`, explicitly writes +the discovered result to `cluster-config.yaml`, and removes the staged input as +soon as discovery exits. The original is never modified, and evidence retains +only its path, size, and SHA-256 provenance rather than its potentially +sensitive contents. Launch Kit still owns the file schema, domain flags, and +defaults. Semantic assertions belong in `isvtest.validations.k8s_launch_kit`. + +Launch Kit `validate` steps use `timeout: null` so the CLI owns its deadline. +l8k calculates and logs a bounded matrix budget by default and honors a user's +explicit `--connectivity-timeout`. The remaining workflow steps keep finite +isvctl watchdogs. Other providers may also use `timeout: null`, but only when +their child command has its own bounded timeout. + +The grouped Network Operator workflow passes only its fabric and deployment +identity during discovery. With no `user_config`, Launch Kit resolves the +default `./cluster-config.yaml` and `./deployment` paths throughout the +workflow. With `user_config`, every selected use case stages an independent +copy, and the adapter owns `--user-config` plus `--save-cluster-config` for +discovery. Each transient copy is deleted after its discovery command. Do not +repeat either flag in the raw discovery argument array or place the source +inside the retained provider working directory. + +Each `clean` step is declared in `phase: teardown` and linked to its matching +`deploy` step with `finalizer_for`. During a normal or `--phase test` run, the +orchestrator executes it directly after that use case's validations and reports +an explicit `-teardown` phase. It runs whenever deployment actually +started, including after a failed deploy or validate. It is not activated when +preflight, discovery, generation, template rendering, or process startup stops +the workflow before deployment. Cleanup failures block later use cases even +though ordinary use-case failures may continue. An explicit `--phase teardown` +run invokes the selected clean steps as a standalone recovery workflow. + +Each workflow envelope records the absolute working directory while retaining +Launch Kit's JSON documents unchanged. Validations use that metadata to resolve +relative `generatedFiles` paths emitted by the CLI. + +Install mode accepts only an immutable full Git commit for the official +`scripts/install.sh` plus a caller-supplied SHA-256, verifies that digest before +writing or executing the script, delegates archive selection and checksum +handling to Launch Kit, then verifies the binary at the install prefix. +When the user pins `installation.version`, both setup and test-phase +verification require `l8k version --output json` to report that exact version. +The captured schema must advertise all five workflow commands, including +`clean`, so an older binary is rejected before deployment. + +The preflight uses the same explicit kubeconfig and forwarded environment as +the Launch Kit workflow. It rejects conflicting `--kubeconfig` arguments and +requires Kubernetes API access plus at least one Ready node before a normal +use case can mutate the cluster. Teardown-only recovery invokes `clean` +directly so cleanup remains available when test prerequisites do not pass. + +The same string-only environment mapping is also passed to the installer and +version/schema verification, so proxy and executable runtime settings do not +change between setup and test phases. + +The production adapter executes `executable` directly. There is no Python-file +special case: a test double must be an executable with a valid shebang, just +like any other CLI implementation. This keeps mock behavior out of the public +provider contract. + +See the [integration guide](../../../../docs/guides/k8s-launch-kit/network-operator.md) +for configuration, use cases, evidence, prerequisites, and current production +gaps. diff --git a/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml b/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml new file mode 100644 index 000000000..9e48593ee --- /dev/null +++ b/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml @@ -0,0 +1,992 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Production Network Operator workflow. It invokes the real l8k and kubectl +# executables inherited from provider.yaml. Test doubles live only under tests/. +# +# Usage: +# ISVTEST_INCLUDE_UNRELEASED=1 uv run isvctl test run \ +# -f isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml \ +# --capability kubernetes --no-upload -- -v + +import: + - provider.yaml + - ../../../suites/k8s-launch-kit/network-operator-use-cases.yaml + +version: "1.0" + +context: + k8s_launch_kit: + executable: l8k + installation: + mode: verify + version: "" + installer_ref: "" + installer_sha256: "" + prefix: "" + # Optional complete Launch Kit configuration. Every selected use case + # stages its own copy before discovery; the source file is never modified. + user_config: "" + # An empty override means the adapter invokes kubectl from PATH. Users may + # replace this with any kubectl-compatible argv list in an overlay. + kubectl_command: [] + environment: {} + shared_artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/shared-evidence + use_cases: + roce_sriov: + working_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/roce-sriov/work + artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/roce-sriov/evidence + discover: + arguments: + - --fabric + - ethernet + - --deployment-type + - sriov + generate: + arguments: [] + deploy: + arguments: [] + validate: + arguments: [] + clean: + arguments: [] + infiniband_sriov: + working_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/infiniband-sriov/work + artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/infiniband-sriov/evidence + discover: + arguments: + - --fabric + - infiniband + - --deployment-type + - sriov + generate: + arguments: [] + deploy: + arguments: [] + validate: + arguments: [] + clean: + arguments: [] + roce_rdma_shared: + working_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/roce-rdma-shared/work + artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/roce-rdma-shared/evidence + discover: + arguments: + - --fabric + - ethernet + - --deployment-type + - rdma_shared + generate: + arguments: [] + deploy: + arguments: [] + validate: + arguments: [] + clean: + arguments: [] + infiniband_rdma_shared: + working_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/infiniband-rdma-shared/work + artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/infiniband-rdma-shared/evidence + discover: + arguments: + - --fabric + - infiniband + - --deployment-type + - rdma_shared + generate: + arguments: [] + deploy: + arguments: [] + validate: + arguments: [] + clean: + arguments: [] + roce_host_device: + working_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/roce-host-device/work + artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/roce-host-device/evidence + discover: + arguments: + - --fabric + - ethernet + - --deployment-type + - host_device + generate: + arguments: [] + deploy: + arguments: [] + validate: + arguments: [] + clean: + arguments: [] + infiniband_host_device: + working_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/infiniband-host-device/work + artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/infiniband-host-device/evidence + discover: + arguments: + - --fabric + - infiniband + - --deployment-type + - host_device + generate: + arguments: [] + deploy: + arguments: [] + validate: + arguments: [] + clean: + arguments: [] + +commands: + network_operator: + phases: + - setup + - launch-kit-verification + - roce-sriov + - infiniband-sriov + - roce-rdma-shared + - infiniband-rdma-shared + - roce-host-device + - infiniband-host-device + - teardown + continue_after_failure: + - roce-sriov + - infiniband-sriov + - roce-rdma-shared + - infiniband-rdma-shared + - roce-host-device + - infiniband-host-device + steps: + - name: launch_kit_prepare + phase: setup + command: python3 ../scripts/adapter.py + args: + - prepare + - --mode + - "{{ context.k8s_launch_kit.installation.mode }}" + - --executable + - "{{ context.k8s_launch_kit.executable }}" + - "--version={{ context.k8s_launch_kit.installation.version }}" + - "--installer-ref={{ context.k8s_launch_kit.installation.installer_ref }}" + - "--installer-sha256={{ context.k8s_launch_kit.installation.installer_sha256 }}" + - "--prefix={{ context.k8s_launch_kit.installation.prefix }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.shared_artifact_dir }}" + timeout: 900 + output_schema: k8s_launch_kit + requires: [kubernetes] + + - name: launch_kit_verify + phase: launch-kit-verification + command: python3 ../scripts/adapter.py + args: + - verify + - --executable + - "{{ steps.launch_kit_prepare.executable | default(context.k8s_launch_kit.executable) }}" + - "--expected-version={{ context.k8s_launch_kit.installation.version }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.shared_artifact_dir }}" + timeout: 60 + output_schema: k8s_launch_kit + requires: [kubernetes] + + # roce-sriov: preflight -> discover -> generate -> deploy -> validate -> clean + - name: launch_kit_roce_sriov_preflight + phase: roce-sriov + command: python3 ../scripts/adapter.py + args: + - preflight + - --kubectl-command-json + - "{{ context.k8s_launch_kit.kubectl_command | tojson }}" + - --workflow-arguments-json + - "{{ {'discover': context.k8s_launch_kit.use_cases.roce_sriov.discover.arguments, 'generate': context.k8s_launch_kit.use_cases.roce_sriov.generate.arguments, 'deploy': context.k8s_launch_kit.use_cases.roce_sriov.deploy.arguments, 'validate': context.k8s_launch_kit.use_cases.roce_sriov.validate.arguments, 'clean': context.k8s_launch_kit.use_cases.roce_sriov.clean.arguments} | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.artifact_dir }}" + timeout: 60 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceSriovCheck] + + - name: launch_kit_roce_sriov_discover + phase: roce-sriov + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - discover + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.discover.arguments | tojson }}" + - "--user-config={{ context.k8s_launch_kit.user_config }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.artifact_dir }}" + timeout: 1800 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceSriovCheck] + + - name: launch_kit_roce_sriov_generate + phase: roce-sriov + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - generate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.generate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.artifact_dir }}" + timeout: 600 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceSriovCheck] + + - name: launch_kit_roce_sriov_deploy + phase: roce-sriov + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - deploy + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.deploy.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.artifact_dir }}" + timeout: 7200 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceSriovCheck] + + - name: launch_kit_roce_sriov_validate + phase: roce-sriov + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - validate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.validate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.artifact_dir }}" + timeout: null + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceSriovCheck] + + - name: launch_kit_roce_sriov_clean + phase: teardown + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable | default(context.k8s_launch_kit.executable) }}" + - --command + - clean + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.clean.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.artifact_dir }}" + timeout: 7200 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceSriovCheck] + finalizer_for: launch_kit_roce_sriov_deploy + + # infiniband-sriov: preflight -> discover -> generate -> deploy -> validate -> clean + - name: launch_kit_infiniband_sriov_preflight + phase: infiniband-sriov + command: python3 ../scripts/adapter.py + args: + - preflight + - --kubectl-command-json + - "{{ context.k8s_launch_kit.kubectl_command | tojson }}" + - --workflow-arguments-json + - "{{ {'discover': context.k8s_launch_kit.use_cases.infiniband_sriov.discover.arguments, 'generate': context.k8s_launch_kit.use_cases.infiniband_sriov.generate.arguments, 'deploy': context.k8s_launch_kit.use_cases.infiniband_sriov.deploy.arguments, 'validate': context.k8s_launch_kit.use_cases.infiniband_sriov.validate.arguments, 'clean': context.k8s_launch_kit.use_cases.infiniband_sriov.clean.arguments} | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.artifact_dir }}" + timeout: 60 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandSriovCheck] + + - name: launch_kit_infiniband_sriov_discover + phase: infiniband-sriov + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - discover + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.discover.arguments | tojson }}" + - "--user-config={{ context.k8s_launch_kit.user_config }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.artifact_dir }}" + timeout: 1800 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandSriovCheck] + + - name: launch_kit_infiniband_sriov_generate + phase: infiniband-sriov + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - generate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.generate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.artifact_dir }}" + timeout: 600 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandSriovCheck] + + - name: launch_kit_infiniband_sriov_deploy + phase: infiniband-sriov + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - deploy + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.deploy.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.artifact_dir }}" + timeout: 7200 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandSriovCheck] + + - name: launch_kit_infiniband_sriov_validate + phase: infiniband-sriov + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - validate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.validate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.artifact_dir }}" + timeout: null + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandSriovCheck] + + - name: launch_kit_infiniband_sriov_clean + phase: teardown + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable | default(context.k8s_launch_kit.executable) }}" + - --command + - clean + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.clean.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.artifact_dir }}" + timeout: 7200 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandSriovCheck] + finalizer_for: launch_kit_infiniband_sriov_deploy + + # roce-rdma-shared: preflight -> discover -> generate -> deploy -> validate -> clean + - name: launch_kit_roce_rdma_shared_preflight + phase: roce-rdma-shared + command: python3 ../scripts/adapter.py + args: + - preflight + - --kubectl-command-json + - "{{ context.k8s_launch_kit.kubectl_command | tojson }}" + - --workflow-arguments-json + - "{{ {'discover': context.k8s_launch_kit.use_cases.roce_rdma_shared.discover.arguments, 'generate': context.k8s_launch_kit.use_cases.roce_rdma_shared.generate.arguments, 'deploy': context.k8s_launch_kit.use_cases.roce_rdma_shared.deploy.arguments, 'validate': context.k8s_launch_kit.use_cases.roce_rdma_shared.validate.arguments, 'clean': context.k8s_launch_kit.use_cases.roce_rdma_shared.clean.arguments} | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.artifact_dir }}" + timeout: 60 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceRdmaSharedCheck] + + - name: launch_kit_roce_rdma_shared_discover + phase: roce-rdma-shared + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - discover + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.discover.arguments | tojson }}" + - "--user-config={{ context.k8s_launch_kit.user_config }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.artifact_dir }}" + timeout: 1800 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceRdmaSharedCheck] + + - name: launch_kit_roce_rdma_shared_generate + phase: roce-rdma-shared + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - generate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.generate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.artifact_dir }}" + timeout: 600 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceRdmaSharedCheck] + + - name: launch_kit_roce_rdma_shared_deploy + phase: roce-rdma-shared + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - deploy + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.deploy.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.artifact_dir }}" + timeout: 7200 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceRdmaSharedCheck] + + - name: launch_kit_roce_rdma_shared_validate + phase: roce-rdma-shared + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - validate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.validate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.artifact_dir }}" + timeout: null + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceRdmaSharedCheck] + + - name: launch_kit_roce_rdma_shared_clean + phase: teardown + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable | default(context.k8s_launch_kit.executable) }}" + - --command + - clean + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.clean.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.artifact_dir }}" + timeout: 7200 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceRdmaSharedCheck] + finalizer_for: launch_kit_roce_rdma_shared_deploy + + # infiniband-rdma-shared: preflight -> discover -> generate -> deploy -> validate -> clean + - name: launch_kit_infiniband_rdma_shared_preflight + phase: infiniband-rdma-shared + command: python3 ../scripts/adapter.py + args: + - preflight + - --kubectl-command-json + - "{{ context.k8s_launch_kit.kubectl_command | tojson }}" + - --workflow-arguments-json + - "{{ {'discover': context.k8s_launch_kit.use_cases.infiniband_rdma_shared.discover.arguments, 'generate': context.k8s_launch_kit.use_cases.infiniband_rdma_shared.generate.arguments, 'deploy': context.k8s_launch_kit.use_cases.infiniband_rdma_shared.deploy.arguments, 'validate': context.k8s_launch_kit.use_cases.infiniband_rdma_shared.validate.arguments, 'clean': context.k8s_launch_kit.use_cases.infiniband_rdma_shared.clean.arguments} | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.artifact_dir }}" + timeout: 60 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandRdmaSharedCheck] + + - name: launch_kit_infiniband_rdma_shared_discover + phase: infiniband-rdma-shared + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - discover + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.discover.arguments | tojson }}" + - "--user-config={{ context.k8s_launch_kit.user_config }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.artifact_dir }}" + timeout: 1800 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandRdmaSharedCheck] + + - name: launch_kit_infiniband_rdma_shared_generate + phase: infiniband-rdma-shared + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - generate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.generate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.artifact_dir }}" + timeout: 600 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandRdmaSharedCheck] + + - name: launch_kit_infiniband_rdma_shared_deploy + phase: infiniband-rdma-shared + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - deploy + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.deploy.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.artifact_dir }}" + timeout: 7200 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandRdmaSharedCheck] + + - name: launch_kit_infiniband_rdma_shared_validate + phase: infiniband-rdma-shared + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - validate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.validate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.artifact_dir }}" + timeout: null + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandRdmaSharedCheck] + + - name: launch_kit_infiniband_rdma_shared_clean + phase: teardown + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable | default(context.k8s_launch_kit.executable) }}" + - --command + - clean + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.clean.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.artifact_dir }}" + timeout: 7200 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandRdmaSharedCheck] + finalizer_for: launch_kit_infiniband_rdma_shared_deploy + + # roce-host-device: preflight -> discover -> generate -> deploy -> validate -> clean + - name: launch_kit_roce_host_device_preflight + phase: roce-host-device + command: python3 ../scripts/adapter.py + args: + - preflight + - --kubectl-command-json + - "{{ context.k8s_launch_kit.kubectl_command | tojson }}" + - --workflow-arguments-json + - "{{ {'discover': context.k8s_launch_kit.use_cases.roce_host_device.discover.arguments, 'generate': context.k8s_launch_kit.use_cases.roce_host_device.generate.arguments, 'deploy': context.k8s_launch_kit.use_cases.roce_host_device.deploy.arguments, 'validate': context.k8s_launch_kit.use_cases.roce_host_device.validate.arguments, 'clean': context.k8s_launch_kit.use_cases.roce_host_device.clean.arguments} | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.artifact_dir }}" + timeout: 60 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceHostDeviceCheck] + + - name: launch_kit_roce_host_device_discover + phase: roce-host-device + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - discover + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.discover.arguments | tojson }}" + - "--user-config={{ context.k8s_launch_kit.user_config }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.artifact_dir }}" + timeout: 1800 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceHostDeviceCheck] + + - name: launch_kit_roce_host_device_generate + phase: roce-host-device + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - generate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.generate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.artifact_dir }}" + timeout: 600 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceHostDeviceCheck] + + - name: launch_kit_roce_host_device_deploy + phase: roce-host-device + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - deploy + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.deploy.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.artifact_dir }}" + timeout: 7200 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceHostDeviceCheck] + + - name: launch_kit_roce_host_device_validate + phase: roce-host-device + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - validate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.validate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.artifact_dir }}" + timeout: null + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceHostDeviceCheck] + + - name: launch_kit_roce_host_device_clean + phase: teardown + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable | default(context.k8s_launch_kit.executable) }}" + - --command + - clean + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.clean.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.artifact_dir }}" + timeout: 7200 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceHostDeviceCheck] + finalizer_for: launch_kit_roce_host_device_deploy + + # infiniband-host-device: preflight -> discover -> generate -> deploy -> validate -> clean + - name: launch_kit_infiniband_host_device_preflight + phase: infiniband-host-device + command: python3 ../scripts/adapter.py + args: + - preflight + - --kubectl-command-json + - "{{ context.k8s_launch_kit.kubectl_command | tojson }}" + - --workflow-arguments-json + - "{{ {'discover': context.k8s_launch_kit.use_cases.infiniband_host_device.discover.arguments, 'generate': context.k8s_launch_kit.use_cases.infiniband_host_device.generate.arguments, 'deploy': context.k8s_launch_kit.use_cases.infiniband_host_device.deploy.arguments, 'validate': context.k8s_launch_kit.use_cases.infiniband_host_device.validate.arguments, 'clean': context.k8s_launch_kit.use_cases.infiniband_host_device.clean.arguments} | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.artifact_dir }}" + timeout: 60 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandHostDeviceCheck] + + - name: launch_kit_infiniband_host_device_discover + phase: infiniband-host-device + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - discover + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.discover.arguments | tojson }}" + - "--user-config={{ context.k8s_launch_kit.user_config }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.artifact_dir }}" + timeout: 1800 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandHostDeviceCheck] + + - name: launch_kit_infiniband_host_device_generate + phase: infiniband-host-device + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - generate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.generate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.artifact_dir }}" + timeout: 600 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandHostDeviceCheck] + + - name: launch_kit_infiniband_host_device_deploy + phase: infiniband-host-device + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - deploy + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.deploy.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.artifact_dir }}" + timeout: 7200 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandHostDeviceCheck] + + - name: launch_kit_infiniband_host_device_validate + phase: infiniband-host-device + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - validate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.validate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.artifact_dir }}" + timeout: null + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandHostDeviceCheck] + + - name: launch_kit_infiniband_host_device_clean + phase: teardown + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable | default(context.k8s_launch_kit.executable) }}" + - --command + - clean + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.clean.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.artifact_dir }}" + timeout: 7200 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandHostDeviceCheck] + finalizer_for: launch_kit_infiniband_host_device_deploy diff --git a/isvctl/configs/providers/k8s-launch-kit/config/provider.yaml b/isvctl/configs/providers/k8s-launch-kit/config/provider.yaml new file mode 100644 index 000000000..716208adb --- /dev/null +++ b/isvctl/configs/providers/k8s-launch-kit/config/provider.yaml @@ -0,0 +1,219 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Generic Kubernetes Launch Kit provider. This production configuration invokes +# l8k and kubectl from PATH unless a user overlay explicitly replaces them. +# +# This file intentionally contains no Network Operator, topology, resource, +# validation, or l8k timeout defaults. The arguments for every command are +# passed directly to l8k. Omitted values are resolved by Launch Kit itself. + +version: "1.0" + +context: + k8s_launch_kit: + executable: l8k + installation: + mode: verify + version: "" + installer_ref: "" + installer_sha256: "" + prefix: "" + # Optional complete Launch Kit configuration. Discovery stages a copy as + # ./user-config.yaml and writes its resolved output to ./cluster-config.yaml. + user_config: "" + kubectl_command: [] + working_dir: ../../../../../_output/k8s-launch-kit/work + artifact_dir: ../../../../../_output/k8s-launch-kit/evidence + environment: {} + discover: + arguments: [] + generate: + arguments: [] + deploy: + arguments: [] + validate: + arguments: [] + clean: + arguments: [] + +commands: + network_operator: + phases: [setup, test, teardown] + steps: + - name: launch_kit_prepare + phase: setup + command: python3 ../scripts/adapter.py + args: + - prepare + - --mode + - "{{ context.k8s_launch_kit.installation.mode }}" + - --executable + - "{{ context.k8s_launch_kit.executable }}" + - "--version={{ context.k8s_launch_kit.installation.version }}" + - "--installer-ref={{ context.k8s_launch_kit.installation.installer_ref }}" + - "--installer-sha256={{ context.k8s_launch_kit.installation.installer_sha256 }}" + - "--prefix={{ context.k8s_launch_kit.installation.prefix }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 900 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_available_validations: [LaunchKitKubernetesPrerequisiteCheck] + + # Test-phase verification is intentional: --phase test may bypass setup. + - name: launch_kit_verify + phase: test + command: python3 ../scripts/adapter.py + args: + - verify + - --executable + - "{{ steps.launch_kit_prepare.executable | default(context.k8s_launch_kit.executable) }}" + - "--expected-version={{ context.k8s_launch_kit.installation.version }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 60 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_available_validations: [LaunchKitKubernetesPrerequisiteCheck] + + # This gate always runs in the test phase, before l8k can mutate a cluster. + - name: launch_kit_kubernetes_preflight + phase: test + command: python3 ../scripts/adapter.py + args: + - preflight + - --kubectl-command-json + - "{{ context.k8s_launch_kit.kubectl_command | tojson }}" + - --workflow-arguments-json + - "{{ {'discover': context.k8s_launch_kit.discover.arguments, 'generate': context.k8s_launch_kit.generate.arguments, 'deploy': context.k8s_launch_kit.deploy.arguments, 'validate': context.k8s_launch_kit.validate.arguments, 'clean': context.k8s_launch_kit.clean.arguments} | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 60 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_available_validations: [LaunchKitKubernetesPrerequisiteCheck] + + - name: launch_kit_discover + phase: test + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - discover + - --arguments-json + - "{{ context.k8s_launch_kit.discover.arguments | tojson }}" + - "--user-config={{ context.k8s_launch_kit.user_config }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 1800 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_available_validations: [LaunchKitTopologyDiscoveryCheck] + + - name: launch_kit_generate + phase: test + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - generate + - --arguments-json + - "{{ context.k8s_launch_kit.generate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 600 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_available_validations: [LaunchKitDeploymentHealthCheck] + + - name: launch_kit_deploy + phase: test + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - deploy + - --arguments-json + - "{{ context.k8s_launch_kit.deploy.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 7200 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_available_validations: [LaunchKitDeploymentHealthCheck] + + - name: launch_kit_validate + phase: test + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - validate + - --arguments-json + - "{{ context.k8s_launch_kit.validate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + # l8k calculates a bounded matrix budget or honors the user's explicit + # connectivity timeout, so it owns the deadline for this command. + timeout: null + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_available_validations: [LaunchKitDeploymentHealthCheck] + + # Runs after phase validations when deploy was attempted, even if deploy, + # validate, or a validation check failed. + - name: launch_kit_clean + phase: teardown + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable | default(context.k8s_launch_kit.executable) }}" + - --command + - clean + - --arguments-json + - "{{ context.k8s_launch_kit.clean.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 7200 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_available_validations: [LaunchKitDeploymentHealthCheck] + finalizer_for: launch_kit_deploy diff --git a/isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py b/isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py new file mode 100644 index 000000000..665f84e51 --- /dev/null +++ b/isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py @@ -0,0 +1,722 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Thin AI Cloud Validation transport for the Kubernetes Launch Kit CLI. + +The provider deliberately exposes the real Launch Kit operations. It forwards +user-supplied arguments verbatim and adds ``--output json`` so stdout can be +preserved as structured evidence. When a complete user config is supplied, the +discover operation also stages it transiently in the working directory, binds +Launch Kit's native ``--user-config`` and ``--save-cluster-config`` flags, and +removes the staged input after discovery. Launch Kit remains the owner of +command flags, configuration schema, and defaults. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +import time +import urllib.error +import urllib.request +from json import JSONDecoder +from pathlib import Path +from typing import Any + +_WORKFLOW_COMMANDS = ("discover", "generate", "deploy", "validate", "clean") +_INSTALLER_URL = "https://raw.githubusercontent.com/NVIDIA/k8s-launch-kit/{ref}/scripts/install.sh" +_STAGED_USER_CONFIG = "user-config.yaml" +_DISCOVERED_CLUSTER_CONFIG = "cluster-config.yaml" + + +def _parse_json_value(raw: str, source: str, expected_type: type[Any]) -> Any: + """Parse a JSON CLI value and enforce its root type.""" + try: + value = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError(f"{source} is not valid JSON: {exc}") from exc + if not isinstance(value, expected_type): + raise ValueError(f"{source} must contain a {expected_type.__name__}") + return value + + +def _parse_json_stream(raw: str, source: str) -> list[dict[str, Any]]: + """Parse zero or more concatenated JSON objects from ``raw``.""" + decoder = JSONDecoder() + documents: list[dict[str, Any]] = [] + offset = 0 + while offset < len(raw): + while offset < len(raw) and raw[offset].isspace(): + offset += 1 + if offset >= len(raw): + break + try: + value, offset = decoder.raw_decode(raw, offset) + except json.JSONDecodeError as exc: + raise ValueError(f"{source} contains invalid JSON at byte {exc.pos}: {exc.msg}") from exc + if not isinstance(value, dict): + raise ValueError(f"{source} document #{len(documents) + 1} is not an object") + documents.append(value) + return documents + + +def _write_json(path: Path, value: Any) -> None: + """Write deterministic structured evidence.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + + +def _resolve_executable(value: str) -> Path: + """Resolve an explicit path or a command available on ``PATH``.""" + candidate = Path(value).expanduser() + if candidate.is_absolute() or candidate.parent != Path("."): + resolved = candidate.resolve() + if not resolved.is_file(): + raise FileNotFoundError(f"Launch Kit executable not found: {resolved}") + return resolved + found = shutil.which(value) + if found is None: + raise FileNotFoundError(f"Launch Kit executable not found on PATH: {value}") + return Path(found).resolve() + + +def _with_json_output(arguments: list[str]) -> list[str]: + """Return workflow arguments that request Launch Kit's automation output.""" + result = list(arguments) + for index, token in enumerate(result): + if token == "--output": + if index + 1 >= len(result): + raise ValueError("--output requires a value") + if result[index + 1] != "json": + raise ValueError("the Launch Kit provider requires --output json") + return result + if token.startswith("--output="): + if token.partition("=")[2] != "json": + raise ValueError("the Launch Kit provider requires --output json") + return result + result.extend(["--output", "json"]) + return result + + +def _structured_error(documents: list[dict[str, Any]]) -> str | None: + """Extract the most actionable Launch Kit structured error.""" + for document in reversed(documents): + error = document.get("error") + if not isinstance(error, dict): + continue + message = error.get("message") + if not isinstance(message, str) or not message: + continue + suggestion = error.get("suggestion") + if isinstance(suggestion, str) and suggestion: + return f"{message}; {suggestion}" + return message + return None + + +def _stderr_excerpt(stderr: str) -> str | None: + """Return the last non-empty stderr line without flooding the envelope.""" + lines = [line.strip() for line in stderr.splitlines() if line.strip()] + return lines[-1] if lines else None + + +def _run_process(argv: list[str], *, cwd: Path, env: dict[str, str]) -> dict[str, Any]: + """Execute a child process and retain both output streams.""" + started = time.monotonic() + try: + completed = subprocess.run( + argv, + cwd=cwd, + env=env, + check=False, + capture_output=True, + text=True, + ) + return { + "exit_code": completed.returncode, + "stdout": completed.stdout, + "stderr": completed.stderr, + "duration_seconds": time.monotonic() - started, + } + except OSError as exc: + return { + "exit_code": -1, + "stdout": "", + "stderr": str(exc), + "duration_seconds": time.monotonic() - started, + } + + +def _record_process(directory: Path, argv: list[str], result: dict[str, Any]) -> dict[str, str]: + """Persist one command, stdout, and stderr as evidence.""" + directory.mkdir(parents=True, exist_ok=True) + stdout_path = directory / "stdout.txt" + stderr_path = directory / "stderr.log" + command_path = directory / "command.json" + stdout_path.write_text(str(result["stdout"]), encoding="utf-8") + stderr_path.write_text(str(result["stderr"]), encoding="utf-8") + _write_json( + command_path, + { + "argv": argv, + "exit_code": result["exit_code"], + "duration_seconds": result["duration_seconds"], + }, + ) + return { + "stdout": str(stdout_path.resolve()), + "stderr": str(stderr_path.resolve()), + "command": str(command_path.resolve()), + } + + +def _environment(raw: str) -> dict[str, str]: + """Merge user-supplied string environment entries with the process environment.""" + supplied = _parse_json_value(raw, "--environment-json", dict) + invalid = [str(key) for key, value in supplied.items() if not isinstance(key, str) or not isinstance(value, str)] + if invalid: + raise ValueError("--environment-json keys and values must be strings") + env = os.environ.copy() + env.update(supplied) + return env + + +def _stage_user_config( + source_value: str, + working_dir: Path, + arguments: list[str], +) -> tuple[list[str], Path | None, dict[str, Any] | None]: + """Stage a complete user config for discovery and return safe provenance.""" + if not source_value: + return arguments, None, None + + conflicting_flags = [ + flag + for flag in ("--user-config", "--save-cluster-config") + if any(token == flag or token.startswith(f"{flag}=") for token in arguments) + ] + if conflicting_flags: + raise ValueError( + "context.k8s_launch_kit.user_config cannot be combined with raw discovery flag(s): " + + ", ".join(conflicting_flags) + ) + + source = Path(source_value).expanduser().resolve() + if not source.is_file(): + raise FileNotFoundError(f"Launch Kit user config not found: {source}") + try: + source.relative_to(working_dir) + except ValueError: + pass + else: + raise ValueError("Launch Kit user config must be outside the retained provider working directory") + + staged = working_dir / _STAGED_USER_CONFIG + content = source.read_bytes() + staged.unlink(missing_ok=True) + try: + staged.write_bytes(content) + staged.chmod(0o600) + except OSError: + staged.unlink(missing_ok=True) + raise + + discovered = working_dir / _DISCOVERED_CLUSTER_CONFIG + return ( + [ + *arguments, + "--user-config", + str(staged), + "--save-cluster-config", + str(discovered), + ], + staged, + { + "source_path": str(source), + "staged_path": str(staged), + "sha256": hashlib.sha256(content).hexdigest(), + "size_bytes": len(content), + "retained": False, + }, + ) + + +def _run_workflow(args: argparse.Namespace) -> tuple[dict[str, Any], int]: + """Invoke exactly one real Launch Kit workflow command.""" + executable = _resolve_executable(args.executable) + arguments = _parse_json_value(args.arguments_json, "--arguments-json", list) + if not all(isinstance(value, str) for value in arguments): + raise ValueError("--arguments-json must contain only strings") + environment = _environment(args.environment_json) + working_dir = Path(args.working_dir).expanduser().resolve() + working_dir.mkdir(parents=True, exist_ok=True) + artifact_dir = Path(args.artifact_dir).expanduser().resolve() + staged_user_config: Path | None = None + user_config_metadata_path: Path | None = None + try: + if args.user_config: + if args.command != "discover": + raise ValueError("--user-config is supported only with the discover workflow command") + arguments, staged_user_config, user_config_metadata = _stage_user_config( + args.user_config, + working_dir, + arguments, + ) + user_config_metadata_path = artifact_dir / "inputs" / "user-config.json" + _write_json(user_config_metadata_path, user_config_metadata) + arguments = _with_json_output(arguments) + argv = [str(executable), args.command, *arguments] + result = _run_process(argv, cwd=working_dir, env=environment) + finally: + if staged_user_config is not None: + staged_user_config.unlink(missing_ok=True) + artifacts = _record_process(artifact_dir / "commands" / args.command, argv, result) + if user_config_metadata_path is not None: + artifacts["user_config"] = str(user_config_metadata_path) + + parse_error: str | None = None + try: + documents = _parse_json_stream(str(result["stdout"]), f"l8k {args.command} stdout") + except ValueError as exc: + documents = [] + parse_error = str(exc) + + success = result["exit_code"] == 0 and parse_error is None + error = parse_error or _structured_error(documents) + if not success and error is None: + error = f"l8k {args.command} exited with code {result['exit_code']}" + if excerpt := _stderr_excerpt(str(result["stderr"])): + error = f"{error}: {excerpt}" + envelope: dict[str, Any] = { + "success": success, + "platform": "kubernetes", + "operation": args.command, + "executable": str(executable), + "argv": argv, + "working_directory": str(working_dir), + "exit_code": result["exit_code"], + "duration_seconds": result["duration_seconds"], + "documents": documents, + "artifacts": artifacts, + } + if error: + envelope["error"] = error + excerpt = _stderr_excerpt(str(result["stderr"])) + if excerpt: + envelope["stderr_excerpt"] = excerpt + exit_code = int(result["exit_code"]) + return envelope, exit_code if exit_code > 0 else (0 if success else 1) + + +def _verify_executable( + executable: Path, + artifact_dir: Path, + expected_version: str = "", + environment: dict[str, str] | None = None, +) -> tuple[dict[str, Any], bool, str | None]: + """Run Launch Kit version and schema commands and preserve both responses.""" + env = environment.copy() if environment is not None else os.environ.copy() + checks: dict[str, Any] = {} + artifacts: dict[str, dict[str, str]] = {} + errors: list[str] = [] + for name, command_args in (("version", ["version", "--output", "json"]), ("schema", ["schema"])): + argv = [str(executable), *command_args] + result = _run_process(argv, cwd=Path.cwd(), env=env) + artifacts[name] = _record_process(artifact_dir / name, argv, result) + try: + documents = _parse_json_stream(str(result["stdout"]), f"l8k {name} stdout") + except ValueError as exc: + documents = [] + errors.append(str(exc)) + if result["exit_code"] != 0: + errors.append( + _stderr_excerpt(str(result["stderr"])) or f"l8k {name} exited with code {result['exit_code']}" + ) + elif len(documents) != 1: + errors.append(f"l8k {name} must emit exactly one JSON object, got {len(documents)}") + passed = result["exit_code"] == 0 and len(documents) == 1 + if name == "schema" and passed: + commands = documents[0].get("commands") + advertised = set(commands) if isinstance(commands, dict) else set() + missing_commands = set(_WORKFLOW_COMMANDS) - advertised + if missing_commands: + passed = False + errors.append( + "l8k schema does not advertise required command(s): " + ", ".join(sorted(missing_commands)) + ) + if name == "version" and passed and expected_version: + actual_version = documents[0].get("version") + if actual_version != expected_version: + passed = False + errors.append(f"l8k version mismatch: expected {expected_version!r}, got {actual_version!r}") + checks[name] = { + "passed": passed, + "documents": documents, + "exit_code": result["exit_code"], + "artifacts": artifacts[name], + } + return {"checks": checks, "artifacts": artifacts}, not errors, "; ".join(errors) or None + + +def _download_installer(installer_ref: str, expected_sha256: str, artifact_dir: Path) -> tuple[Path, str]: + """Download an immutable installer and verify its trusted SHA-256 digest.""" + if not re.fullmatch(r"[0-9a-fA-F]{40}", installer_ref): + raise ValueError("Launch Kit installer_ref must be a full 40-character Git commit SHA") + if not re.fullmatch(r"[0-9a-fA-F]{64}", expected_sha256): + raise ValueError("Launch Kit installer_sha256 must be a 64-character SHA-256 digest") + expected_sha256 = expected_sha256.lower() + url = _INSTALLER_URL.format(ref=installer_ref.lower()) + installer = artifact_dir / "installer.sh" + installer.parent.mkdir(parents=True, exist_ok=True) + installer.unlink(missing_ok=True) + request = urllib.request.Request(url, headers={"User-Agent": "ai-cloud-validation"}) + with urllib.request.urlopen(request, timeout=30) as response: + content = response.read() + digest = hashlib.sha256(content).hexdigest() + verified = digest == expected_sha256 + _write_json( + artifact_dir / "installer-download.json", + { + "url": url, + "ref": installer_ref.lower(), + "expected_sha256": expected_sha256, + "sha256": digest, + "verified": verified, + }, + ) + if not verified: + raise ValueError(f"Launch Kit installer SHA-256 mismatch: expected {expected_sha256}, got {digest}") + installer.write_bytes(content) + return installer, url + + +def _installed_executable(prefix: str) -> Path: + """Resolve the executable installed by the official Launch Kit installer.""" + install_prefix = Path(prefix).expanduser() if prefix else Path("/usr/local") + return _resolve_executable(str(install_prefix / "bin" / "l8k")) + + +def _prepare(args: argparse.Namespace) -> tuple[dict[str, Any], int]: + """Optionally install Launch Kit, then verify version and schema.""" + artifact_dir = Path(args.artifact_dir).expanduser().resolve() / "prepare" + environment = _environment(args.environment_json) + installed = False + install_details: dict[str, Any] | None = None + if args.mode == "install": + installer, url = _download_installer(args.installer_ref, args.installer_sha256, artifact_dir) + installer_argv = ["/bin/sh", str(installer)] + if args.prefix: + installer_argv.extend(["-d", args.prefix]) + env = environment.copy() + if args.version: + env["L8K_VERSION"] = args.version + result = _run_process(installer_argv, cwd=Path.cwd(), env=env) + install_artifacts = _record_process(artifact_dir / "install", installer_argv, result) + install_details = { + "url": url, + "exit_code": result["exit_code"], + "installer": str(installer.resolve()), + "download_metadata": str((artifact_dir / "installer-download.json").resolve()), + "artifacts": install_artifacts, + } + if result["exit_code"] != 0: + error = _stderr_excerpt(str(result["stderr"])) or "Launch Kit installer failed" + return { + "success": False, + "platform": "kubernetes", + "operation": "prepare", + "installed": False, + "install": install_details, + "error": error, + }, 1 + executable = _installed_executable(args.prefix) + installed = True + else: + executable = _resolve_executable(args.executable) + + verification, success, error = _verify_executable( + executable, + artifact_dir / "verify", + args.version, + environment, + ) + envelope: dict[str, Any] = { + "success": success, + "platform": "kubernetes", + "operation": "prepare", + "installed": installed, + "executable": str(executable), + **verification, + } + if install_details is not None: + envelope["install"] = install_details + envelope["artifacts"]["installer"] = install_details["installer"] + envelope["artifacts"]["installer_download"] = install_details["download_metadata"] + envelope["artifacts"]["install"] = install_details["artifacts"] + if error: + envelope["error"] = error + return envelope, 0 if success else 1 + + +def _verify(args: argparse.Namespace) -> tuple[dict[str, Any], int]: + """Verify an existing Launch Kit executable without installing it.""" + executable = _resolve_executable(args.executable) + environment = _environment(args.environment_json) + verification, success, error = _verify_executable( + executable, + Path(args.artifact_dir).expanduser().resolve() / "verify-test", + args.expected_version, + environment, + ) + envelope: dict[str, Any] = { + "success": success, + "platform": "kubernetes", + "operation": "verify", + "executable": str(executable), + **verification, + } + if error: + envelope["error"] = error + return envelope, 0 if success else 1 + + +def _kubeconfig_from_workflow(raw: str) -> str | None: + """Extract one consistent explicit kubeconfig from the real workflow arguments.""" + workflow = _parse_json_value(raw, "--workflow-arguments-json", dict) + if set(workflow) != set(_WORKFLOW_COMMANDS): + raise ValueError("--workflow-arguments-json must contain exactly: " + ", ".join(_WORKFLOW_COMMANDS)) + found: set[str] = set() + for command, values in workflow.items(): + if command not in _WORKFLOW_COMMANDS or not isinstance(values, list): + raise ValueError("--workflow-arguments-json must map Launch Kit workflow commands to argument lists") + if not all(isinstance(value, str) for value in values): + raise ValueError(f"workflow arguments for {command} must contain only strings") + index = 0 + while index < len(values): + token = values[index] + if token == "--kubeconfig": + if index + 1 >= len(values): + raise ValueError(f"{command} --kubeconfig requires a value") + value = values[index + 1] + if not value: + raise ValueError(f"{command} --kubeconfig requires a non-empty value") + found.add(value) + index += 2 + continue + if token.startswith("--kubeconfig="): + value = token.partition("=")[2] + if not value: + raise ValueError(f"{command} --kubeconfig requires a non-empty value") + found.add(value) + index += 1 + if len(found) > 1: + raise ValueError(f"Launch Kit workflow commands select different kubeconfigs: {sorted(found)}") + return next(iter(found), None) + + +def _kubectl_prefix(raw: str, environment: dict[str, str]) -> list[str]: + """Resolve the configured kubectl-compatible invocation.""" + supplied = _parse_json_value(raw, "--kubectl-command-json", list) + if supplied: + if not all(isinstance(value, str) and value for value in supplied): + raise ValueError("--kubectl-command-json must contain non-empty strings") + invocation_dir = Path.cwd() + return [ + str((invocation_dir / value).resolve()) + if not Path(value).is_absolute() and "/" in value and (invocation_dir / value).exists() + else value + for value in supplied + ] + override = environment.get("KUBECTL", "").strip() + return shlex.split(override) if override else ["kubectl"] + + +def _preflight_check( + name: str, + argv: list[str], + *, + cwd: Path, + artifact_dir: Path, + environment: dict[str, str], +) -> tuple[dict[str, Any], dict[str, Any]]: + """Run and record one Kubernetes prerequisite command.""" + result = _run_process(argv, cwd=cwd, env=environment) + artifacts = _record_process(artifact_dir / name, argv, result) + passed = result["exit_code"] == 0 + message = "command succeeded" if passed else (_stderr_excerpt(str(result["stderr"])) or "command failed") + return { + "name": name, + "passed": passed, + "message": message, + "exit_code": result["exit_code"], + "artifacts": artifacts, + }, result + + +def _preflight(args: argparse.Namespace) -> tuple[dict[str, Any], int]: + """Prove that the Kubernetes API and at least one Ready node are available.""" + working_dir = Path(args.working_dir).expanduser().resolve() + working_dir.mkdir(parents=True, exist_ok=True) + artifact_dir = Path(args.artifact_dir).expanduser().resolve() / "kubernetes-preflight" + kubeconfig = _kubeconfig_from_workflow(args.workflow_arguments_json) + environment = _environment(args.environment_json) + prefix = _kubectl_prefix(args.kubectl_command_json, environment) + kubeconfig_args = ["--kubeconfig", kubeconfig] if kubeconfig else [] + + version_check, version_result = _preflight_check( + "api-version", + [*prefix, *kubeconfig_args, "version", "-o", "json"], + cwd=working_dir, + artifact_dir=artifact_dir, + environment=environment, + ) + nodes_check, nodes_result = _preflight_check( + "nodes", + [*prefix, *kubeconfig_args, "get", "nodes", "-o", "json"], + cwd=working_dir, + artifact_dir=artifact_dir, + environment=environment, + ) + + server_version: str | None = None + if version_check["passed"]: + try: + version_payload = json.loads(str(version_result["stdout"])) + server = version_payload.get("serverVersion") if isinstance(version_payload, dict) else None + if isinstance(server, dict) and isinstance(server.get("gitVersion"), str): + server_version = server["gitVersion"] + else: + version_check["passed"] = False + version_check["message"] = "kubectl output has no serverVersion.gitVersion" + except json.JSONDecodeError as exc: + version_check["passed"] = False + version_check["message"] = f"kubectl version output is invalid JSON: {exc}" + + total_nodes = 0 + ready_nodes = 0 + if nodes_check["passed"]: + try: + nodes_payload = json.loads(str(nodes_result["stdout"])) + items = nodes_payload.get("items") if isinstance(nodes_payload, dict) else None + if not isinstance(items, list): + raise ValueError("kubectl node output has no items list") + total_nodes = len(items) + ready_nodes = sum( + any( + isinstance(condition, dict) + and condition.get("type") == "Ready" + and condition.get("status") == "True" + for condition in (node.get("status", {}).get("conditions", []) if isinstance(node, dict) else []) + ) + for node in items + ) + except (json.JSONDecodeError, ValueError) as exc: + nodes_check["passed"] = False + nodes_check["message"] = str(exc) + + inventory_check = { + "name": "non-empty-cluster", + "passed": total_nodes > 0, + "message": f"found {total_nodes} node(s)" if total_nodes else "cluster contains no nodes", + } + readiness_check = { + "name": "ready-node", + "passed": ready_nodes > 0, + "message": f"found {ready_nodes}/{total_nodes} Ready node(s)" if ready_nodes else "cluster has no Ready nodes", + } + checks = [version_check, nodes_check, inventory_check, readiness_check] + success = all(check["passed"] is True for check in checks) + envelope: dict[str, Any] = { + "success": success, + "platform": "kubernetes", + "operation": "kubernetes-preflight", + "kubeconfig_source": "workflow arguments" if kubeconfig else "kubectl environment/default resolution", + "server_version": server_version, + "node_count": total_nodes, + "ready_node_count": ready_nodes, + "checks": checks, + "artifacts": { + "api_version": version_check["artifacts"], + "nodes": nodes_check["artifacts"], + }, + } + if not success: + failures = [f"{check['name']}: {check['message']}" for check in checks if check["passed"] is not True] + envelope["error"] = "Kubernetes prerequisite failed: " + "; ".join(failures) + envelope["remediation"] = "Select a reachable cluster and verify Kubernetes API and Ready-node access" + return envelope, 0 if success else 1 + + +def _parser() -> argparse.ArgumentParser: + """Build the provider command-line parser.""" + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="action", required=True) + + prepare = subparsers.add_parser("prepare", help="Install when requested, then verify l8k") + prepare.add_argument("--mode", choices=("verify", "install"), required=True) + prepare.add_argument("--executable", required=True) + prepare.add_argument("--version", default="") + prepare.add_argument("--installer-ref", default="") + prepare.add_argument("--installer-sha256", default="") + prepare.add_argument("--prefix", default="") + prepare.add_argument("--environment-json", default="{}") + prepare.add_argument("--artifact-dir", required=True) + + verify = subparsers.add_parser("verify", help="Verify l8k version and schema") + verify.add_argument("--executable", required=True) + verify.add_argument("--expected-version", default="") + verify.add_argument("--environment-json", default="{}") + verify.add_argument("--artifact-dir", required=True) + + preflight = subparsers.add_parser("preflight", help="Verify Kubernetes API and Ready-node access") + preflight.add_argument("--kubectl-command-json", default="[]") + preflight.add_argument("--workflow-arguments-json", required=True) + preflight.add_argument("--environment-json", default="{}") + preflight.add_argument("--working-dir", required=True) + preflight.add_argument("--artifact-dir", required=True) + + run = subparsers.add_parser("run", help="Run one real Launch Kit workflow command") + run.add_argument("--executable", required=True) + run.add_argument("--command", choices=_WORKFLOW_COMMANDS, required=True) + run.add_argument("--arguments-json", required=True) + run.add_argument("--user-config", default="") + run.add_argument("--environment-json", default="{}") + run.add_argument("--working-dir", required=True) + run.add_argument("--artifact-dir", required=True) + return parser + + +def main(argv: list[str] | None = None) -> int: + """Execute one provider operation and emit a single JSON envelope.""" + args = _parser().parse_args(argv) + try: + if args.action == "prepare": + envelope, exit_code = _prepare(args) + elif args.action == "verify": + envelope, exit_code = _verify(args) + elif args.action == "preflight": + envelope, exit_code = _preflight(args) + else: + envelope, exit_code = _run_workflow(args) + except (FileNotFoundError, OSError, TypeError, ValueError, urllib.error.URLError) as exc: + envelope = { + "success": False, + "platform": "kubernetes", + "operation": args.action, + "error": str(exc), + } + exit_code = 1 + print(json.dumps(envelope)) + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/suites/README.md b/isvctl/configs/suites/README.md index 4c71c3bba..eaaa032b3 100644 --- a/isvctl/configs/suites/README.md +++ b/isvctl/configs/suites/README.md @@ -75,7 +75,11 @@ Suites: [`slurm`](slurm.yaml), [`control-plane`](control-plane.yaml), [`image-registry`](image-registry.yaml), -[`security`](security.yaml). +[`security`](security.yaml), +[`network-operator`](k8s-launch-kit/network-operator.yaml). +The Network Operator Launch Kit integration is unreleased; see the +[Launch Kit integration guide](../../../docs/guides/k8s-launch-kit/network-operator.md) +before running its cluster-mutating workflows. For the domain / script-count / AWS-reference overview see the [my-isv scaffold README](../providers/my-isv/scripts/README.md#domains). @@ -109,6 +113,11 @@ part that broke, and every member runs even after an earlier one fails. A member that needs parameters takes them inline (`- CheckName: {...}`); one that does not stays a single line. +If a member reports its own subtests, the composite forwards them as +`MemberName/probe-name`. Successful parents are summarized automatically by +subtest count in `isvctl` output; failures retain their complete diagnostic +message. This is shared renderer behavior, not a suite option. + Because a composite has no validation class to borrow from, it declares its own `description` (the catalog uses it) and its name must not shadow a class name. A check that wires one purpose-built class — `SerialConsoleCheck`, @@ -207,6 +216,67 @@ its plan item is not platform-scoped. | `switch_syslogs` | test | `providers/my-isv/scripts/observability/log_availability_test.py` | `tests.*.probes.switches_checked`, `log_source`, `entry_count`, `latest_timestamp` | | `switch_kernel_logs` | test | `providers/my-isv/scripts/observability/log_availability_test.py` | `tests.*.probes.switches_checked`, `log_source`, `entry_count`, `latest_timestamp` | +### Network Operator (`k8s-launch-kit/network-operator.yaml`, `k8s-launch-kit/network-operator-use-cases.yaml`) + +Plain suite for Kubernetes Launch Kit Network Operator self-validation. The +generic provider in `providers/k8s-launch-kit/config/provider.yaml` mirrors the real CLI as +separate verify, prerequisite, discover, generate, deploy, and validate steps. +It forwards user-supplied argument arrays and does not own Network Operator, +profile, topology, resource, or validation defaults. The suite binds fifteen +checks (one prerequisite plus fourteen currently supported PRD areas) directly +to the command output that proves them. + +GPUDirect RDMA is registered from Launch Kit's `gpudirect_dmabuf` result family; +the check skips when that family is disabled or not selected and fails on +emitted GPU topology or bandwidth errors. State restoration remains deferred +until Launch Kit provides the required snapshot/restore/verify workflow. + +`k8s-launch-kit/network-operator-use-cases.yaml` reuses those global check classes in six +separate composite tests: RoCE and InfiniBand across SR-IOV, RDMA Shared, and +host-device deployment modes. Each composite includes only checks applicable to +that use case, so unrelated fabric/deployment checks do not appear as skips in +the middle of a run. The Ethernet/RoCE composites carry `ethernet` and `roce`; +the InfiniBand composites carry `infiniband`. All six also carry `gpudirect` +because Launch Kit discovery decides whether the GPUDirect family is applicable. + +`providers/k8s-launch-kit/config/network-operator.yaml` is the production +entrypoint. It uses `l8k` and `kubectl` from `PATH` by default. In one invocation +it executes the six use-case phases sequentially, each with its own preflight, +discover, generate, deploy, validate, and evidence directories. The phases are +independent, so a failed case records a failed overall run but does not prevent +later cases from producing results. Mock executables exist only under +`isvctl/tests/providers/k8s_launch_kit/fixtures/` and are injected by tests. + +```bash +ISVTEST_INCLUDE_UNRELEASED=1 uv run isvctl test run \ + -f isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml \ + --capability kubernetes --no-upload -- -v +``` + +Add `--label ethernet` or `--label infiniband` before `--no-upload` to run only +that fabric's three workflows. Their steps use +`requires_selected_validations`, so the other fabric's mutating commands are +pruned before execution. Use `--label sriov`, `--label rdma_shared`, or +`--label host_device` to run the matching two-fabric deployment mode. Labels +compose, so `--label ethernet --label sriov` selects one use case. Omitting +labels runs all six. + +| Step | Phase | Script | Key JSON Fields | +|------|-------|--------|-----------------| +| `launch_kit_prepare` | setup | `providers/k8s-launch-kit/scripts/adapter.py prepare` | `installed`, `executable`, `checks.{version,schema}`, `artifacts` | +| `launch_kit_verify` | test | `providers/k8s-launch-kit/scripts/adapter.py verify` | `executable`, `checks.{version,schema}`, `artifacts` | +| `launch_kit_kubernetes_preflight` | test | `providers/k8s-launch-kit/scripts/adapter.py preflight` | `server_version`, `node_count`, `ready_node_count`, `checks`, `artifacts` | +| `launch_kit_discover` | test | `providers/k8s-launch-kit/scripts/adapter.py run` -> `l8k discover` | raw `documents`, `argv`, `exit_code`, `artifacts` | +| `launch_kit_generate` | test | `providers/k8s-launch-kit/scripts/adapter.py run` -> `l8k generate` | raw `documents`, `argv`, `exit_code`, `artifacts` | +| `launch_kit_deploy` | test | `providers/k8s-launch-kit/scripts/adapter.py run` -> `l8k deploy` | raw `documents` (currently empty on success), `argv`, `exit_code`, `artifacts` | +| `launch_kit_validate` | test | `providers/k8s-launch-kit/scripts/adapter.py run` -> `l8k validate` | raw static, connectivity, and report-path `documents`, `argv`, `exit_code`, `artifacts` | + +Those are the generic provider's single-workflow names. The grouped production configuration performs +prepare and verify in `setup`, then repeats the remaining five operations under +each custom use-case phase with names such as +`launch_kit_roce_sriov_preflight` through +`launch_kit_roce_sriov_validate`. + ### VM (`vm.yaml`) | Step | Phase | Script | Key JSON Fields | diff --git a/isvctl/configs/suites/k8s-launch-kit/network-operator-use-cases.yaml b/isvctl/configs/suites/k8s-launch-kit/network-operator-use-cases.yaml new file mode 100644 index 000000000..442314f2c --- /dev/null +++ b/isvctl/configs/suites/k8s-launch-kit/network-operator-use-cases.yaml @@ -0,0 +1,187 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Grouped Network Operator Launch Kit use cases for end-to-end execution. +# +# The individual PRD checks remain catalogued in the adjacent +# network-operator.yaml. This +# suite composes those shared check implementations into six concrete profile +# tests. Providers bind each test to that profile's own validate step and pass +# the other real workflow outputs through the standard step context. + +version: "1.0" + +tests: + description: "Network Operator Kubernetes Launch Kit end-to-end use cases" + + settings: + show_skipped_tests: false + + validations: + network_operator_roce_sriov: + step: launch_kit_roce_sriov_validate + checks: + EastWestNetworkRoceSriovCheck: + test_id: "K8S42-16" + labels: ["era", "ethernet", "gpudirect", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "roce", "slow", "sriov"] + requires: [kubernetes] + description: "Ethernet/RoCE with SR-IOV Network RDMA" + prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" + verify_output: "{{ steps.launch_kit_verify | tojson }}" + preflight_output: "{{ steps.launch_kit_roce_sriov_preflight | tojson }}" + discover_output: "{{ steps.launch_kit_roce_sriov_discover | tojson }}" + generate_output: "{{ steps.launch_kit_roce_sriov_generate | tojson }}" + deploy_output: "{{ steps.launch_kit_roce_sriov_deploy | tojson }}" + compose: + - LaunchKitKubernetesPrerequisiteCheck + - LaunchKitTopologyDiscoveryCheck + - LaunchKitDeploymentHealthCheck + - LaunchKitRoceCheck + - LaunchKitSriovReadinessCheck + - LaunchKitSecondaryNetworkCheck + - LaunchKitIcmpConnectivityCheck + - LaunchKitRdmaConnectivityCheck + - LaunchKitRdmaBandwidthCheck + - LaunchKitGpuDirectRdmaCheck + - LaunchKitMultirailCheck + - LaunchKitEvidenceCaptureCheck + + network_operator_infiniband_sriov: + step: launch_kit_infiniband_sriov_validate + checks: + EastWestNetworkInfiniBandSriovCheck: + test_id: "K8S42-17" + labels: ["era", "gpudirect", "infiniband", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "slow", "sriov"] + requires: [kubernetes] + description: "InfiniBand with SR-IOV Network RDMA" + prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" + verify_output: "{{ steps.launch_kit_verify | tojson }}" + preflight_output: "{{ steps.launch_kit_infiniband_sriov_preflight | tojson }}" + discover_output: "{{ steps.launch_kit_infiniband_sriov_discover | tojson }}" + generate_output: "{{ steps.launch_kit_infiniband_sriov_generate | tojson }}" + deploy_output: "{{ steps.launch_kit_infiniband_sriov_deploy | tojson }}" + compose: + - LaunchKitKubernetesPrerequisiteCheck + - LaunchKitTopologyDiscoveryCheck + - LaunchKitDeploymentHealthCheck + - LaunchKitInfiniBandCheck + - LaunchKitSriovReadinessCheck + - LaunchKitSecondaryNetworkCheck + - LaunchKitIcmpConnectivityCheck + - LaunchKitRdmaConnectivityCheck + - LaunchKitRdmaBandwidthCheck + - LaunchKitGpuDirectRdmaCheck + - LaunchKitMultirailCheck + - LaunchKitEvidenceCaptureCheck + + network_operator_roce_rdma_shared: + step: launch_kit_roce_rdma_shared_validate + checks: + EastWestNetworkRoceRdmaSharedCheck: + test_id: "K8S42-18" + labels: ["era", "ethernet", "gpudirect", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "rdma_shared", "roce", "slow"] + requires: [kubernetes] + description: "Ethernet/RoCE with the RDMA Shared Device Plugin" + prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" + verify_output: "{{ steps.launch_kit_verify | tojson }}" + preflight_output: "{{ steps.launch_kit_roce_rdma_shared_preflight | tojson }}" + discover_output: "{{ steps.launch_kit_roce_rdma_shared_discover | tojson }}" + generate_output: "{{ steps.launch_kit_roce_rdma_shared_generate | tojson }}" + deploy_output: "{{ steps.launch_kit_roce_rdma_shared_deploy | tojson }}" + compose: + - LaunchKitKubernetesPrerequisiteCheck + - LaunchKitTopologyDiscoveryCheck + - LaunchKitDeploymentHealthCheck + - LaunchKitRoceCheck + - LaunchKitRdmaSharedCheck + - LaunchKitSecondaryNetworkCheck + - LaunchKitIcmpConnectivityCheck + - LaunchKitRdmaConnectivityCheck + - LaunchKitRdmaBandwidthCheck + - LaunchKitGpuDirectRdmaCheck + - LaunchKitMultirailCheck + - LaunchKitEvidenceCaptureCheck + + network_operator_infiniband_rdma_shared: + step: launch_kit_infiniband_rdma_shared_validate + checks: + EastWestNetworkInfiniBandRdmaSharedCheck: + test_id: "K8S42-19" + labels: ["era", "gpudirect", "infiniband", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "rdma_shared", "slow"] + requires: [kubernetes] + description: "InfiniBand/IPoIB with the RDMA Shared Device Plugin" + prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" + verify_output: "{{ steps.launch_kit_verify | tojson }}" + preflight_output: "{{ steps.launch_kit_infiniband_rdma_shared_preflight | tojson }}" + discover_output: "{{ steps.launch_kit_infiniband_rdma_shared_discover | tojson }}" + generate_output: "{{ steps.launch_kit_infiniband_rdma_shared_generate | tojson }}" + deploy_output: "{{ steps.launch_kit_infiniband_rdma_shared_deploy | tojson }}" + compose: + - LaunchKitKubernetesPrerequisiteCheck + - LaunchKitTopologyDiscoveryCheck + - LaunchKitDeploymentHealthCheck + - LaunchKitInfiniBandCheck + - LaunchKitRdmaSharedCheck + - LaunchKitSecondaryNetworkCheck + - LaunchKitIcmpConnectivityCheck + - LaunchKitRdmaConnectivityCheck + - LaunchKitRdmaBandwidthCheck + - LaunchKitGpuDirectRdmaCheck + - LaunchKitMultirailCheck + - LaunchKitEvidenceCaptureCheck + + network_operator_roce_host_device: + step: launch_kit_roce_host_device_validate + checks: + EastWestNetworkRoceHostDeviceCheck: + test_id: "K8S42-20" + labels: ["era", "ethernet", "gpudirect", "host_device", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "roce", "slow"] + requires: [kubernetes] + description: "Ethernet/RoCE host-device networking for worker VMs" + prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" + verify_output: "{{ steps.launch_kit_verify | tojson }}" + preflight_output: "{{ steps.launch_kit_roce_host_device_preflight | tojson }}" + discover_output: "{{ steps.launch_kit_roce_host_device_discover | tojson }}" + generate_output: "{{ steps.launch_kit_roce_host_device_generate | tojson }}" + deploy_output: "{{ steps.launch_kit_roce_host_device_deploy | tojson }}" + compose: + - LaunchKitKubernetesPrerequisiteCheck + - LaunchKitTopologyDiscoveryCheck + - LaunchKitDeploymentHealthCheck + - LaunchKitRoceCheck + - LaunchKitHostDeviceCheck + - LaunchKitSecondaryNetworkCheck + - LaunchKitIcmpConnectivityCheck + - LaunchKitRdmaConnectivityCheck + - LaunchKitRdmaBandwidthCheck + - LaunchKitGpuDirectRdmaCheck + - LaunchKitMultirailCheck + - LaunchKitEvidenceCaptureCheck + + network_operator_infiniband_host_device: + step: launch_kit_infiniband_host_device_validate + checks: + EastWestNetworkInfiniBandHostDeviceCheck: + test_id: "K8S42-21" + labels: ["era", "gpudirect", "host_device", "infiniband", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "slow"] + requires: [kubernetes] + description: "InfiniBand host-device networking for worker VMs" + prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" + verify_output: "{{ steps.launch_kit_verify | tojson }}" + preflight_output: "{{ steps.launch_kit_infiniband_host_device_preflight | tojson }}" + discover_output: "{{ steps.launch_kit_infiniband_host_device_discover | tojson }}" + generate_output: "{{ steps.launch_kit_infiniband_host_device_generate | tojson }}" + deploy_output: "{{ steps.launch_kit_infiniband_host_device_deploy | tojson }}" + compose: + - LaunchKitKubernetesPrerequisiteCheck + - LaunchKitTopologyDiscoveryCheck + - LaunchKitDeploymentHealthCheck + - LaunchKitInfiniBandCheck + - LaunchKitHostDeviceCheck + - LaunchKitSecondaryNetworkCheck + - LaunchKitIcmpConnectivityCheck + - LaunchKitRdmaConnectivityCheck + - LaunchKitRdmaBandwidthCheck + - LaunchKitGpuDirectRdmaCheck + - LaunchKitMultirailCheck + - LaunchKitEvidenceCaptureCheck diff --git a/isvctl/configs/suites/k8s-launch-kit/network-operator.yaml b/isvctl/configs/suites/k8s-launch-kit/network-operator.yaml new file mode 100644 index 000000000..9aa055f92 --- /dev/null +++ b/isvctl/configs/suites/k8s-launch-kit/network-operator.yaml @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Network Operator validation through the Kubernetes Launch Kit CLI. +# +# This suite contains catalog and result interpretation only. Providers own the +# command sequence and bind these checks to actual l8k command output. The +# generic provider is ../../providers/k8s-launch-kit/config/provider.yaml. + +version: "1.0" + +tests: + description: "Network Operator self-validation through Kubernetes Launch Kit" + + settings: + show_skipped_tests: false + + validations: + network_operator: + checks: + LaunchKitKubernetesPrerequisiteCheck: + step: launch_kit_kubernetes_preflight + test_id: "N/A" + labels: ["era", "kubernetes", "ncp", "network_operator", "prerequisite", "slow"] + requires: [kubernetes] + + LaunchKitDeploymentHealthCheck: + step: launch_kit_validate + test_id: "K8S42-01" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + + LaunchKitSriovReadinessCheck: + step: launch_kit_validate + test_id: "K8S42-02" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + discover_output: "{{ steps.launch_kit_discover | tojson }}" + + LaunchKitRdmaConnectivityCheck: + step: launch_kit_validate + test_id: "K8S42-03" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + + LaunchKitRoceCheck: + step: launch_kit_validate + test_id: "K8S42-04" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + discover_output: "{{ steps.launch_kit_discover | tojson }}" + + LaunchKitInfiniBandCheck: + step: launch_kit_validate + test_id: "K8S42-05" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + discover_output: "{{ steps.launch_kit_discover | tojson }}" + + LaunchKitHostDeviceCheck: + step: launch_kit_validate + test_id: "K8S42-06" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + discover_output: "{{ steps.launch_kit_discover | tojson }}" + + LaunchKitGpuDirectRdmaCheck: + step: launch_kit_validate + test_id: "K8S42-07" + labels: ["era", "gpudirect", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + + LaunchKitTopologyDiscoveryCheck: + step: launch_kit_discover + test_id: "K8S42-08" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + + LaunchKitSecondaryNetworkCheck: + step: launch_kit_validate + test_id: "K8S42-09" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + discover_output: "{{ steps.launch_kit_discover | tojson }}" + + LaunchKitRdmaSharedCheck: + step: launch_kit_validate + test_id: "K8S42-10" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + discover_output: "{{ steps.launch_kit_discover | tojson }}" + + LaunchKitIcmpConnectivityCheck: + step: launch_kit_validate + test_id: "K8S42-11" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + + LaunchKitRdmaBandwidthCheck: + step: launch_kit_validate + test_id: "K8S42-12" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + + LaunchKitMultirailCheck: + step: launch_kit_validate + test_id: "K8S42-13" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + discover_output: "{{ steps.launch_kit_discover | tojson }}" + + LaunchKitEvidenceCaptureCheck: + step: launch_kit_validate + test_id: "K8S42-15" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" + verify_output: "{{ steps.launch_kit_verify | tojson }}" + preflight_output: "{{ steps.launch_kit_kubernetes_preflight | tojson }}" + discover_output: "{{ steps.launch_kit_discover | tojson }}" + generate_output: "{{ steps.launch_kit_generate | tojson }}" + deploy_output: "{{ steps.launch_kit_deploy | tojson }}" diff --git a/isvctl/src/isvctl/cli/test.py b/isvctl/src/isvctl/cli/test.py index 88c9ee163..9228129c5 100644 --- a/isvctl/src/isvctl/cli/test.py +++ b/isvctl/src/isvctl/cli/test.py @@ -165,6 +165,23 @@ def _reported_capability(config: RunConfig, capability_context: str | None) -> s return capability_context +def _validation_result_detail(validation: dict[str, Any], reason: str | None = None) -> str: + """Return concise success text while preserving skip and failure diagnostics.""" + message = str(validation.get("message", "")) + summary = validation.get("subtest_summary") + is_success = validation.get("passed", False) and not validation.get("skipped") + if validation.get("state") != "error" and is_success and isinstance(summary, dict): + passed = int(summary.get("passed", 0) or 0) + failed = int(summary.get("failed", 0) or 0) + skipped = int(summary.get("skipped", 0) or 0) + total = int(summary.get("total", passed + failed + skipped) or 0) + if total > 0: + if passed == total: + return f"{total} subtests passed" + return f"{total} subtests: {passed} passed, {failed} failed, {skipped} skipped" + return f"{reason}: {message}" if reason and message else str(reason or message) + + def _human_readable_dry_run( config: RunConfig, capability: str | None, @@ -667,19 +684,28 @@ def run( typer.echo("ORCHESTRATION RESULTS") typer.echo("=" * 60) + show_skipped_tests = bool(config.tests and config.tests.settings.get("show_skipped_tests", False)) for phase_result in result.phases: + phase_details = phase_result.details or {} + displayed_validations = phase_details.get("validations", []) + if not show_skipped_tests: + displayed_validations = [ + validation for validation in displayed_validations if not validation.get("skipped") + ] + if phase_details.get("validations") and not displayed_validations and not phase_details.get("steps"): + continue if phase_result.message.startswith("SKIPPED:"): status = typer.style("[SKIP]", fg=typer.colors.YELLOW) elif phase_result.success: status = typer.style("[PASS]", fg=typer.colors.GREEN) else: status = typer.style("[FAIL]", fg=typer.colors.RED) - phase_name = phase_result.phase.value.upper().ljust(8) + phase_name = (phase_result.name or phase_result.phase.value).upper().ljust(24) typer.echo(f"{status} {phase_name}: {phase_result.message}") # Display step details (schema validation, errors) - if phase_result.details and "steps" in phase_result.details: - for step in phase_result.details["steps"]: + if "steps" in phase_details: + for step in phase_details["steps"]: step_name = step.get("name", "unknown") step_success = step.get("success", False) schema_valid = step.get("schema_valid", True) @@ -709,31 +735,28 @@ def run( typer.echo(f" Output: {json.dumps(output, indent=2)[:500]}") # Display centralized validation results - if phase_result.details and "validations" in phase_result.details: - validations = phase_result.details["validations"] - if validations: - for vr in validations: - vr_name = vr.get("name", "unknown") - # Handle case where name might be a dict (extract class name) - if isinstance(vr_name, dict): - vr_name = next(iter(vr_name.keys()), "unknown") - vr_message = vr.get("message", "") - vr_category = vr.get("category", "") - category_prefix = f"[{vr_category}] " if vr_category else "" - if vr.get("state") == "error": - vr_status = typer.style("ERROR", fg=typer.colors.RED) - reason = vr.get("error_reason") - elif vr.get("skipped"): - vr_status = typer.style("SKIPPED", fg=typer.colors.YELLOW) - reason = vr.get("skip_reason") - elif vr.get("passed", False): - vr_status = typer.style("PASSED", fg=typer.colors.GREEN) - reason = None - else: - vr_status = typer.style("FAILED", fg=typer.colors.RED) - reason = None - detail = f"{reason}: {vr_message}" if reason and vr_message else (reason or vr_message) - typer.echo(f" {category_prefix}{vr_name}: {vr_status} - {detail}") + if displayed_validations: + for vr in displayed_validations: + vr_name = vr.get("name", "unknown") + # Handle case where name might be a dict (extract class name) + if isinstance(vr_name, dict): + vr_name = next(iter(vr_name.keys()), "unknown") + vr_category = vr.get("category", "") + category_prefix = f"[{vr_category}] " if vr_category else "" + if vr.get("state") == "error": + vr_status = typer.style("ERROR", fg=typer.colors.RED) + reason = vr.get("error_reason") + elif vr.get("skipped"): + vr_status = typer.style("SKIPPED", fg=typer.colors.YELLOW) + reason = vr.get("skip_reason") + elif vr.get("passed", False): + vr_status = typer.style("PASSED", fg=typer.colors.GREEN) + reason = None + else: + vr_status = typer.style("FAILED", fg=typer.colors.RED) + reason = None + detail = _validation_result_detail(vr, reason) + typer.echo(f" {category_prefix}{vr_name}: {vr_status} - {detail}") typer.echo("-" * 60) if result.success: diff --git a/isvctl/src/isvctl/config/output_schemas.py b/isvctl/src/isvctl/config/output_schemas.py index 5cf3eb3ed..8ff9a0221 100644 --- a/isvctl/src/isvctl/config/output_schemas.py +++ b/isvctl/src/isvctl/config/output_schemas.py @@ -944,6 +944,50 @@ "additionalProperties": True, "description": "Generic schema for unrecognized step names", }, + "k8s_launch_kit": { + "type": "object", + "required": ["success", "platform", "operation"], + "properties": { + **COMMON_PROPERTIES, + "operation": { + "type": "string", + "enum": [ + "prepare", + "verify", + "kubernetes-preflight", + "discover", + "generate", + "deploy", + "validate", + "clean", + ], + "description": "The actual Launch Kit or provider prerequisite operation", + }, + "executable": {"type": "string"}, + "argv": {"type": "array", "items": {"type": "string"}}, + "working_directory": { + "type": "string", + "description": "Absolute working directory used for the Launch Kit command", + }, + "exit_code": {"type": "integer"}, + "documents": { + "type": "array", + "items": {"type": "object"}, + "description": "Unmodified JSON documents emitted by l8k", + }, + "checks": { + "oneOf": [ + {"type": "object"}, + {"type": "array", "items": {"type": "object"}}, + ] + }, + "artifacts": {"type": "object"}, + "error": {"type": "string"}, + "remediation": {"type": "string"}, + }, + "additionalProperties": True, + "description": "Transport envelope around an unmodified Kubernetes Launch Kit CLI operation", + }, # ========================================================================= # Multi-cluster schemas # ========================================================================= diff --git a/isvctl/src/isvctl/config/schema.py b/isvctl/src/isvctl/config/schema.py index 5cf37495c..83149fa8b 100644 --- a/isvctl/src/isvctl/config/schema.py +++ b/isvctl/src/isvctl/config/schema.py @@ -83,7 +83,10 @@ class StepConfig(BaseModel): default_factory=list, description="Command arguments (supports Jinja2 templating with {{ steps.prev_step.field }})", ) - timeout: int = Field(default=300, description="Timeout in seconds") + timeout: int | None = Field( + default=300, + description="Timeout in seconds; null disables the orchestration watchdog", + ) env: dict[str, str] = Field(default_factory=dict, description="Additional environment variables") working_dir: str | None = Field(default=None, description="Working directory for command execution") skip: bool = Field(default=False, description="Skip this step") @@ -100,7 +103,23 @@ class StepConfig(BaseModel): "Unreleased validations are available only when ISVTEST_INCLUDE_UNRELEASED=1." ), ) + requires_selected_validations: list[str] = Field( + default_factory=list, + description=( + "Configured validation names that must be selected after release, capability, label, and suite " + "exclusion filtering for this step to run. A failed step is also reported as an error on these " + "owning validations." + ), + ) continue_on_failure: bool = Field(default=False, description="Continue to next step even if this step fails") + finalizer_for: str | None = Field( + default=None, + min_length=1, + description=( + "Step whose attempted execution activates this finalizer. A finalizer may be in the target phase " + "or in the teardown phase; it runs immediately after the target phase validations." + ), + ) phase: str = Field( default="setup", description="Phase this step belongs to: 'setup', 'test', or 'teardown'", @@ -160,11 +179,76 @@ class PlatformCommands(BaseModel): default_factory=lambda: ["setup", "teardown"], description="Ordered list of phases to execute. Steps are grouped by phase and run in this order.", ) + continue_after_failure: list[str] = Field( + default_factory=list, + description=( + "Phases whose failure must not prevent later phases from running. " + "Use this for independent test cases, never for prerequisite/setup phases." + ), + ) steps: list[StepConfig] = Field( default_factory=list, description="Sequential command steps grouped by phase", ) + @model_validator(mode="after") + def validate_continuation_phases(self) -> "PlatformCommands": + """Reject invalid continuation and linked-finalizer declarations.""" + if len(self.phases) != len(set(self.phases)): + raise ValueError("phases must not contain duplicate names") + unknown = [phase for phase in self.continue_after_failure if phase not in self.phases] + if unknown: + raise ValueError(f"continue_after_failure contains phases not listed in phases: {unknown}") + unsafe = [phase for phase in self.continue_after_failure if phase in {"setup", "teardown"}] + if unsafe: + raise ValueError(f"continue_after_failure cannot contain lifecycle phases: {unsafe}") + if len(self.continue_after_failure) != len(set(self.continue_after_failure)): + raise ValueError("continue_after_failure must not contain duplicate phase names") + + for finalizer in (step for step in self.steps if step.finalizer_for is not None): + targets = [step for step in self.steps if step.name == finalizer.finalizer_for] + if len(targets) != 1: + raise ValueError( + f"step '{finalizer.name}' finalizer_for must name exactly one configured step: " + f"{finalizer.finalizer_for!r}" + ) + target = targets[0] + target_phase = target.phase.lower() + finalizer_phase = finalizer.phase.lower() + if target_phase != finalizer_phase and finalizer_phase != "teardown": + raise ValueError( + f"step '{finalizer.name}' finalizer_for target '{target.name}' must be in the same phase " + "or the finalizer must use phase 'teardown'" + ) + if finalizer_phase == "teardown" and target_phase != finalizer_phase: + normalized_phases = [phase.lower() for phase in self.phases] + if "teardown" not in normalized_phases: + raise ValueError(f"step '{finalizer.name}' uses phase 'teardown', which is not listed in phases") + if target_phase not in normalized_phases: + raise ValueError( + f"step '{finalizer.name}' finalizer_for target '{target.name}' has an unknown phase" + ) + if normalized_phases.index(target_phase) >= normalized_phases.index("teardown"): + raise ValueError(f"step '{finalizer.name}' teardown must be ordered after target '{target.name}'") + if target.finalizer_for is not None: + raise ValueError(f"step '{finalizer.name}' cannot finalize finalizer step '{target.name}'") + gate_fields = ( + "requires", + "requires_available_validations", + "requires_selected_validations", + ) + mismatched_gates = [ + field_name + for field_name in gate_fields + if getattr(finalizer, field_name) != getattr(target, field_name) + ] + if mismatched_gates: + raise ValueError( + f"step '{finalizer.name}' must use the same gates as target '{target.name}': " + + ", ".join(mismatched_gates) + ) + return self + class KubernetesNodeOutput(BaseModel): """Schema for a single Kubernetes node in command output.""" diff --git a/isvctl/src/isvctl/config/suite_resolution.py b/isvctl/src/isvctl/config/suite_resolution.py index d0cfbbd75..8e9fdcf9f 100644 --- a/isvctl/src/isvctl/config/suite_resolution.py +++ b/isvctl/src/isvctl/config/suite_resolution.py @@ -40,7 +40,7 @@ def platform_vocabulary(configs_root: Path) -> frozenset[str]: several entry points ask for the vocabulary two or three times per run. """ platforms: set[str] = set() - for path in (configs_root / "suites").glob("*.yaml"): + for path in (configs_root / "suites").rglob("*.yaml"): try: data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} except (OSError, yaml.YAMLError): @@ -55,7 +55,7 @@ def platform_vocabulary(configs_root: Path) -> frozenset[str]: def suite_vocabulary(configs_root: Path) -> frozenset[str]: """Return plain suite names declared by canonical suite YAML.""" declarable = platform_vocabulary(configs_root) - names = {_normalize_name(path.stem) for path in (configs_root / "suites").glob("*.yaml")} + names = {_normalize_name(path.stem) for path in (configs_root / "suites").rglob("*.yaml")} return frozenset(names - declarable) @@ -162,7 +162,8 @@ def resolve_suite(provider: str | None, suite: str, *, configs_root: Path) -> Re # Best-effort per file: one malformed config must not fail `--suite` for the # whole provider. Matches `resolve_suite_name`, which already skips them. classified = [] - for path in sorted(config_dir.glob("*.yaml")): + pattern = "*.yaml" if provider is not None else "**/*.yaml" + for path in sorted(config_dir.glob(pattern)): try: classified.append((path, *_suite_name(path, declarable))) except SuiteResolutionError: diff --git a/isvctl/src/isvctl/doctor/checks/config.py b/isvctl/src/isvctl/doctor/checks/config.py index 386555691..f81f4e85d 100644 --- a/isvctl/src/isvctl/doctor/checks/config.py +++ b/isvctl/src/isvctl/doctor/checks/config.py @@ -46,7 +46,7 @@ def _check_repo_layout(root: Path) -> list[CheckResult]: results: list[CheckResult] = [] suites_dir = root / "isvctl" / "configs" / "suites" - suite_yamls = sorted(suites_dir.glob("*.yaml")) if suites_dir.is_dir() else [] + suite_yamls = sorted(suites_dir.rglob("*.yaml")) if suites_dir.is_dir() else [] if suite_yamls: results.append( CheckResult( diff --git a/isvctl/src/isvctl/orchestrator/commands.py b/isvctl/src/isvctl/orchestrator/commands.py index 373541343..b03177b54 100644 --- a/isvctl/src/isvctl/orchestrator/commands.py +++ b/isvctl/src/isvctl/orchestrator/commands.py @@ -30,6 +30,7 @@ from isvctl.config.schema import CommandConfig, CommandOutput from isvctl.orchestrator.context import _create_jinja_env +from isvctl.orchestrator.process import run_command_process from isvctl.redaction import mask_sensitive_args logger = logging.getLogger(__name__) @@ -132,12 +133,10 @@ def execute( logger.debug(f"Working directory: {cwd}") try: - result = subprocess.run( + result = run_command_process( cmd_parts, cwd=cwd, env=env, - capture_output=True, - text=True, timeout=config.timeout, ) diff --git a/isvctl/src/isvctl/orchestrator/loop.py b/isvctl/src/isvctl/orchestrator/loop.py index 312bdd00c..e5718b7ab 100644 --- a/isvctl/src/isvctl/orchestrator/loop.py +++ b/isvctl/src/isvctl/orchestrator/loop.py @@ -41,6 +41,7 @@ requirements_satisfied, resolve_class_key, resolve_entries, + resolve_entry_selection, ) from isvtest.main import run_validations_via_pytest from isvtest.release_manifest import INCLUDE_UNRELEASED_ENV, load_released_test_filter @@ -48,7 +49,7 @@ from isvctl.config.schema import RunConfig, StepConfig from isvctl.orchestrator.commands import CommandExecutor from isvctl.orchestrator.context import Context -from isvctl.orchestrator.step_executor import StepExecutor, StepResults +from isvctl.orchestrator.step_executor import StepExecutor, StepResult, StepResults from isvctl.redaction import redact_dict, redact_junit_xml_tree logger = logging.getLogger(__name__) @@ -82,6 +83,7 @@ class PhaseResult: success: bool message: str details: dict[str, Any] | None = None + name: str | None = None @dataclass @@ -281,9 +283,74 @@ def _resolved_entry_to_result_dict(entry: ResolvedEntry) -> dict[str, Any]: "state": entry.state.value if entry.state else None, "skip_reason": entry.skip_reason.value if entry.skip_reason else None, "error_reason": entry.error_reason.value if entry.error_reason else None, + "subtest_summary": { + "total": entry.subtest_summary.total, + "passed": entry.subtest_summary.passed, + "failed": entry.subtest_summary.failed, + "skipped": entry.subtest_summary.skipped, + }, } +def _step_failure_message(result: StepResult) -> str: + """Return an operator-facing diagnostic for one failed workflow step.""" + detail = result.error + if not detail and result.schema_errors: + detail = f"output schema validation failed: {'; '.join(result.schema_errors)}" + if not detail: + detail = f"command exited with code {result.exit_code}" + return f"workflow step '{result.name}' failed: {detail}" + + +def _apply_owned_step_failures( + entries: list[ResolvedEntry], + phase_steps: list[StepConfig], + step_results: StepResults, +) -> list[ResolvedEntry]: + """Turn failed lifecycle steps into errors on their owning validations. + + ``requires_selected_validations`` is both the command-selection gate and + the explicit ownership edge between a workflow step and its selectable + tests. Without this propagation, an early step failure prevents the bound + validation step from producing output and JUnit incorrectly records a + harmless ``step_no_output`` skip. + """ + configs_by_name = {step.name: step for step in phase_steps} + errors_by_validation: dict[str, list[tuple[str, str]]] = {} + + for result in step_results.steps: + if result.success: + continue + step = configs_by_name.get(result.name) + if step is None: + continue + message = _step_failure_message(result) + for validation_name in step.requires_selected_validations: + errors_by_validation.setdefault(validation_name, []).append((step.name, message)) + + propagated: list[ResolvedEntry] = [] + for entry in entries: + step_errors = errors_by_validation.get(entry.entry.name, []) + if entry.is_ready: + # A failed validation-producing step can still return structured + # output that the validation interprets into failures/subtests. + # Only earlier owned lifecycle failures should suppress that run. + step_errors = [(name, message) for name, message in step_errors if name != entry.entry.step] + may_override = entry.is_ready or entry.skip_reason == SkipReason.STEP_NO_OUTPUT + if not step_errors or not may_override: + propagated.append(entry) + continue + propagated.append( + ResolvedEntry( + entry=entry.entry, + state=State.ERROR, + error_reason=ErrorReason.STEP_FAILED, + message="; ".join(message for _, message in step_errors), + ) + ) + return propagated + + def _resolved_entry_success(entry: ResolvedEntry) -> bool: """Return whether a resolved validation outcome should keep the phase successful.""" return entry.state in {State.PASSED, State.SKIPPED} @@ -303,8 +370,7 @@ def _requested_config_phases(config_phases: list[str], requested_phases: list[Ph if Phase.ALL in requested_phases: return config_phases - requested_phase_names = {phase.value for phase in requested_phases} - return [phase for phase in config_phases if phase in requested_phase_names] + return [phase for phase in config_phases if _phase_enum_for_name(phase) in requested_phases] def _has_explicit_pytest_selection(extra_pytest_args: list[str] | None) -> bool: @@ -340,6 +406,50 @@ def _apply_step_validation_gates(steps: list[Any], released_tests: set[str] | No return gated_steps +def _apply_selected_validation_gates( + steps: list[Any], + validation_entries: list[ValidationEntry], + *, + include_labels: set[str], + exclude_labels: set[str], + exclude_tests: set[str], + released_tests: set[str] | None, + capability: str | None, +) -> list[Any]: + """Skip lifecycle steps whose required validations are not selected.""" + entries_by_name = {entry.name: entry for entry in validation_entries} + gated_steps: list[Any] = [] + for step in steps: + required_validations = getattr(step, "requires_selected_validations", []) + unselected: list[str] = [] + for validation_name in required_validations: + entry = entries_by_name.get(validation_name) + if entry is None: + unselected.append(f"{validation_name} (not configured)") + continue + result = resolve_entry_selection( + entry, + include_labels=include_labels, + exclude_labels=exclude_labels, + exclude_tests=exclude_tests, + released_tests=released_tests, + capability=capability, + ) + if result is not None: + unselected.append(f"{validation_name} ({result.message})") + if not unselected: + gated_steps.append(step) + continue + skipped_step = step.model_copy(update={"skip": True}) + logger.info( + "Skipping step '%s' because required validation(s) are not selected: %s", + skipped_step.name, + "; ".join(unselected), + ) + gated_steps.append(skipped_step) + return gated_steps + + def _apply_capability_step_gates( steps: list[Any], validation_entries: list[ValidationEntry], @@ -499,6 +609,7 @@ def _run_steps_mode( phase=_phase_enum_for_name(phase_name), success=True, message=f"SKIPPED: platform '{platform}' is skipped by configuration", + name=phase_name, ) for phase_name in skipped_phases ], @@ -518,6 +629,10 @@ def _run_steps_mode( ], ) + continuation_phases = ( + set(self.config.commands[platform].continue_after_failure) if self.config.commands else set() + ) + released_tests = load_released_test_filter() if released_tests is None: logger.info(f"Including unreleased validations because {INCLUDE_UNRELEASED_ENV} is enabled") @@ -541,6 +656,33 @@ def _run_steps_mode( ) ], ) + exclude_labels: list[str] = [] + exclude_tests: list[str] = [] + if self.config.tests and self.config.tests.exclude: + exclude_labels = self.config.tests.exclude.get("labels", []) + exclude_tests = self.config.tests.exclude.get("tests", []) + skip_config_label_exclusions = bool(self._include_labels) or _has_explicit_pytest_selection( + self._extra_pytest_args + ) + resolution_exclude_labels = set(self._exclude_labels) + if not skip_config_label_exclusions: + resolution_exclude_labels.update(exclude_labels) + + steps_before_selection = steps + steps = _apply_selected_validation_gates( + steps, + validation_entries, + include_labels=set(self._include_labels), + exclude_labels=resolution_exclude_labels, + exclude_tests=set(exclude_tests), + released_tests=released_tests, + capability=self._capability, + ) + selection_skipped_steps = { + selected.name + for original, selected in zip(steps_before_selection, steps, strict=True) + if not original.skip and selected.skip + } steps = _apply_capability_step_gates(steps, validation_entries, self._capability) logger.info(f"Configured phases: {config_phases}") @@ -561,6 +703,8 @@ def _run_steps_mode( steps_by_phase: dict[str, list] = {phase: [] for phase in config_phases} for step in steps: + if step.name in selection_skipped_steps: + continue step_phase = (step.phase or "setup").lower() steps_by_phase[step_phase].append(step) @@ -575,25 +719,30 @@ def _run_steps_mode( step_phase = (step.phase or "setup").lower() self.context.set_step_phase(step.name, step_phase) - resolved_validations_by_index: dict[int, ResolvedEntry] = {} + configured_steps_by_name = {step.name: step for step in steps} + active_steps = [step for phase_steps in steps_by_phase.values() for step in phase_steps] + finalizers_by_target_phase: dict[str, list[StepConfig]] = {} + for finalizer in (step for step in active_steps if step.finalizer_for is not None): + target = configured_steps_by_name[finalizer.finalizer_for] + target_phase = (target.phase or "setup").lower() + finalizers_by_target_phase.setdefault(target_phase, []).append(finalizer) - exclude_labels: list[str] = [] - exclude_tests: list[str] = [] - if self.config.tests and self.config.tests.exclude: - exclude_labels = self.config.tests.exclude.get("labels", []) - exclude_tests = self.config.tests.exclude.get("tests", []) - skip_config_label_exclusions = bool(self._include_labels) or _has_explicit_pytest_selection( - self._extra_pytest_args - ) - resolution_exclude_labels = set(self._exclude_labels) - if not skip_config_label_exclusions: - resolution_exclude_labels.update(exclude_labels) + resolved_validations_by_index: dict[int, ResolvedEntry] = {} phase_results: list[PhaseResult] = [] overall_success = True + block_following_phases = False setup_steps_ran = False requested_phase_names = {p.value for p in requested_phases} + selected_config_phases = _requested_config_phases(config_phases, requested_phases) + selected_config_phase_names = set(selected_config_phases) + selected_finalizer_target_phases = selected_config_phase_names.intersection(finalizers_by_target_phase) + run_finalizers_as_teardown_recovery = ( + "teardown" in selected_config_phase_names and not selected_finalizer_target_phases + ) + attempted_step_names: set[str] = set() + executed_finalizer_names: set[str] = set() # Per-phase JUnit XML files merge at the end so later phases don't # overwrite earlier ones. @@ -605,15 +754,24 @@ def _run_steps_mode( junit_tmpdir = tempfile.mkdtemp(prefix="junit-phases-") for phase_name in config_phases: - if phase_name not in requested_phase_names and Phase.ALL not in requested_phases: + if phase_name not in selected_config_phase_names: continue - phase_steps = steps_by_phase.get(phase_name, []) + configured_phase_steps = steps_by_phase.get(phase_name, []) + phase_steps = [step for step in configured_phase_steps if step.finalizer_for is None] + declared_phase_finalizers = [step for step in configured_phase_steps if step.finalizer_for is not None] + if phase_name == "teardown" and run_finalizers_as_teardown_recovery: + phase_steps.extend(declared_phase_finalizers) + phase_finalizers = [ + step + for step in finalizers_by_target_phase.get(phase_name, []) + if step.name not in executed_finalizer_names + ] phase_enum = _phase_enum_for_name(phase_name) is_teardown = phase_name == "teardown" skip_reason: str | None = None - if not overall_success and not is_teardown: + if block_following_phases and not is_teardown: skip_reason = "previous phase failed" # Teardown gating depends on whether setup was part of this run: @@ -636,6 +794,7 @@ def _run_steps_mode( phase=phase_enum, success=True, message=f"SKIPPED: {skip_reason}", + name=phase_name, ) ) continue @@ -644,6 +803,7 @@ def _run_steps_mode( step_results = self.step_executor.execute_steps(phase_steps, self.context, best_effort=is_teardown) else: step_results = StepResults() + attempted_step_names.update(result.name for result in step_results.steps if result.attempted) # ``step_results.steps`` includes placeholder records for skip:true # steps; require at least one step that wasn't skipped before letting @@ -666,12 +826,17 @@ def _run_steps_mode( phase_entries = [validation_entries[index] for index in phase_entry_indexes] resolved_phase_entries = self._resolve_validation_entries( phase_entries, - requested_phase_names if Phase.ALL not in requested_phases else set(config_phases), + selected_config_phase_names, set(self._include_labels), resolution_exclude_labels, set(exclude_tests), released_tests, ) + resolved_phase_entries = _apply_owned_step_failures( + resolved_phase_entries, + phase_steps, + step_results, + ) ready_entries = [entry for entry in resolved_phase_entries if entry.is_ready] terminal_before_pytest = [entry for entry in resolved_phase_entries if not entry.is_ready] @@ -718,14 +883,54 @@ def _run_steps_mode( phase_validations = [_resolved_entry_to_result_dict(entry) for entry in terminal_phase_entries] - if phase_steps or phase_validations: + if step_results.steps or phase_validations: phase_results.append( self._create_phase_result(phase_enum, step_results, phase_validations, phase_name) ) + eligible_finalizers = [step for step in phase_finalizers if step.finalizer_for in attempted_step_names] + for finalizer in phase_finalizers: + if finalizer not in eligible_finalizers: + logger.info( + "Skipping finalizer '%s': target step '%s' was not attempted", + finalizer.name, + finalizer.finalizer_for, + ) + finalizer_results = self.step_executor.execute_steps( + eligible_finalizers, + self.context, + best_effort=True, + ) + executed_finalizer_names.update(finalizer.name for finalizer in eligible_finalizers) + if eligible_finalizers: + phase_results.append( + self._create_phase_result( + Phase.TEARDOWN, + finalizer_results, + [], + f"{phase_name}-teardown", + ) + ) + elif phase_finalizers: + target_names = ", ".join(finalizer.finalizer_for or "unknown" for finalizer in phase_finalizers) + phase_results.append( + PhaseResult( + phase=Phase.TEARDOWN, + success=True, + message=f"SKIPPED: target step(s) were not attempted: {target_names}", + details={"steps": [], "validations": []}, + name=f"{phase_name}-teardown", + ) + ) + phase_success = step_results.success and all(v.get("passed", False) for v in phase_validations) if not phase_success: overall_success = False + if phase_name not in continuation_phases: + block_following_phases = True + if not finalizer_results.success: + overall_success = False + block_following_phases = True remaining_entries = [ (index, entry) @@ -735,7 +940,7 @@ def _run_steps_mode( if remaining_entries: terminal_remaining = self._resolve_remaining_validation_entries( remaining_entries, - requested_phase_names if Phase.ALL not in requested_phases else set(config_phases), + selected_config_phase_names, set(self._include_labels), resolution_exclude_labels, set(exclude_tests), @@ -829,6 +1034,7 @@ def _create_phase_result( { "name": s.name, "success": s.success, + "attempted": s.attempted, "error": s.error, "output": redact_dict(s.output), "schema_name": s.schema_name, @@ -839,6 +1045,7 @@ def _create_phase_result( ], "validations": validation_results, }, + name=display_name, ) def _resolve_validation_entries( @@ -926,6 +1133,7 @@ def _append_resolution_only_phase_results( "steps": [], "validations": [_resolved_entry_to_result_dict(entry) for entry in resolved_entries], }, + name=phase_name, ) ) diff --git a/isvctl/src/isvctl/orchestrator/process.py b/isvctl/src/isvctl/orchestrator/process.py new file mode 100644 index 000000000..ab39123ac --- /dev/null +++ b/isvctl/src/isvctl/orchestrator/process.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Subprocess execution shared by orchestration command models.""" + +import os +import signal +import subprocess +from collections.abc import Mapping, Sequence +from pathlib import Path + +_TERMINATION_GRACE_SECONDS = 2.0 + + +def run_command_process( + args: Sequence[str], + *, + cwd: str | Path, + env: Mapping[str, str] | None, + timeout: float | None, +) -> subprocess.CompletedProcess[str]: + """Run an orchestration command and terminate its process group on timeout. + + Orchestration steps commonly invoke wrappers which then start a provider + CLI. Killing only the wrapper can leave that CLI running after the step is + reported as timed out. On POSIX, every command therefore starts in a new + session and timeout handling signals the whole process group. Other + platforms fall back to terminating the direct child process. + + Args: + args: Command and arguments to execute without a shell. + cwd: Working directory for the command. + env: Complete process environment, or ``None`` to inherit it. + timeout: Maximum execution time in seconds, or ``None`` for no limit. + + Returns: + The completed process with captured text stdout and stderr. + + Raises: + subprocess.TimeoutExpired: The command exceeded ``timeout``. Captured + stdout and stderr are attached after the process tree is stopped. + OSError: The command could not be started. + """ + command = list(args) + process = subprocess.Popen( + command, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=os.name == "posix", + ) + + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + _terminate_process_tree(process) + try: + stdout, stderr = process.communicate(timeout=_TERMINATION_GRACE_SECONDS) + except subprocess.TimeoutExpired: + _kill_process_tree(process) + stdout, stderr = process.communicate() + else: + # The direct child may exit while a descendant that closed the + # inherited pipes remains alive. Ensure the process group is gone. + _kill_process_tree(process) + + raise subprocess.TimeoutExpired(command, timeout, output=stdout, stderr=stderr) from None + + return subprocess.CompletedProcess(command, process.returncode, stdout, stderr) + + +def _terminate_process_tree(process: subprocess.Popen[str]) -> None: + """Request graceful termination of a process group or direct process.""" + try: + if os.name == "posix": + os.killpg(process.pid, signal.SIGTERM) + else: + process.terminate() + except ProcessLookupError: + pass + + +def _kill_process_tree(process: subprocess.Popen[str]) -> None: + """Force termination of a process group or direct process.""" + try: + if os.name == "posix": + os.killpg(process.pid, signal.SIGKILL) + else: + process.kill() + except ProcessLookupError: + pass diff --git a/isvctl/src/isvctl/orchestrator/step_executor.py b/isvctl/src/isvctl/orchestrator/step_executor.py index 9f8945e44..d4feacc48 100644 --- a/isvctl/src/isvctl/orchestrator/step_executor.py +++ b/isvctl/src/isvctl/orchestrator/step_executor.py @@ -60,6 +60,7 @@ from isvctl.config.output_schemas import get_schema_for_step, validate_output from isvctl.config.schema import StepConfig from isvctl.orchestrator.context import Context, _create_jinja_env +from isvctl.orchestrator.process import run_command_process from isvctl.redaction import mask_sensitive_args, redact_text logger = logging.getLogger(__name__) @@ -199,6 +200,7 @@ class StepResult: schema_errors: Schema validation error messages validation_results: Results from bound validations error: Error message if step failed + attempted: Whether the command process was actually started """ name: str @@ -212,6 +214,7 @@ class StepResult: schema_errors: list[str] = field(default_factory=list) validation_results: list[dict[str, Any]] = field(default_factory=list) error: str | None = None + attempted: bool = True @dataclass @@ -288,6 +291,7 @@ def execute_steps( stdout="", stderr="", error="Step skipped", + attempted=False, ) ) continue @@ -335,6 +339,7 @@ def _execute_step(self, step: StepConfig, context: Context) -> StepResult: stdout="", stderr="", error=f"Skipped: missing step reference steps.{e.missing_path}", + attempted=False, ) # Normalize command - replace python/python3 with current interpreter @@ -378,12 +383,10 @@ def _execute_step(self, step: StepConfig, context: Context) -> StepResult: logger.debug(f"Working directory: {cwd}") try: - result = subprocess.run( + result = run_command_process( cmd_parts, cwd=cwd, env=env, - capture_output=True, - text=True, timeout=step.timeout, ) @@ -438,6 +441,7 @@ def _execute_step(self, step: StepConfig, context: Context) -> StepResult: stdout="", stderr="", error=f"Command not found: {step.command}", + attempted=False, ) except Exception as e: return StepResult( diff --git a/isvctl/tests/providers/k8s_launch_kit/__init__.py b/isvctl/tests/providers/k8s_launch_kit/__init__.py new file mode 100644 index 000000000..77ba303a9 --- /dev/null +++ b/isvctl/tests/providers/k8s_launch_kit/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the Kubernetes Launch Kit provider.""" diff --git a/isvctl/tests/providers/k8s_launch_kit/fixtures/launch_kit_scenarios.json b/isvctl/tests/providers/k8s_launch_kit/fixtures/launch_kit_scenarios.json new file mode 100644 index 000000000..c1fe18205 --- /dev/null +++ b/isvctl/tests/providers/k8s_launch_kit/fixtures/launch_kit_scenarios.json @@ -0,0 +1,104 @@ +{ + "_copyright": "Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.", + "_license": "Apache-2.0", + "contract": { + "source_commit": "db32e4b98170", + "source_files": [ + "pkg/config/config.go", + "pkg/ui/json_output.go", + "pkg/cmd/clean.go", + "pkg/cmd/deploy.go", + "pkg/cmd/discover.go", + "pkg/cmd/generate.go", + "pkg/cmd/validate.go", + "pkg/networkoperatorplugin/clean.go", + "pkg/networkoperatorplugin/validate.go", + "pkg/networkoperatorplugin/connectivity/connectivity.go", + "pkg/networkoperatorplugin/connectivity/daemonset.go", + "pkg/networkoperatorplugin/connectivity/matrix.go", + "pkg/networkoperatorplugin/connectivity/result.go", + "pkg/networkoperatorplugin/connectivity/rdma.go", + "pkg/networkoperatorplugin/discovery/discover.go", + "profiles/host-device-rdma/40-example-daemonset.yaml", + "profiles/ipoib-rdma-shared/40-example-daemonset.yaml", + "profiles/macvlan-rdma-shared/40-example-daemonset.yaml", + "profiles/sriov-ethernet-rdma/60-example-daemonset.yaml", + "profiles/sriov-ib-rdma/60-example-daemonset.yaml" + ], + "supported_validation_checks": [ + "icmp", + "rping", + "ib_write_bw" + ], + "result_families": [ + "icmp", + "rping", + "ib_write_bw", + "gpudirect_dmabuf" + ], + "notes": [ + "discover and generate emit one ui.JSONResult object", + "successful standalone deploy emits no stdout object", + "validate emits manifest, connectivity, and reportPath JSON documents", + "connectivity DaemonSets declare test-container for RDMA and netshoot for ICMP", + "enabled GPUDirect validation follows ib_write_bw and emits a distinct gpudirect_dmabuf result family", + "clean emits one ui.JSONResult object with a cleanup summary" + ] + }, + "scenarios": { + "roce-sriov": { + "fabric": "ethernet", + "deployment": "sriov", + "profile_name": "SR-IOV Ethernet RDMA", + "network_kind": "SriovNetwork", + "network_api_version": "sriovnetwork.openshift.io/v1", + "requires_sriov": true, + "requires_ib": false + }, + "infiniband-sriov": { + "fabric": "infiniband", + "deployment": "sriov", + "profile_name": "SR-IOV Infiniband RDMA", + "network_kind": "SriovIBNetwork", + "network_api_version": "sriovnetwork.openshift.io/v1", + "requires_sriov": true, + "requires_ib": true + }, + "roce-rdma-shared": { + "fabric": "ethernet", + "deployment": "rdma_shared", + "profile_name": "Macvlan with RDMA shared device", + "network_kind": "MacvlanNetwork", + "network_api_version": "mellanox.com/v1alpha1", + "requires_sriov": false, + "requires_ib": false + }, + "infiniband-rdma-shared": { + "fabric": "infiniband", + "deployment": "rdma_shared", + "profile_name": "IP over Infiniband with RDMA shared device", + "network_kind": "IPoIBNetwork", + "network_api_version": "mellanox.com/v1alpha1", + "requires_sriov": false, + "requires_ib": true + }, + "roce-host-device": { + "fabric": "ethernet", + "deployment": "host_device", + "profile_name": "Host device RDMA", + "network_kind": "HostDeviceNetwork", + "network_api_version": "mellanox.com/v1alpha1", + "requires_sriov": false, + "requires_ib": false + }, + "infiniband-host-device": { + "fabric": "infiniband", + "deployment": "host_device", + "profile_name": "Host device RDMA", + "network_kind": "HostDeviceNetwork", + "network_api_version": "mellanox.com/v1alpha1", + "requires_sriov": false, + "requires_ib": true + } + } +} diff --git a/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_kubectl.py b/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_kubectl.py new file mode 100755 index 000000000..737863593 --- /dev/null +++ b/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_kubectl.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Small kubectl-compatible test double for Launch Kit prerequisite checks.""" + +from __future__ import annotations + +import json +import os +import sys + + +def main(argv: list[str] | None = None) -> int: + """Return Kubernetes version or Ready-node JSON for the requested command.""" + args = list(sys.argv[1:] if argv is None else argv) + if os.environ.get("L8K_MOCK_KUBERNETES_FAIL") == "1": + print("Unable to connect to the server: connection refused", file=sys.stderr) + return 1 + expected_kubeconfig = os.environ.get("L8K_MOCK_EXPECT_KUBECONFIG") + if expected_kubeconfig and os.environ.get("KUBECONFIG") != expected_kubeconfig: + print( + f"expected KUBECONFIG={expected_kubeconfig!r}, got {os.environ.get('KUBECONFIG')!r}", + file=sys.stderr, + ) + return 1 + if "version" in args: + print( + json.dumps( + { + "clientVersion": {"gitVersion": "v1.34.1"}, + "serverVersion": {"gitVersion": "v1.34.1"}, + } + ) + ) + return 0 + if "get" in args and "nodes" in args: + print( + json.dumps( + { + "apiVersion": "v1", + "items": [ + { + "metadata": {"name": "worker-a"}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}, + }, + { + "metadata": {"name": "worker-b"}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}, + }, + ], + } + ) + ) + return 0 + print(f"mock kubectl does not support: {' '.join(args)}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_l8k.py b/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_l8k.py new file mode 100755 index 000000000..dcf5a41a4 --- /dev/null +++ b/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_l8k.py @@ -0,0 +1,706 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Executable Launch Kit test double for provider unit tests. + +Values in this file are fixed mock output, not AI Cloud Validation defaults. +The provider passes only real l8k arguments and receives the same distinct +stdout forms used by version, schema, discover, generate, deploy, validate, and clean. +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path +from typing import Any + +import yaml + +_TIMESTAMP = "2026-08-05T08:27:43Z" +_SOURCE_COMMIT = "db32e4b98170" +_FIXTURE = Path(__file__).with_name("launch_kit_scenarios.json") +_VALUE_FLAGS: dict[str, set[str]] = { + "version": {"--output"}, + "discover": { + "--kubeconfig", + "--save-cluster-config", + "--user-config", + "--fabric", + "--deployment-type", + "--multirail", + "--node-selector", + "--network-operator-release", + "--output", + }, + "generate": { + "--user-config", + "--save-deployment-files", + "--network-operator-namespace", + "--output", + }, + "deploy": { + "--kubeconfig", + "--user-config", + "--deployment-files", + "--network-operator-namespace", + "--deploy-timeout", + "--output", + }, + "validate": { + "--kubeconfig", + "--user-config", + "--deployment-files", + "--network-operator-namespace", + "--connectivity", + "--connectivity-timeout", + "--validation-mode", + "--validation-checks", + "--rdma-rping-iterations", + "--rdma-ib-write-size", + "--rdma-ib-write-min-bandwidth-gbps", + "--wait", + "--report-path", + "--output", + }, + "clean": { + "--kubeconfig", + "--user-config", + "--network-operator-namespace", + "--keep-helm-chart", + "--output", + }, +} +_BOOLEAN_FLAGS: dict[str, set[str]] = { + "clean": {"--keep-helm-chart"}, +} + + +def _load_fixture() -> dict[str, Any]: + """Load the pinned mock contract and scenario definitions.""" + value = json.loads(_FIXTURE.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError("Launch Kit fixture must contain an object") + return value + + +def _parse_flags(command: str, argv: list[str]) -> dict[str, str]: + """Parse the real flag subset exercised by the provider tests.""" + supported = _VALUE_FLAGS[command] + values: dict[str, str] = {} + index = 0 + while index < len(argv): + token = argv[index] + if not token.startswith("--"): + raise ValueError(f"unexpected positional argument: {token}") + if "=" in token: + flag, value = token.split("=", 1) + else: + flag = token + if flag in _BOOLEAN_FLAGS.get(command, set()): + value = "true" + else: + index += 1 + if index >= len(argv): + raise ValueError(f"missing value for {flag}") + value = argv[index] + if flag not in supported: + raise ValueError(f"unknown flag for l8k {command}: {flag}") + values[flag] = value + index += 1 + return values + + +def _emit(value: dict[str, Any], *, pretty: bool = False) -> None: + """Write one JSON document to stdout.""" + print(json.dumps(value, indent=2 if pretty else None)) + + +def _message(level: str, message: str) -> dict[str, str]: + """Build one ui.LogEntry-compatible record.""" + return {"level": level, "message": message, "timestamp": _TIMESTAMP} + + +def _profile(scenario: dict[str, Any]) -> dict[str, str]: + """Return the profile fields emitted by Launch Kit.""" + return { + "deployment": str(scenario["deployment"]), + "fabric": str(scenario["fabric"]), + "ignoreARP": "false", + "multirail": "true", + "routing": "destination-based", + } + + +def _json_result( + phase: str, + *, + profile: dict[str, str] | None = None, + generated_files: list[str] | None = None, +) -> dict[str, Any]: + """Build a successful ui.JSONResult-compatible object.""" + value: dict[str, Any] = { + "success": True, + "phase": phase, + "deployed": False, + "messages": [ + _message("info", f"Running {phase}"), + _message("success", "Workflow completed successfully"), + ], + } + if profile is not None: + value["profile"] = profile + if generated_files: + value["generatedFiles"] = generated_files + return value + + +def _structured_error(command: str, message: str) -> tuple[dict[str, Any], int]: + """Build the JSON error emitted by a failed Launch Kit command.""" + category = "cluster" if command == "discover" else "deployment" if command == "deploy" else "validation" + exit_code = 3 if category == "cluster" else 4 if category == "deployment" else 2 + return { + "success": False, + "phase": "", + "deployed": False, + "error": { + "code": f"{category.upper()}_ERROR", + "message": message, + "category": category, + "transient": category == "cluster", + "suggestion": "Inspect the preserved Launch Kit logs and correct the reported condition", + }, + "messages": None, + }, exit_code + + +def _scenario_for_profile(fabric: str, deployment: str) -> tuple[str, dict[str, Any]]: + """Find fixture data for one explicit profile.""" + scenarios = _load_fixture().get("scenarios") + if not isinstance(scenarios, dict): + raise ValueError("Launch Kit fixture has no scenarios map") + for name, scenario in scenarios.items(): + if isinstance(scenario, dict) and scenario.get("fabric") == fabric and scenario.get("deployment") == deployment: + return str(name), scenario + raise ValueError(f"unsupported mock profile: fabric={fabric!r}, deployment={deployment!r}") + + +def _load_cluster_config(path: Path) -> dict[str, Any]: + """Load a cluster configuration produced by the mock discovery command.""" + value = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"{path} must contain a YAML object") + return value + + +def _scenario_from_config(flags: dict[str, str]) -> tuple[str, dict[str, Any]]: + """Resolve a scenario from the persisted Launch Kit profile.""" + raw = flags.get("--user-config") + if not raw: + deployment = Path(flags.get("--deployment-files", "deployment")) + raw = str(deployment.parent / "cluster-config.yaml") + config = _load_cluster_config(Path(raw)) + profile = config.get("profile") + if not isinstance(profile, dict): + raise ValueError("cluster config has no profile") + return _scenario_for_profile(str(profile.get("fabric", "")), str(profile.get("deployment", ""))) + + +def _cluster_config_yaml(scenario: dict[str, Any]) -> str: + """Render representative output from l8k discovery.""" + return f"""# Generated by `l8k discover`. +networkOperator: + selectedRelease: "26.4" + version: v26.4.1 + namespace: nvidia-network-operator +validation: + gpuDirect: + enabled: true + gpuResourceType: nvidia.com/gpu + connectivity: true + mode: strict + checks: [icmp, rping, ib_write_bw] + rdma: + rpingIterations: 5 + ibWriteSize: 65536 + ibWriteMinBandwidthGbps: 100 +profile: + fabric: {scenario["fabric"]} + deployment: {scenario["deployment"]} + multirail: true + routing: destination-based +clusterConfig: + - identifier: mock-group + machineType: mock-vm + gpuType: NVIDIA-H100-80GB-HBM3 + workerNodes: [worker-a, worker-b] + nodeSelector: + feature.node.kubernetes.io/pci-15b3.present: "true" + pfs: + - networkInterface: ens5f0np0 + pciAddress: "0000:17:00.0" + rdmaDevice: mlx5_0 + traffic: east-west + rail: 0 + connectedGPU: GPU0 + connectedGPUPCIAddress: "0000:41:00.0" + - networkInterface: ens6f0np0 + pciAddress: "0000:31:00.0" + rdmaDevice: mlx5_1 + traffic: east-west + rail: 1 + connectedGPU: GPU1 + connectedGPUPCIAddress: "0000:71:00.0" +""" + + +def _resource_name(kind: str, rail: int) -> str: + """Return a deterministic resource name for generated mock manifests.""" + prefix = { + "IPPool": "nv-ipam-pool", + "SriovNetwork": "sriov-network", + "SriovIBNetwork": "sriov-ib-network", + "MacvlanNetwork": "macvlan-network", + "IPoIBNetwork": "ipoib-network", + "HostDeviceNetwork": "hostdev-network", + "SriovNetworkNodePolicy": "sriov-policy", + "NicNodePolicy": "nic-node-policy", + }.get(kind, kind.lower()) + return f"{prefix}-rail-{rail}-mock-group" + + +def _manifest_specs(scenario: dict[str, Any]) -> list[tuple[str, str, str, int]]: + """Return generated API/kind/file/count tuples for one profile.""" + specs = [ + ("mellanox.com/v1alpha1", "NicClusterPolicy", "10-nic-cluster-policy", 1), + ("configuration.net.nvidia.com/v1alpha1", "NicNodePolicy", "20-nic-node-policy", 1), + ("nv-ipam.nvidia.com/v1alpha1", "IPPool", "30-ip-pool", 2), + ] + if scenario.get("requires_sriov"): + specs.append(("sriovnetwork.openshift.io/v1", "SriovNetworkNodePolicy", "40-sriov-policy", 2)) + specs.append( + ( + str(scenario["network_api_version"]), + str(scenario["network_kind"]), + "50-secondary-network", + 2, + ) + ) + return specs + + +def _manifest_yaml(api_version: str, kind: str, count: int) -> str: + """Render a valid multi-document mock manifest.""" + documents: list[str] = [] + for rail in range(count): + name = "nic-cluster-policy" if kind == "NicClusterPolicy" else _resource_name(kind, rail) + namespace = "" if kind in {"NicClusterPolicy", "NicNodePolicy"} else " namespace: default\n" + documents.append( + f"apiVersion: {api_version}\nkind: {kind}\nmetadata:\n name: {name}\n{namespace}spec:\n mock: true\n" + ) + return "---\n".join(documents) + + +def _write_generated_files(root: Path, scenario: dict[str, Any]) -> list[str]: + """Materialize profile manifests and return their absolute paths.""" + manifest_dir = root / "network-operator" + manifest_dir.mkdir(parents=True, exist_ok=True) + generated: list[Path] = [] + values = manifest_dir / "values.yaml" + values.write_text("operator:\n namespace: nvidia-network-operator\n", encoding="utf-8") + generated.append(values) + for api_version, kind, stem, count in _manifest_specs(scenario): + path = manifest_dir / f"{stem}.yaml" + path.write_text(_manifest_yaml(api_version, kind, count), encoding="utf-8") + generated.append(path) + example = manifest_dir / "60-example-daemonset-mock-group.yaml" + example.write_text( + "apiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: l8k-network-test\n namespace: default\n" + "spec:\n" + " selector:\n" + " matchLabels:\n" + " app: l8k-network-test\n" + " template:\n" + " metadata:\n" + " labels:\n" + " app: l8k-network-test\n" + " spec:\n" + " containers:\n" + " - name: test-container\n" + " image: nvcr.io/nvidia/doca/doca:3.3.0-full-rt-host\n" + " command: [/bin/bash, -c, sleep infinity]\n" + " resources:\n" + " requests:\n" + " nvidia.com/gpu: '2'\n" + " limits:\n" + " nvidia.com/gpu: '2'\n" + " - name: netshoot\n" + " image: nicolaka/netshoot:latest\n" + " command: [/bin/bash, -c, sleep infinity]\n", + encoding="utf-8", + ) + generated.append(example) + return [str(path.resolve()) for path in generated] + + +def _manifest_results(scenario: dict[str, Any]) -> list[dict[str, Any]]: + """Build the exported Launch Kit manifest-validation results.""" + results: list[dict[str, Any]] = [] + for api_version, kind, stem, count in _manifest_specs(scenario): + for rail in range(count): + name = "nic-cluster-policy" if kind == "NicClusterPolicy" else _resource_name(kind, rail) + reason = "resource exists and is Ready" + results.append( + { + "Kind": kind, + "APIVersion": api_version, + "Name": name, + "Namespace": "" if kind in {"NicClusterPolicy", "NicNodePolicy"} else "default", + "SourceFile": f"{stem}.yaml", + "State": "success", + "Reason": reason, + "Details": {}, + "Found": True, + "Missing": False, + "Detail": reason, + } + ) + return results + + +def _ping_result( + kind: int, + src_node: str, + dst_node: str, + src_rail: str, + dst_rail: str, + fail_family: str | None, +) -> dict[str, Any]: + """Build one exported connectivity matrix row.""" + family = "icmp" if kind < 2 else "rping" if kind < 4 else "ib_write_bw" if kind < 6 else "gpudirect_dmabuf" + bandwidth_family = family in {"ib_write_bw", "gpudirect_dmabuf"} + cross_rail = src_rail != dst_rail + expectation = "forbidden" if cross_rail else "required" + observed_ok = not cross_rail + ok = True + stderr = "" + stdout = "" + bandwidth = 0.0 + if family == "icmp": + stdout = "1 packets transmitted, 1 received" if observed_ok else "" + stderr = "Network is unreachable" if cross_rail else "" + elif family == "rping": + stdout = "client DISCONNECT EVENT" if observed_ok else "" + stderr = "rping: connection timed out" if cross_rail else "" + elif observed_ok: + bandwidth = 191.25 if family == "gpudirect_dmabuf" else 187.6 + stdout = f"65536 5000 {bandwidth:.2f} {bandwidth:.2f} 0.3578" + else: + stderr = "ib_write_bw: failed to connect" + + if fail_family == family and not cross_rail and src_node == "worker-a" and src_rail == "rail-0": + ok = False + observed_ok = False + if bandwidth_family: + bandwidth = 42.5 + stderr = "observed bandwidth 42.5 Gbps below minimum 100 Gbps" + else: + stderr = f"{family}: connection refused" + + test = { + "Kind": kind, + "SrcPod": f"network-test-{src_node}", + "DstPod": f"network-test-{dst_node}", + "SrcNode": src_node, + "DstNode": dst_node, + "Rail": src_rail if not cross_rail else f"{src_rail}→{dst_rail}", + "SrcIP": "192.168.128.10" if src_node == "worker-a" else "192.168.128.11", + "DstIP": "192.168.128.11" if dst_node == "worker-b" else "192.168.128.10", + "SrcRail": src_rail, + "DstRail": dst_rail, + "SrcIface": "net1" if src_rail == "rail-0" else "net2", + "DstIface": "net1" if dst_rail == "rail-0" else "net2", + "SrcRDMADev": "mlx5_0" if src_rail == "rail-0" else "mlx5_1", + "DstRDMADev": "mlx5_0" if dst_rail == "rail-0" else "mlx5_1", + "Expectation": expectation, + } + if family == "gpudirect_dmabuf": + test.update( + { + "SrcGPUIndex": 0 if src_rail == "rail-0" else 1, + "DstGPUIndex": 0 if dst_rail == "rail-0" else 1, + "SrcGPUPCIAddress": "0000:41:00.0" if src_rail == "rail-0" else "0000:71:00.0", + "DstGPUPCIAddress": "0000:41:00.0" if dst_rail == "rail-0" else "0000:71:00.0", + } + ) + return { + "Test": test, + "Family": family, + "OK": ok, + "ObservedOK": observed_ok, + "Expectation": expectation, + "Route": {"OK": not cross_rail}, + "BandwidthGbps": bandwidth, + "MsgRateMpps": 0.3578 if bandwidth else 0.0, + "MinBandwidthGbps": 100.0 if bandwidth_family else 0.0, + "Stdout": stdout, + "Stderr": stderr, + **({"Error": stderr} if not ok else {}), + } + + +def _connectivity_result(scenario_name: str, fail_family: str | None) -> dict[str, Any]: + """Build a strict two-node, two-rail matrix.""" + rails = ["rail-0", "rail-1"] + rows: list[dict[str, Any]] = [] + for kind in range(8): + for src_node, dst_node in (("worker-a", "worker-b"), ("worker-b", "worker-a")): + pairs = [(rail, rail) for rail in rails] if kind % 2 == 0 else [(rails[0], rails[1]), (rails[1], rails[0])] + for src_rail, dst_rail in pairs: + rows.append(_ping_result(kind, src_node, dst_node, src_rail, dst_rail, fail_family)) + failed = sum(row["OK"] is not True for row in rows) + return { + "DaemonSets": [ + { + "Ref": { + "Namespace": "default", + "Name": f"l8k-network-test-{scenario_name}", + "Container": "test-container", + "RDMAContainer": "test-container", + "ICMPContainer": "netshoot", + "SourceFile": "60-example-daemonset-mock-group.yaml", + }, + "Rollout": {"Desired": 2, "Updated": 2, "Available": 2, "Ready": 2, "NotReady": 0}, + "PodCount": 2, + } + ], + "PingResults": rows, + "Skipped": None, + "Summary": {"TotalTests": len(rows), "Passed": len(rows) - failed, "Failed": failed}, + } + + +def _failure(command: str) -> tuple[bool, str | None]: + """Resolve optional failure injection as ``command[:family]``.""" + parts = os.environ.get("L8K_MOCK_FAIL", "").split(":") + if not parts or parts[0] != command: + return False, None + return len(parts) == 1, parts[1] if len(parts) > 1 else None + + +def _run_version(flags: dict[str, str]) -> int: + """Mock ``l8k version``.""" + if flags.get("--output") == "json": + _emit({"version": "v0.1.0-mock", "gitCommit": _SOURCE_COMMIT, "buildDate": _TIMESTAMP}, pretty=True) + else: + print("l8k v0.1.0-mock") + return 0 + + +def _run_schema() -> int: + """Mock ``l8k schema`` with Launch Kit-owned capabilities and defaults.""" + fixture = _load_fixture() + _emit( + { + "version": "v0.1.0-mock", + "description": "CLI tool for deploying NVIDIA cloud-native networking solutions on Kubernetes", + "commands": { + command: {"description": f"Mock l8k {command}", "example": f"l8k {command}"} + for command in ("discover", "generate", "deploy", "validate", "clean", "schema") + }, + "phases": ["discover", "generate", "deploy"], + "fabrics": sorted({str(value["fabric"]) for value in fixture["scenarios"].values()}), + "deploymentTypes": sorted({str(value["deployment"]) for value in fixture["scenarios"].values()}), + "outputFormats": ["text", "json"], + "supportedNetworkOperatorReleases": ["26.4"], + "exitCodes": {"0": "success", "2": "validation_error", "3": "cluster_error", "4": "deployment_error"}, + "flags": { + "--node-selector": { + "type": "string", + "default": "feature.node.kubernetes.io/pci-15b3.present=true", + "description": "Node selector", + }, + "--validation-mode": { + "type": "string", + "default": "inherit from validation.mode", + "description": "Validation mode", + }, + "--validation-checks": { + "type": "[]string", + "default": "inherit from validation.checks", + "description": ( + "Comma-separated checks: icmp, rping, ib_write_bw. " + "Enabled GPUDirect DMA-BUF validation follows ib_write_bw." + ), + }, + }, + }, + pretty=True, + ) + return 0 + + +def _run_discover(flags: dict[str, str]) -> int: + """Mock ``l8k discover``.""" + fail, _ = _failure("discover") + if fail: + result, exit_code = _structured_error("discover", "cluster discovery failed") + _emit(result, pretty=True) + print("Error: cluster discovery failed", file=sys.stderr) + return exit_code + _, scenario = _scenario_for_profile(flags.get("--fabric", ""), flags.get("--deployment-type", "")) + path = Path(flags.get("--save-cluster-config", "cluster-config.yaml")) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(_cluster_config_yaml(scenario), encoding="utf-8") + _emit(_json_result("discover", profile=_profile(scenario)), pretty=True) + print(f"[success] Configuration saved: {path}", file=sys.stderr) + return 0 + + +def _run_generate(flags: dict[str, str]) -> int: + """Mock ``l8k generate``.""" + fail, _ = _failure("generate") + if fail: + result, exit_code = _structured_error("generate", "deployment file generation failed") + _emit(result, pretty=True) + return exit_code + _, scenario = _scenario_from_config(flags) + root = Path(flags.get("--save-deployment-files", "deployment")) + files = _write_generated_files(root, scenario) + _emit(_json_result("generate", profile=_profile(scenario), generated_files=files), pretty=True) + print(f"[success] Generated {len(files)} files", file=sys.stderr) + return 0 + + +def _run_deploy(flags: dict[str, str]) -> int: + """Mock standalone ``l8k deploy`` including empty success stdout.""" + fail, _ = _failure("deploy") + if fail: + result, exit_code = _structured_error("deploy", "deployment failed") + _emit(result, pretty=True) + print("Error: deployment failed", file=sys.stderr) + return exit_code + _scenario_from_config(flags) + print(f"[success] Deployment completed from {flags.get('--deployment-files', 'deployment')}", file=sys.stderr) + return 0 + + +def _run_validate(flags: dict[str, str]) -> int: + """Mock the current three-document ``l8k validate`` JSON stream.""" + fail, family = _failure("validate") + if fail: + result, exit_code = _structured_error("validate", "failed to create Kubernetes client") + _emit(result) + print("Error: failed to create Kubernetes client", file=sys.stderr) + return exit_code + scenario_name, scenario = _scenario_from_config(flags) + manifests = _manifest_results(scenario) + static = { + "versionCheck": { + "Skipped": False, + "Reason": "", + "SelectedRelease": "26.4", + "ExpectedVersion": "v26.4.1", + "DeployedRelease": { + "Name": "network-operator", + "Namespace": "nvidia-network-operator", + "ChartName": "network-operator", + "ChartVersion": "26.4.1", + "AppVersion": "v26.4.1", + "Revision": 1, + "Status": "deployed", + }, + "Match": True, + }, + "manifests": manifests, + "presetDeviations": [], + "summary": { + "totalManifests": len(manifests), + "successManifests": len(manifests), + "inProgress": 0, + "errorManifests": 0, + "missingManifests": 0, + "versionMatch": True, + "deviationGroups": 0, + "success": True, + }, + } + connectivity = _connectivity_result(scenario_name, family) + deployment = Path(flags.get("--deployment-files", "deployment")) + report = Path(flags.get("--report-path", str(deployment / "k8s-launch-kit-validation-report.html"))).resolve() + report.parent.mkdir(parents=True, exist_ok=True) + verdict = "FAILED" if connectivity["Summary"]["Failed"] else "PASSED" + report.write_text(f"

VALIDATION {verdict}

\n", encoding="utf-8") + _emit(static) + _emit({"connectivity": connectivity}) + _emit({"reportPath": str(report)}) + print(f"HTML report written to {report}", file=sys.stderr) + return 4 if connectivity["Summary"]["Failed"] else 0 + + +def _run_clean(flags: dict[str, str]) -> int: + """Mock the current one-document ``l8k clean`` JSON result.""" + fail, _ = _failure("clean") + if fail: + result, exit_code = _structured_error("clean", "Network Operator cleanup failed") + _emit(result, pretty=True) + print("Error: Network Operator cleanup failed", file=sys.stderr) + return exit_code + keep_helm = flags.get("--keep-helm-chart", "false").lower() == "true" + _emit( + { + "success": True, + "phase": "clean", + "deployed": False, + "cleanup": { + "namespace": flags.get("--network-operator-namespace", "nvidia-network-operator"), + "customResourcesDeleted": 12, + "helmReleaseRemoved": not keep_helm, + "keepHelmChart": keep_helm, + }, + "messages": [], + }, + pretty=True, + ) + return 0 + + +def main(argv: list[str] | None = None) -> int: + """Execute one mocked Launch Kit command.""" + args = list(sys.argv[1:] if argv is None else argv) + if not args: + print("mock l8k expects a command", file=sys.stderr) + return 2 + command = args[0] + try: + if command == "schema": + if len(args) != 1: + raise ValueError("l8k schema accepts no arguments") + return _run_schema() + if command not in _VALUE_FLAGS: + raise ValueError(f"unknown l8k command: {command}") + flags = _parse_flags(command, args[1:]) + if command in {"discover", "generate", "deploy", "validate", "clean"} and flags.get("--output") != "json": + raise ValueError("mock workflow commands require --output json") + if command == "version": + return _run_version(flags) + if command == "discover": + return _run_discover(flags) + if command == "generate": + return _run_generate(flags) + if command == "deploy": + return _run_deploy(flags) + if command == "validate": + return _run_validate(flags) + return _run_clean(flags) + except (KeyError, OSError, TypeError, ValueError, yaml.YAMLError) as exc: + result, exit_code = _structured_error(command, str(exc)) + _emit(result, pretty=command != "validate") + print(f"Error: {exc}", file=sys.stderr) + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/tests/providers/k8s_launch_kit/test_provider.py b/isvctl/tests/providers/k8s_launch_kit/test_provider.py new file mode 100644 index 000000000..75e6fff16 --- /dev/null +++ b/isvctl/tests/providers/k8s_launch_kit/test_provider.py @@ -0,0 +1,1220 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Contract and framework tests for the generic Kubernetes Launch Kit provider.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import os +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest +import yaml +from isvtest.core.resolution import ErrorReason, State + +from isvctl.config.merger import merge_yaml_files +from isvctl.config.output_schemas import validate_output +from isvctl.config.schema import RunConfig +from isvctl.orchestrator.loop import Orchestrator, Phase + +_ISVCTL_ROOT = Path(__file__).resolve().parents[3] +_PROVIDERS = _ISVCTL_ROOT / "configs" / "providers" +_LAUNCH_KIT_PROVIDER = _PROVIDERS / "k8s-launch-kit" +_PROVIDER = _LAUNCH_KIT_PROVIDER / "scripts" / "adapter.py" +_FIXTURES = Path(__file__).resolve().parent / "fixtures" +_MOCK_L8K = _FIXTURES / "mock_l8k.py" +_MOCK_KUBECTL = _FIXTURES / "mock_kubectl.py" +_GENERIC_CONFIG = _LAUNCH_KIT_PROVIDER / "config" / "provider.yaml" +_NETWORK_OPERATOR_CONFIG = _LAUNCH_KIT_PROVIDER / "config" / "network-operator.yaml" + + +def _load_provider_module() -> ModuleType: + """Load the provider script for isolated installer tests.""" + spec = importlib.util.spec_from_file_location("k8s_launch_kit_provider", _PROVIDER) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot import {_PROVIDER}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _run_provider( + *arguments: str, + env: dict[str, str] | None = None, +) -> tuple[subprocess.CompletedProcess[str], dict[str, Any]]: + """Run one provider operation from the same directory used by isvctl.""" + completed = subprocess.run( + [sys.executable, str(_PROVIDER), *arguments], + cwd=_PROVIDERS, + env=env, + check=False, + capture_output=True, + text=True, + ) + output = json.loads(completed.stdout) + assert isinstance(output, dict) + return completed, output + + +def _run_workflow( + command: str, + arguments: list[str], + *, + working_dir: Path, + artifact_dir: Path, + user_config: Path | None = None, + env: dict[str, str] | None = None, +) -> tuple[subprocess.CompletedProcess[str], dict[str, Any]]: + """Run a mocked l8k workflow command through the generic transport.""" + provider_arguments = [ + "run", + "--executable", + str(_MOCK_L8K), + "--command", + command, + "--arguments-json", + json.dumps(arguments), + "--environment-json", + "{}", + "--working-dir", + str(working_dir), + "--artifact-dir", + str(artifact_dir), + ] + if user_config is not None: + provider_arguments.extend(["--user-config", str(user_config)]) + return _run_provider(*provider_arguments, env=env) + + +def _mocked_network_operator_config(tmp_path: Path) -> RunConfig: + """Load production wiring, then inject test-owned executables and paths.""" + merged = merge_yaml_files([_NETWORK_OPERATOR_CONFIG]) + context = merged["context"]["k8s_launch_kit"] + context["executable"] = str(_MOCK_L8K) + context["kubectl_command"] = [sys.executable, str(_MOCK_KUBECTL)] + context["shared_artifact_dir"] = str(tmp_path / "shared-evidence") + for name, use_case in context["use_cases"].items(): + use_case["working_dir"] = str(tmp_path / "use-cases" / name / "work") + use_case["artifact_dir"] = str(tmp_path / "use-cases" / name / "evidence") + return RunConfig.model_validate(merged) + + +def test_generic_provider_has_no_launch_kit_domain_defaults() -> None: + """AI Cloud Validation exposes raw argv while Launch Kit owns domain defaults.""" + merged = merge_yaml_files([_GENERIC_CONFIG]) + config = RunConfig.model_validate(merged) + context = merged["context"]["k8s_launch_kit"] + + assert set(context) == { + "executable", + "installation", + "user_config", + "kubectl_command", + "working_dir", + "artifact_dir", + "environment", + "discover", + "generate", + "deploy", + "validate", + "clean", + } + assert context["user_config"] == "" + assert context["installation"] == { + "mode": "verify", + "version": "", + "installer_ref": "", + "installer_sha256": "", + "prefix": "", + } + assert all( + context[command]["arguments"] == [] for command in ("discover", "generate", "deploy", "validate", "clean") + ) + assert [step.name for step in config.commands["network_operator"].steps] == [ + "launch_kit_prepare", + "launch_kit_verify", + "launch_kit_kubernetes_preflight", + "launch_kit_discover", + "launch_kit_generate", + "launch_kit_deploy", + "launch_kit_validate", + "launch_kit_clean", + ] + assert config.commands["network_operator"].phases == ["setup", "test", "teardown"] + discover_step = next( + step for step in config.commands["network_operator"].steps if step.name == "launch_kit_discover" + ) + prepare_step = next(step for step in config.commands["network_operator"].steps if step.name == "launch_kit_prepare") + assert "--installer-ref={{ context.k8s_launch_kit.installation.installer_ref }}" in prepare_step.args + assert "--installer-sha256={{ context.k8s_launch_kit.installation.installer_sha256 }}" in prepare_step.args + assert "--user-config={{ context.k8s_launch_kit.user_config }}" in discover_step.args + assert config.commands["network_operator"].steps[-1].phase == "teardown" + assert config.commands["network_operator"].steps[-1].finalizer_for == "launch_kit_deploy" + forbidden = { + "namespace", + "node_selector", + "expected_network_operator_version", + "driver_mode", + "rail_names", + "sriov_resource_names", + "ip_pool_names", + "gpu_count", + "validation_mode", + "validation_checks", + "rdma_rping_iterations", + "rdma_ib_write_size", + "rdma_min_bandwidth_gbps", + "timeout_seconds", + } + assert forbidden.isdisjoint(context) + + +def test_network_operator_provider_defaults_to_real_cli_tools() -> None: + """The shipped use-case provider cannot select repository test doubles.""" + merged = merge_yaml_files([_NETWORK_OPERATOR_CONFIG]) + config = RunConfig.model_validate(merged) + context = merged["context"]["k8s_launch_kit"] + + assert context["executable"] == "l8k" + assert context["installation"]["installer_ref"] == "" + assert context["installation"]["installer_sha256"] == "" + assert context["user_config"] == "" + assert context["kubectl_command"] == [] + assert "mock" not in json.dumps(merged).lower() + assert "poc" not in json.dumps(merged).lower() + assert len(config.commands["network_operator"].steps) == 38 + assert config.commands["network_operator"].phases[-1] == "teardown" + discover_steps = [step for step in config.commands["network_operator"].steps if step.name.endswith("_discover")] + assert len(discover_steps) == 6 + assert all("--user-config={{ context.k8s_launch_kit.user_config }}" in step.args for step in discover_steps) + prepare_step = next(step for step in config.commands["network_operator"].steps if step.name == "launch_kit_prepare") + assert "--installer-ref={{ context.k8s_launch_kit.installation.installer_ref }}" in prepare_step.args + assert "--installer-sha256={{ context.k8s_launch_kit.installation.installer_sha256 }}" in prepare_step.args + clean_steps = [step for step in config.commands["network_operator"].steps if step.name.endswith("_clean")] + assert len(clean_steps) == 6 + assert all(step.phase == "teardown" for step in clean_steps) + assert all(step.finalizer_for and step.finalizer_for.endswith("_deploy") for step in clean_steps) + + +def test_network_operator_workflows_use_launch_kit_default_paths() -> None: + """Grouped use cases leave config and deployment paths to Launch Kit.""" + merged = merge_yaml_files([_NETWORK_OPERATOR_CONFIG]) + use_cases = merged["context"]["k8s_launch_kit"]["use_cases"] + default_path_flags = { + "--user-config", + "--deployment-files", + "--save-cluster-config", + "--save-deployment-files", + } + + for use_case in use_cases.values(): + all_arguments = { + argument + for phase in ("discover", "generate", "deploy", "validate", "clean") + for argument in use_case[phase]["arguments"] + } + assert default_path_flags.isdisjoint(all_arguments) + assert use_case["discover"]["arguments"][0] == "--fabric" + assert use_case["generate"]["arguments"] == [] + assert use_case["deploy"]["arguments"] == [] + assert use_case["validate"]["arguments"] == [] + assert use_case["clean"]["arguments"] == [] + + +def test_kubectl_defaults_to_the_real_binary() -> None: + """An empty provider override resolves to kubectl from PATH.""" + module = _load_provider_module() + + assert module._kubectl_prefix("[]", {}) == ["kubectl"] + + +def test_launch_kit_executable_resolves_from_path(tmp_path: Path, monkeypatch: Any) -> None: + """The production `l8k` setting is resolved as a normal executable.""" + module = _load_provider_module() + executable = tmp_path / "l8k" + executable.write_text("test executable", encoding="utf-8") + monkeypatch.setattr(module.shutil, "which", lambda value: str(executable) if value == "l8k" else None) + + assert module._resolve_executable("l8k") == executable.resolve() + + +def test_prepare_verifies_version_and_schema(tmp_path: Path) -> None: + """Verify mode proves the executable and captures Launch Kit's schema.""" + completed, output = _run_provider( + "prepare", + "--mode", + "verify", + "--executable", + str(_MOCK_L8K), + "--artifact-dir", + str(tmp_path), + ) + + assert completed.returncode == 0 + assert output["success"] is True + assert output["operation"] == "prepare" + assert output["installed"] is False + assert set(output["checks"]) == {"version", "schema"} + assert all(check["passed"] is True for check in output["checks"].values()) + assert validate_output(output, "k8s_launch_kit") == (True, []) + + +def test_verification_rejects_an_unexpected_launch_kit_version(tmp_path: Path) -> None: + """A pinned installation cannot silently verify a different binary on PATH.""" + module = _load_provider_module() + + verification, success, error = module._verify_executable( + _MOCK_L8K, + tmp_path, + "v9.9.9", + ) + + assert success is False + assert verification["checks"]["version"]["passed"] is False + assert error == "l8k version mismatch: expected 'v9.9.9', got 'v0.1.0-mock'" + + +def test_verification_requires_the_launch_kit_clean_command(tmp_path: Path) -> None: + """A pre-clean Launch Kit binary is rejected before deployment begins.""" + module = _load_provider_module() + executable = tmp_path / "l8k" + executable.write_text( + "#!/bin/sh\n" + 'if [ "$1" = version ]; then\n' + ' echo \'{"version": "v0.1.0"}\'\n' + "else\n" + ' echo \'{"commands": {"discover": {}, "generate": {}, "deploy": {}, "validate": {}}}\'\n' + "fi\n", + encoding="utf-8", + ) + executable.chmod(0o755) + + verification, success, error = module._verify_executable(executable, tmp_path) + + assert success is False + assert verification["checks"]["schema"]["passed"] is False + assert error == "l8k schema does not advertise required command(s): clean" + + +def test_installed_executable_is_resolved_from_the_installer_prefix(tmp_path: Path) -> None: + """Install mode verifies the binary written by the installer, not a stale PATH entry.""" + module = _load_provider_module() + executable = tmp_path / "bin" / "l8k" + executable.parent.mkdir(parents=True) + executable.write_text("mock", encoding="utf-8") + + assert module._installed_executable(str(tmp_path)) == executable.resolve() + + +def test_installer_download_verifies_expected_digest(tmp_path: Path, monkeypatch: Any) -> None: + """Install mode verifies an immutable official installer before writing it.""" + module = _load_provider_module() + content = b"#!/bin/sh\nset -eu\n" + installer_ref = "a" * 40 + expected_sha256 = hashlib.sha256(content).hexdigest() + + class Response: + """Minimal context-managed urllib response.""" + + def __enter__(self) -> Response: + return self + + def __exit__(self, *_args: Any) -> None: + return None + + def read(self) -> bytes: + return content + + monkeypatch.setattr(module.urllib.request, "urlopen", lambda *_args, **_kwargs: Response()) + + installer, url = module._download_installer(installer_ref, expected_sha256, tmp_path) + metadata = json.loads((tmp_path / "installer-download.json").read_text(encoding="utf-8")) + + assert installer.read_bytes() == content + assert url.endswith(f"/{installer_ref}/scripts/install.sh") + assert metadata == { + "url": url, + "ref": installer_ref, + "expected_sha256": expected_sha256, + "sha256": expected_sha256, + "verified": True, + } + + +def test_installer_download_rejects_a_digest_mismatch(tmp_path: Path, monkeypatch: Any) -> None: + """A downloaded installer is never persisted when its trusted digest differs.""" + module = _load_provider_module() + content = b"#!/bin/sh\nexit 0\n" + + class Response: + """Minimal context-managed urllib response.""" + + def __enter__(self) -> Response: + return self + + def __exit__(self, *_args: Any) -> None: + return None + + def read(self) -> bytes: + return content + + monkeypatch.setattr(module.urllib.request, "urlopen", lambda *_args, **_kwargs: Response()) + + with pytest.raises(ValueError, match="installer SHA-256 mismatch"): + module._download_installer("b" * 40, "0" * 64, tmp_path) + + metadata = json.loads((tmp_path / "installer-download.json").read_text(encoding="utf-8")) + assert metadata["verified"] is False + assert metadata["sha256"] == hashlib.sha256(content).hexdigest() + assert not (tmp_path / "installer.sh").exists() + + +@pytest.mark.parametrize("installer_ref", ["", "main", "v0.1.0", "a" * 39]) +def test_installer_download_requires_an_immutable_commit_ref(tmp_path: Path, installer_ref: str) -> None: + """Install mode rejects mutable or abbreviated installer references before download.""" + module = _load_provider_module() + + with pytest.raises(ValueError, match="full 40-character Git commit SHA"): + module._download_installer(installer_ref, "0" * 64, tmp_path) + + +def test_install_mode_delegates_to_the_upstream_installer(tmp_path: Path, monkeypatch: Any) -> None: + """The provider does not reimplement Launch Kit archive or checksum logic.""" + module = _load_provider_module() + installer = tmp_path / "installer.sh" + installer.write_text("#!/bin/sh\n", encoding="utf-8") + calls: list[tuple[list[str], dict[str, str]]] = [] + + monkeypatch.setattr( + module, + "_download_installer", + lambda _ref, _sha256, _artifact_dir: (installer, "https://example.invalid/installer.sh"), + ) + monkeypatch.setattr(module, "_installed_executable", lambda _prefix: tmp_path / "bin" / "l8k") + monkeypatch.setattr( + module, + "_verify_executable", + lambda _executable, _artifact_dir, _expected_version, _environment: ( + {"checks": {}, "artifacts": {}}, + True, + None, + ), + ) + + def fake_run(argv: list[str], *, cwd: Path, env: dict[str, str]) -> dict[str, Any]: + del cwd + calls.append((argv, env)) + return {"exit_code": 0, "stdout": "", "stderr": "", "duration_seconds": 0.1} + + monkeypatch.setattr(module, "_run_process", fake_run) + monkeypatch.setattr(module, "_record_process", lambda *_args, **_kwargs: {}) + args = argparse.Namespace( + mode="install", + executable="l8k", + version="v0.1.0", + installer_ref="a" * 40, + installer_sha256="0" * 64, + prefix=str(tmp_path), + environment_json=json.dumps({"HTTPS_PROXY": "http://proxy.example.test"}), + artifact_dir=str(tmp_path / "evidence"), + ) + + output, exit_code = module._prepare(args) + + assert exit_code == 0 + assert output["installed"] is True + assert calls[0][0] == ["/bin/sh", str(installer), "-d", str(tmp_path)] + assert calls[0][1]["L8K_VERSION"] == "v0.1.0" + assert calls[0][1]["HTTPS_PROXY"] == "http://proxy.example.test" + + +def test_provider_runs_the_real_launch_kit_workflow_shape(tmp_path: Path) -> None: + """The transport runs the full Launch Kit lifecycle with raw argv.""" + working_dir = tmp_path / "work" + artifact_dir = tmp_path / "evidence" + kubeconfig = "kubeconfig" + discover_args = [ + "--kubeconfig", + kubeconfig, + "--fabric", + "ethernet", + "--deployment-type", + "sriov", + ] + commands = [ + ("discover", discover_args), + ("generate", []), + ("deploy", ["--kubeconfig", kubeconfig]), + ("validate", ["--kubeconfig", kubeconfig]), + ("clean", ["--kubeconfig", kubeconfig]), + ] + outputs: dict[str, dict[str, Any]] = {} + + for command, arguments in commands: + completed, output = _run_workflow( + command, + arguments, + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + assert completed.returncode == 0 + assert output["success"] is True + assert output["operation"] == command + assert output["working_directory"] == str(working_dir.resolve()) + command_index = output["argv"].index(command) + assert output["argv"][command_index + 1 :] == [*arguments, "--output", "json"] + assert validate_output(output, "k8s_launch_kit") == (True, []) + assert all(Path(path).is_file() for path in output["artifacts"].values()) + outputs[command] = output + + assert len(outputs["discover"]["documents"]) == 1 + assert len(outputs["generate"]["documents"]) == 1 + generated_files = [Path(path) for path in outputs["generate"]["documents"][0]["generatedFiles"]] + daemonset_path = next(path for path in generated_files if "example-daemonset" in path.name) + daemonset = yaml.safe_load(daemonset_path.read_text(encoding="utf-8")) + assert [container["name"] for container in daemonset["spec"]["template"]["spec"]["containers"]] == [ + "test-container", + "netshoot", + ] + test_container = daemonset["spec"]["template"]["spec"]["containers"][0] + assert test_container["resources"]["requests"]["nvidia.com/gpu"] == "2" + assert test_container["resources"]["limits"]["nvidia.com/gpu"] == "2" + assert outputs["deploy"]["documents"] == [] + assert len(outputs["validate"]["documents"]) == 3 + families = {row["Family"] for row in outputs["validate"]["documents"][1]["connectivity"]["PingResults"]} + assert families == {"icmp", "rping", "ib_write_bw", "gpudirect_dmabuf"} + assert outputs["clean"]["documents"][0]["cleanup"] == { + "namespace": "nvidia-network-operator", + "customResourcesDeleted": 12, + "helmReleaseRemoved": True, + "keepHelmChart": False, + } + assert (working_dir / "cluster-config.yaml").is_file() + assert (working_dir / "deployment" / "k8s-launch-kit-validation-report.html").is_file() + + +def test_discover_stages_user_config_transiently_without_retaining_secrets(tmp_path: Path) -> None: + """Discovery uses a private staged config but retains only safe input provenance.""" + source = tmp_path / "customer-cluster-config.yaml" + secret_values = ("customer-api-token", "registry-password", "embedded-kubeconfig") + source_contents = """networkOperator: + selectedRelease: "26.4" +profile: + fabric: ethernet + deployment: sriov +clusterConfig: [] +credentials: + token: customer-api-token + registryPassword: registry-password + kubeconfig: embedded-kubeconfig +""" + source.write_text(source_contents, encoding="utf-8") + working_dir = tmp_path / "work" + artifact_dir = tmp_path / "evidence" + + completed, output = _run_workflow( + "discover", + ["--fabric", "ethernet", "--deployment-type", "sriov"], + working_dir=working_dir, + artifact_dir=artifact_dir, + user_config=source, + ) + + staged = working_dir / "user-config.yaml" + discovered = working_dir / "cluster-config.yaml" + assert completed.returncode == 0 + assert output["success"] is True + assert source.read_text(encoding="utf-8") == source_contents + assert not staged.exists() + assert discovered.is_file() + metadata_path = artifact_dir / "inputs" / "user-config.json" + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + assert metadata == { + "source_path": str(source.resolve()), + "staged_path": str(staged.resolve()), + "sha256": hashlib.sha256(source_contents.encode()).hexdigest(), + "size_bytes": len(source_contents.encode()), + "retained": False, + } + assert output["artifacts"]["user_config"] == str(metadata_path.resolve()) + assert output["argv"][-6:] == [ + "--user-config", + str(staged.resolve()), + "--save-cluster-config", + str(discovered.resolve()), + "--output", + "json", + ] + retained_text = "\n".join( + path.read_text(encoding="utf-8", errors="replace") + for root in (working_dir, artifact_dir) + for path in root.rglob("*") + if path.is_file() + ) + assert all(secret not in retained_text for secret in secret_values) + + +def test_user_config_must_not_be_inside_the_retained_working_directory(tmp_path: Path) -> None: + """A source inside the retained output tree is rejected before it can leak as evidence.""" + working_dir = tmp_path / "work" + working_dir.mkdir() + source = working_dir / "customer-cluster-config.yaml" + source.write_text("profile: {}\ncredentials: customer-api-token\n", encoding="utf-8") + + completed, output = _run_workflow( + "discover", + [], + working_dir=working_dir, + artifact_dir=tmp_path / "evidence", + user_config=source, + ) + + assert completed.returncode == 1 + assert output["success"] is False + assert "must be outside the retained provider working directory" in output["error"] + assert not (working_dir / "user-config.yaml").exists() + + +def test_staged_user_config_is_removed_when_discovery_fails(tmp_path: Path) -> None: + """A failed l8k discovery cannot leave the sensitive staged input behind.""" + source = tmp_path / "customer-cluster-config.yaml" + source_contents = "profile: {fabric: ethernet, deployment: sriov}\ncredentials: customer-api-token\n" + source.write_text(source_contents, encoding="utf-8") + working_dir = tmp_path / "work" + + completed, output = _run_workflow( + "discover", + [], + working_dir=working_dir, + artifact_dir=tmp_path / "evidence", + user_config=source, + env={**os.environ, "L8K_MOCK_FAIL": "discover"}, + ) + + assert completed.returncode != 0 + assert output["success"] is False + assert source.read_text(encoding="utf-8") == source_contents + assert not (working_dir / "user-config.yaml").exists() + + +@pytest.mark.parametrize("flag", ["--user-config", "--save-cluster-config"]) +def test_staged_user_config_rejects_conflicting_raw_discovery_paths(tmp_path: Path, flag: str) -> None: + """The first-class input owns both discovery config paths.""" + source = tmp_path / "customer-cluster-config.yaml" + source.write_text("profile: {}\n", encoding="utf-8") + + completed, output = _run_workflow( + "discover", + [flag, str(tmp_path / "raw.yaml")], + working_dir=tmp_path / "work", + artifact_dir=tmp_path / "evidence", + user_config=source, + ) + + assert completed.returncode == 1 + assert output["success"] is False + assert f"cannot be combined with raw discovery flag(s): {flag}" in output["error"] + + +def test_staged_user_config_must_exist(tmp_path: Path) -> None: + """A missing first-class user config fails before l8k starts.""" + working_dir = tmp_path / "work" + + completed, output = _run_workflow( + "discover", + [], + working_dir=working_dir, + artifact_dir=tmp_path / "evidence", + user_config=tmp_path / "missing.yaml", + ) + + assert completed.returncode == 1 + assert output["success"] is False + assert "Launch Kit user config not found" in output["error"] + assert not (working_dir / "user-config.yaml").exists() + + +def test_clean_forwards_launch_kit_boolean_flags_unchanged(tmp_path: Path) -> None: + """The transport accepts Launch Kit's native bare boolean flag syntax.""" + working_dir = tmp_path / "work" + artifact_dir = tmp_path / "evidence" + completed, _ = _run_workflow( + "discover", + ["--fabric", "ethernet", "--deployment-type", "sriov"], + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + assert completed.returncode == 0 + + completed, output = _run_workflow( + "clean", + ["--keep-helm-chart"], + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + + assert completed.returncode == 0 + assert output["argv"][-3:] == ["--keep-helm-chart", "--output", "json"] + assert output["documents"][0]["cleanup"] == { + "namespace": "nvidia-network-operator", + "customResourcesDeleted": 12, + "helmReleaseRemoved": False, + "keepHelmChart": True, + } + + +@pytest.mark.parametrize( + ("fabric", "deployment", "network_kind"), + [ + ("ethernet", "sriov", "SriovNetwork"), + ("infiniband", "sriov", "SriovIBNetwork"), + ("ethernet", "rdma_shared", "MacvlanNetwork"), + ("infiniband", "rdma_shared", "IPoIBNetwork"), + ("ethernet", "host_device", "HostDeviceNetwork"), + ("infiniband", "host_device", "HostDeviceNetwork"), + ], +) +def test_mock_supports_each_launch_kit_profile( + tmp_path: Path, + fabric: str, + deployment: str, + network_kind: str, +) -> None: + """Every pinned profile can traverse the same real command sequence.""" + working_dir = tmp_path / f"{fabric}-{deployment}" + artifact_dir = working_dir / "evidence" + commands = [ + ( + "discover", + [ + "--fabric", + fabric, + "--deployment-type", + deployment, + ], + ), + ("generate", []), + ("deploy", []), + ("validate", []), + ("clean", []), + ] + outputs: dict[str, dict[str, Any]] = {} + + for command, arguments in commands: + completed, output = _run_workflow( + command, + arguments, + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + assert completed.returncode == 0 + outputs[command] = output + + manifest_kinds = {row["Kind"] for row in outputs["validate"]["documents"][0]["manifests"]} + assert network_kind in manifest_kinds + assert outputs["clean"]["documents"][0]["phase"] == "clean" + + +def test_preflight_uses_the_workflow_kubeconfig(tmp_path: Path) -> None: + """kubectl probes target the same explicit kubeconfig supplied to l8k.""" + workflow = { + command: ["--kubeconfig", "partner.kubeconfig"] if command != "generate" else [] + for command in ("discover", "generate", "deploy", "validate", "clean") + } + completed, output = _run_provider( + "preflight", + "--kubectl-command-json", + json.dumps([sys.executable, str(_MOCK_KUBECTL)]), + "--workflow-arguments-json", + json.dumps(workflow), + "--working-dir", + str(tmp_path / "work"), + "--artifact-dir", + str(tmp_path / "evidence"), + ) + + assert completed.returncode == 0 + assert output["success"] is True + assert output["kubeconfig_source"] == "workflow arguments" + assert output["node_count"] == 2 + assert output["ready_node_count"] == 2 + command_file = Path(output["artifacts"]["api_version"]["command"]) + argv = json.loads(command_file.read_text(encoding="utf-8"))["argv"] + kubeconfig_index = argv.index("--kubeconfig") + assert argv[kubeconfig_index : kubeconfig_index + 2] == ["--kubeconfig", "partner.kubeconfig"] + + +def test_preflight_forwards_the_launch_kit_environment(tmp_path: Path) -> None: + """The safety probes use the same environment that the provider gives l8k.""" + workflow = {command: [] for command in ("discover", "generate", "deploy", "validate", "clean")} + completed, output = _run_provider( + "preflight", + "--kubectl-command-json", + json.dumps([sys.executable, str(_MOCK_KUBECTL)]), + "--workflow-arguments-json", + json.dumps(workflow), + "--environment-json", + json.dumps( + { + "KUBECONFIG": "environment.kubeconfig", + "L8K_MOCK_EXPECT_KUBECONFIG": "environment.kubeconfig", + } + ), + "--working-dir", + str(tmp_path / "work"), + "--artifact-dir", + str(tmp_path / "evidence"), + ) + + assert completed.returncode == 0 + assert output["success"] is True + + +def test_preflight_rejects_conflicting_workflow_kubeconfigs(tmp_path: Path) -> None: + """The safety gate fails closed when l8k commands would target different clusters.""" + workflow = { + "discover": ["--kubeconfig", "cluster-a"], + "generate": [], + "deploy": ["--kubeconfig=cluster-b"], + "validate": [], + "clean": [], + } + completed, output = _run_provider( + "preflight", + "--kubectl-command-json", + "[]", + "--workflow-arguments-json", + json.dumps(workflow), + "--working-dir", + str(tmp_path / "work"), + "--artifact-dir", + str(tmp_path / "evidence"), + ) + + assert completed.returncode == 1 + assert output["success"] is False + assert "different kubeconfigs" in output["error"] + + +def test_network_operator_provider_runs_end_to_end(tmp_path: Path, monkeypatch: Any) -> None: + """The production configuration executes all six named use cases in order.""" + monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") + config = _mocked_network_operator_config(tmp_path) + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( + phases=[Phase.SETUP, Phase.TEST], + capability="kubernetes", + ) + + assert result.success is True + expected_use_cases = [ + "roce_sriov", + "infiniband_sriov", + "roce_rdma_shared", + "infiniband_rdma_shared", + "roce_host_device", + "infiniband_host_device", + ] + expected_steps = ["launch_kit_prepare", "launch_kit_verify"] + for use_case in expected_use_cases: + expected_steps.extend( + f"launch_kit_{use_case}_{operation}" + for operation in ("preflight", "discover", "generate", "deploy", "validate", "clean") + ) + assert list(result.inventory) == expected_steps + expected_phase_names = ["setup", "launch-kit-verification"] + for use_case in expected_use_cases: + phase_name = use_case.replace("_", "-") + expected_phase_names.extend([phase_name, f"{phase_name}-teardown"]) + assert [phase.name for phase in result.phases] == expected_phase_names + for use_case in expected_use_cases: + phase_name = use_case.replace("_", "-") + test_phase = next(phase for phase in result.phases if phase.name == phase_name) + teardown_phase = next(phase for phase in result.phases if phase.name == f"{phase_name}-teardown") + assert test_phase.phase is Phase.TEST + assert all(not step["name"].endswith("_clean") for step in test_phase.details["steps"]) + assert teardown_phase.phase is Phase.TEARDOWN + assert [step["name"] for step in teardown_phase.details["steps"]] == [f"launch_kit_{use_case}_clean"] + states = {entry.entry.name: entry.state for entry in result.validations} + assert states == { + "EastWestNetworkRoceSriovCheck": State.PASSED, + "EastWestNetworkInfiniBandSriovCheck": State.PASSED, + "EastWestNetworkRoceRdmaSharedCheck": State.PASSED, + "EastWestNetworkInfiniBandRdmaSharedCheck": State.PASSED, + "EastWestNetworkRoceHostDeviceCheck": State.PASSED, + "EastWestNetworkInfiniBandHostDeviceCheck": State.PASSED, + } + expected_subtest_counts = { + "EastWestNetworkRoceSriovCheck": 122, + "EastWestNetworkInfiniBandSriovCheck": 122, + "EastWestNetworkRoceRdmaSharedCheck": 117, + "EastWestNetworkInfiniBandRdmaSharedCheck": 117, + "EastWestNetworkRoceHostDeviceCheck": 117, + "EastWestNetworkInfiniBandHostDeviceCheck": 117, + } + for entry in result.validations: + assert entry.subtest_summary.passed == expected_subtest_counts[entry.entry.name] + assert entry.subtest_summary.failed == 0 + assert entry.subtest_summary.skipped == 0 + for use_case in expected_use_cases: + assert (tmp_path / "use-cases" / use_case / "work" / "cluster-config.yaml").is_file() + + +@pytest.mark.parametrize( + ("label", "selected_use_cases", "excluded_use_cases"), + [ + ( + "ethernet", + ["roce_sriov", "roce_rdma_shared", "roce_host_device"], + ["infiniband_sriov", "infiniband_rdma_shared", "infiniband_host_device"], + ), + ( + "infiniband", + ["infiniband_sriov", "infiniband_rdma_shared", "infiniband_host_device"], + ["roce_sriov", "roce_rdma_shared", "roce_host_device"], + ), + ( + "sriov", + ["roce_sriov", "infiniband_sriov"], + ["roce_rdma_shared", "infiniband_rdma_shared", "roce_host_device", "infiniband_host_device"], + ), + ( + "rdma_shared", + ["roce_rdma_shared", "infiniband_rdma_shared"], + ["roce_sriov", "infiniband_sriov", "roce_host_device", "infiniband_host_device"], + ), + ( + "host_device", + ["roce_host_device", "infiniband_host_device"], + ["roce_sriov", "infiniband_sriov", "roce_rdma_shared", "infiniband_rdma_shared"], + ), + ( + "gpudirect", + [ + "roce_sriov", + "infiniband_sriov", + "roce_rdma_shared", + "infiniband_rdma_shared", + "roce_host_device", + "infiniband_host_device", + ], + [], + ), + ( + ["ethernet", "sriov"], + ["roce_sriov"], + [ + "infiniband_sriov", + "roce_rdma_shared", + "infiniband_rdma_shared", + "roce_host_device", + "infiniband_host_device", + ], + ), + ], +) +def test_network_operator_provider_grouping_label_prunes_unselected_workflows( + tmp_path: Path, + monkeypatch: Any, + label: str | list[str], + selected_use_cases: list[str], + excluded_use_cases: list[str], +) -> None: + """A grouping label runs only the matching mutating workflows.""" + monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") + config = _mocked_network_operator_config(tmp_path) + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( + phases=[Phase.SETUP, Phase.TEST], + include_labels=[label] if isinstance(label, str) else label, + capability="kubernetes", + ) + + assert result.success is True + inventory_names = set(result.inventory) + for use_case in selected_use_cases: + assert f"launch_kit_{use_case}_validate" in inventory_names + assert f"launch_kit_{use_case}_clean" in inventory_names + assert (tmp_path / "use-cases" / use_case / "work" / "cluster-config.yaml").is_file() + for use_case in excluded_use_cases: + assert not any(name.startswith(f"launch_kit_{use_case}_") for name in inventory_names) + assert not (tmp_path / "use-cases" / use_case / "work" / "cluster-config.yaml").exists() + + +def test_network_operator_stages_user_config_only_for_selected_use_cases( + tmp_path: Path, + monkeypatch: Any, +) -> None: + """Each selected use case receives and removes an isolated user-config copy.""" + monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") + source = tmp_path / "customer-cluster-config.yaml" + source_contents = """networkOperator: + selectedRelease: "26.4" +profile: + fabric: ethernet + deployment: sriov +clusterConfig: [] +""" + source.write_text(source_contents, encoding="utf-8") + config = _mocked_network_operator_config(tmp_path) + config.context["k8s_launch_kit"]["user_config"] = str(source) + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( + phases=[Phase.TEST], + include_labels=["ethernet", "sriov"], + capability="kubernetes", + ) + + selected_work = tmp_path / "use-cases" / "roce_sriov" / "work" + selected_evidence = tmp_path / "use-cases" / "roce_sriov" / "evidence" + assert result.success is True + assert source.read_text(encoding="utf-8") == source_contents + assert not (selected_work / "user-config.yaml").exists() + assert (selected_work / "cluster-config.yaml").is_file() + metadata = json.loads((selected_evidence / "inputs" / "user-config.json").read_text(encoding="utf-8")) + assert metadata["sha256"] == hashlib.sha256(source_contents.encode()).hexdigest() + assert metadata["retained"] is False + discover = result.inventory["launch_kit_roce_sriov_discover"] + assert discover["argv"][discover["argv"].index("--user-config") + 1] == str( + (selected_work / "user-config.yaml").resolve() + ) + for use_case in ( + "infiniband_sriov", + "roce_rdma_shared", + "infiniband_rdma_shared", + "roce_host_device", + "infiniband_host_device", + ): + assert not (tmp_path / "use-cases" / use_case / "work" / "user-config.yaml").exists() + + +def test_network_operator_provider_test_phase_verifies_without_setup(tmp_path: Path, monkeypatch: Any) -> None: + """A test-only run verifies the configured binary instead of requiring setup output.""" + monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") + config = _mocked_network_operator_config(tmp_path) + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( + phases=[Phase.TEST], + capability="kubernetes", + ) + + assert result.success is True + assert "launch_kit_prepare" not in result.inventory + assert result.inventory["launch_kit_verify"]["success"] is True + assert all(entry.state is State.PASSED for entry in result.validations) + + +def test_network_operator_teardown_only_runs_selected_cleanup(tmp_path: Path, monkeypatch: Any) -> None: + """Explicit teardown is a standalone recovery path and needs no prior step output.""" + monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") + config = _mocked_network_operator_config(tmp_path) + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( + phases=[Phase.TEARDOWN], + include_labels=["ethernet", "sriov"], + capability="kubernetes", + ) + + assert result.success is True + assert list(result.inventory) == ["launch_kit_roce_sriov_clean"] + teardown_phases = [phase for phase in result.phases if phase.phase is Phase.TEARDOWN] + assert [(phase.name, phase.phase) for phase in teardown_phases] == [("teardown", Phase.TEARDOWN)] + cleanup = result.inventory["launch_kit_roce_sriov_clean"]["documents"][0]["cleanup"] + assert cleanup["namespace"] == "nvidia-network-operator" + assert cleanup["helmReleaseRemoved"] is True + + +def test_network_operator_teardown_only_attempts_every_cleanup_best_effort( + tmp_path: Path, + monkeypatch: Any, +) -> None: + """Recovery attempts every selected cleanup even when an earlier one fails.""" + monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") + monkeypatch.setenv("L8K_MOCK_FAIL", "clean") + config = _mocked_network_operator_config(tmp_path) + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( + phases=[Phase.TEARDOWN], + capability="kubernetes", + ) + + assert result.success is False + clean_steps = [name for name in result.inventory if name.endswith("_clean")] + assert clean_steps == [ + "launch_kit_roce_sriov_clean", + "launch_kit_infiniband_sriov_clean", + "launch_kit_roce_rdma_shared_clean", + "launch_kit_infiniband_rdma_shared_clean", + "launch_kit_roce_host_device_clean", + "launch_kit_infiniband_host_device_clean", + ] + teardown_phase = next(phase for phase in result.phases if phase.name == "teardown") + assert teardown_phase.phase is Phase.TEARDOWN + assert teardown_phase.success is False + + +def test_kubernetes_preflight_failure_stops_before_discovery(tmp_path: Path, monkeypatch: Any) -> None: + """An unreachable cluster blocks each use case before discovery without hiding later cases.""" + monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") + monkeypatch.setenv("L8K_MOCK_KUBERNETES_FAIL", "1") + config = _mocked_network_operator_config(tmp_path) + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run(capability="kubernetes") + + assert result.success is False + assert list(result.inventory) == [ + "launch_kit_prepare", + "launch_kit_verify", + "launch_kit_roce_sriov_preflight", + "launch_kit_infiniband_sriov_preflight", + "launch_kit_roce_rdma_shared_preflight", + "launch_kit_infiniband_rdma_shared_preflight", + "launch_kit_roce_host_device_preflight", + "launch_kit_infiniband_host_device_preflight", + ] + assert all(entry.state is State.ERROR for entry in result.validations) + assert all(entry.error_reason is ErrorReason.STEP_FAILED for entry in result.validations) + assert all("preflight" in entry.message for entry in result.validations) + assert not list((tmp_path / "use-cases").glob("*/work/cluster-config.yaml")) + assert not list((tmp_path / "use-cases").glob("*/evidence/commands/discover")) + skipped_teardowns = [phase for phase in result.phases if phase.phase is Phase.TEARDOWN and phase.name != "teardown"] + assert len(skipped_teardowns) == 6 + assert all(phase.message.startswith("SKIPPED: target step(s) were not attempted") for phase in skipped_teardowns) + + +def test_failed_deploy_still_runs_launch_kit_cleanup(tmp_path: Path, monkeypatch: Any) -> None: + """A failed deployment is a use-case error and still activates cleanup.""" + monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") + monkeypatch.setenv("L8K_MOCK_FAIL", "deploy") + config = _mocked_network_operator_config(tmp_path) + junit_path = tmp_path / "junit.xml" + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( + phases=[Phase.TEST], + include_labels=["ethernet", "sriov"], + capability="kubernetes", + junitxml=str(junit_path), + ) + + assert result.success is False + assert list(result.inventory) == [ + "launch_kit_verify", + "launch_kit_roce_sriov_preflight", + "launch_kit_roce_sriov_discover", + "launch_kit_roce_sriov_generate", + "launch_kit_roce_sriov_deploy", + "launch_kit_roce_sriov_clean", + ] + assert result.inventory["launch_kit_roce_sriov_clean"]["documents"][0]["cleanup"]["helmReleaseRemoved"] is True + teardown_phase = next(phase for phase in result.phases if phase.name == "roce-sriov-teardown") + assert teardown_phase.phase is Phase.TEARDOWN + assert teardown_phase.success is True + + validation = next(entry for entry in result.validations if entry.entry.name == "EastWestNetworkRoceSriovCheck") + assert validation.state is State.ERROR + assert validation.error_reason is ErrorReason.STEP_FAILED + assert "launch_kit_roce_sriov_deploy" in validation.message + assert "deployment failed" in validation.message + + case = next( + case + for case in ET.parse(junit_path).getroot().iter("testcase") + if case.get("name") == "EastWestNetworkRoceSriovCheck" + ) + error = case.find("error") + assert error is not None + assert error.get("type") == ErrorReason.STEP_FAILED.value + assert "launch_kit_roce_sriov_deploy" in (error.get("message") or "") + assert case.find("skipped") is None + + +def test_failed_validate_still_runs_launch_kit_cleanup(tmp_path: Path, monkeypatch: Any) -> None: + """A failed validation command cannot bypass cleanup of its deployment.""" + monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") + monkeypatch.setenv("L8K_MOCK_FAIL", "validate:ib_write_bw") + config = _mocked_network_operator_config(tmp_path) + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( + phases=[Phase.TEST], + include_labels=["ethernet", "sriov"], + capability="kubernetes", + ) + + assert result.success is False + assert list(result.inventory)[-2:] == [ + "launch_kit_roce_sriov_validate", + "launch_kit_roce_sriov_clean", + ] + assert result.inventory["launch_kit_roce_sriov_clean"]["success"] is True + assert result.validations[0].state is State.FAILED + teardown_phase = next(phase for phase in result.phases if phase.name == "roce-sriov-teardown") + assert teardown_phase.phase is Phase.TEARDOWN + assert teardown_phase.success is True + + +def test_failed_cleanup_blocks_the_next_use_case(tmp_path: Path, monkeypatch: Any) -> None: + """Continuation is unsafe when the preceding use case could not be cleaned.""" + monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") + monkeypatch.setenv("L8K_MOCK_FAIL", "clean") + config = _mocked_network_operator_config(tmp_path) + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( + phases=[Phase.TEST], + include_labels=["sriov"], + capability="kubernetes", + ) + + assert result.success is False + assert "launch_kit_roce_sriov_clean" in result.inventory + assert not any(name.startswith("launch_kit_infiniband_sriov_") for name in result.inventory) + teardown_phase = next(phase for phase in result.phases if phase.name == "roce-sriov-teardown") + assert teardown_phase.phase is Phase.TEARDOWN + assert teardown_phase.success is False + + +def test_failed_validate_preserves_documents_and_process_error(tmp_path: Path) -> None: + """A non-zero l8k result retains every JSON document and a clear exit diagnostic.""" + working_dir = tmp_path / "work" + artifact_dir = tmp_path / "evidence" + _run_workflow( + "discover", + [ + "--fabric", + "ethernet", + "--deployment-type", + "sriov", + ], + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + _run_workflow( + "generate", + [], + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + env = os.environ.copy() + env["L8K_MOCK_FAIL"] = "validate:ib_write_bw" + + completed, output = _run_workflow( + "validate", + [], + working_dir=working_dir, + artifact_dir=artifact_dir, + env=env, + ) + + assert completed.returncode == 4 + assert output["success"] is False + assert len(output["documents"]) == 3 + assert "l8k validate exited with code 4" in output["error"] + assert Path(output["artifacts"]["stdout"]).read_text(encoding="utf-8") diff --git a/isvctl/tests/providers/k8s_launch_kit/test_timeout_config.py b/isvctl/tests/providers/k8s_launch_kit/test_timeout_config.py new file mode 100644 index 000000000..92123a892 --- /dev/null +++ b/isvctl/tests/providers/k8s_launch_kit/test_timeout_config.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Regression tests for Launch Kit timeout ownership.""" + +from pathlib import Path +from typing import Any + +import yaml + +PROVIDER_CONFIG_DIR = Path(__file__).parents[3] / "configs" / "providers" / "k8s-launch-kit" / "config" + + +def _steps(config_name: str) -> list[dict[str, Any]]: + """Return Network Operator command steps from a provider configuration.""" + config = yaml.safe_load((PROVIDER_CONFIG_DIR / config_name).read_text()) + return config["commands"]["network_operator"]["steps"] + + +def test_generic_validate_delegates_timeout_to_launch_kit() -> None: + """The generic validate workflow must not preempt l8k's matrix budget.""" + validate_steps = [step for step in _steps("provider.yaml") if step["name"] == "launch_kit_validate"] + + assert len(validate_steps) == 1 + assert validate_steps[0]["timeout"] is None + + +def test_network_operator_validates_delegate_timeout_to_launch_kit() -> None: + """Every grouped use case must leave its validation deadline to l8k.""" + validate_steps = [step for step in _steps("network-operator.yaml") if step["name"].endswith("_validate")] + + assert len(validate_steps) == 6 + assert all(step["timeout"] is None for step in validate_steps) diff --git a/isvctl/tests/test_orchestrator_loop.py b/isvctl/tests/test_orchestrator_loop.py index c0da2cb2c..0fc9dc414 100644 --- a/isvctl/tests/test_orchestrator_loop.py +++ b/isvctl/tests/test_orchestrator_loop.py @@ -35,6 +35,7 @@ Orchestrator, Phase, _apply_capability_step_gates, + _apply_selected_validation_gates, _entries_missing_from_junit, _merge_junit_xmls, _write_terminal_junit_xml, @@ -79,6 +80,60 @@ def test_explicit_step_requires_gate_unbound_lifecycle_steps() -> None: assert all(not step.skip for step in kubernetes_steps) +def test_selected_validation_gate_prunes_unselected_lifecycle_steps() -> None: + """Label selection prevents commands owned by another test group from running.""" + steps = [ + StepConfig( + name="run_ethernet", + command="ethernet", + phase="test", + requires_selected_validations=["EthernetCheck"], + ), + StepConfig( + name="run_infiniband", + command="infiniband", + phase="test", + requires_selected_validations=["InfiniBandCheck"], + ), + ] + entries = [ + ValidationEntry( + name="EthernetCheck", + category="network", + params_template={}, + labels=("ethernet",), + ), + ValidationEntry( + name="InfiniBandCheck", + category="network", + params_template={}, + labels=("infiniband",), + ), + ] + + all_steps = _apply_selected_validation_gates( + steps, + entries, + include_labels=set(), + exclude_labels=set(), + exclude_tests=set(), + released_tests=None, + capability=None, + ) + ethernet_steps = _apply_selected_validation_gates( + steps, + entries, + include_labels={"ethernet"}, + exclude_labels=set(), + exclude_tests=set(), + released_tests=None, + capability=None, + ) + + assert all(not step.skip for step in all_steps) + assert [step.skip for step in ethernet_steps] == [False, True] + + def test_python_script_path_falls_back_to_current_working_directory( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -403,6 +458,268 @@ def test_run_all_phases_with_failure(self) -> None: assert len(teardown_phases) == 1 assert teardown_phases[0].success + def test_independent_custom_phase_runs_after_an_allowed_failure(self) -> None: + """An opted-in failed use case does not hide results from later independent cases.""" + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two"], + continue_after_failure=["case-one"], + steps=[ + StepConfig(name="case_one", command="false", phase="case-one"), + StepConfig( + name="case_two", + command="echo", + args=['{"success": true, "platform": "kubernetes"}'], + phase="case-two", + output_schema="generic", + ), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is False + assert [(phase.name, phase.success) for phase in result.phases] == [ + ("case-one", False), + ("case-two", True), + ] + + def test_phase_finalizer_runs_after_its_target_fails(self, tmp_path: Path) -> None: + """An attempted mutating step activates cleanup even when the step fails.""" + marker = tmp_path / "cleaned" + cleanup = _write_script( + tmp_path, + "cleanup.sh", + f"#!/bin/sh\ntouch {marker}\necho '{{\"success\": true}}'\n", + ) + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two"], + continue_after_failure=["case-one"], + steps=[ + StepConfig(name="deploy", command="false", phase="case-one"), + StepConfig( + name="cleanup", + command=cleanup, + phase="case-one", + finalizer_for="deploy", + ), + StepConfig(name="case_two", command="true", phase="case-two"), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is False + assert marker.is_file() + assert [step["name"] for step in result.phases[0].details["steps"]] == ["deploy"] + assert [step["name"] for step in result.phases[1].details["steps"]] == ["cleanup"] + assert result.phases[1].phase is Phase.TEARDOWN + assert result.phases[1].name == "case-one-teardown" + assert result.phases[2].name == "case-two" + assert result.phases[2].success is True + + def test_phase_finalizer_skips_when_target_was_not_attempted(self, tmp_path: Path) -> None: + """A prerequisite failure cannot activate destructive cleanup before deployment.""" + marker = tmp_path / "cleaned" + cleanup = _write_script(tmp_path, "cleanup.sh", f"#!/bin/sh\ntouch {marker}\n") + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two"], + continue_after_failure=["case-one"], + steps=[ + StepConfig(name="preflight", command="false", phase="case-one"), + StepConfig(name="deploy", command="true", phase="case-one"), + StepConfig( + name="cleanup", + command=cleanup, + phase="case-one", + finalizer_for="deploy", + ), + StepConfig(name="case_two", command="true", phase="case-two"), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is False + assert not marker.exists() + assert [step["name"] for step in result.phases[0].details["steps"]] == ["preflight"] + assert result.phases[1].name == "case-one-teardown" + assert result.phases[1].message.startswith("SKIPPED: target step(s) were not attempted") + assert result.phases[2].success is True + + def test_phase_finalizer_skips_when_target_process_never_started(self, tmp_path: Path) -> None: + """A command-resolution failure cannot imply that a cluster mutation occurred.""" + marker = tmp_path / "cleaned" + cleanup = _write_script(tmp_path, "cleanup.sh", f"#!/bin/sh\ntouch {marker}\n") + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two"], + continue_after_failure=["case-one"], + steps=[ + StepConfig( + name="deploy", + command=str(tmp_path / "does-not-exist"), + phase="case-one", + ), + StepConfig( + name="cleanup", + command=cleanup, + phase="case-one", + finalizer_for="deploy", + ), + StepConfig(name="case_two", command="true", phase="case-two"), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is False + assert not marker.exists() + assert result.phases[0].details["steps"][0]["attempted"] is False + assert result.phases[1].message.startswith("SKIPPED: target step(s) were not attempted") + assert result.phases[2].success is True + + def test_failed_phase_finalizer_blocks_later_independent_phases(self) -> None: + """A failed cleanup leaves unsafe state and overrides continuation policy.""" + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two"], + continue_after_failure=["case-one"], + steps=[ + StepConfig(name="deploy", command="true", phase="case-one"), + StepConfig( + name="cleanup", + command="false", + phase="case-one", + finalizer_for="deploy", + ), + StepConfig(name="case_two", command="true", phase="case-two"), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is False + assert [(phase.name, phase.message) for phase in result.phases] == [ + ("case-one", "deploy: passed"), + ("case-one-teardown", "cleanup: failed"), + ("case-two", "SKIPPED: previous phase failed"), + ] + + def test_teardown_finalizer_runs_between_test_phases(self, tmp_path: Path) -> None: + """A step declared in teardown executes directly after its target test phase.""" + marker = tmp_path / "cleaned" + cleanup = _write_script(tmp_path, "cleanup.sh", f"#!/bin/sh\ntouch {marker}\n") + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two", "teardown"], + steps=[ + StepConfig(name="deploy", command="true", phase="case-one"), + StepConfig(name="case_two", command="true", phase="case-two"), + StepConfig( + name="cleanup", + command=cleanup, + phase="teardown", + finalizer_for="deploy", + ), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is True + assert marker.is_file() + assert [(phase.name, phase.phase) for phase in result.phases] == [ + ("case-one", Phase.TEST), + ("case-one-teardown", Phase.TEARDOWN), + ("case-two", Phase.TEST), + ] + + def test_teardown_only_runs_linked_finalizer_as_recovery(self, tmp_path: Path) -> None: + """An explicit teardown-only run does not require an in-memory target attempt.""" + marker = tmp_path / "cleaned" + cleanup = _write_script(tmp_path, "cleanup.sh", f"#!/bin/sh\ntouch {marker}\n") + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["test", "teardown"], + steps=[ + StepConfig(name="deploy", command="true", phase="test"), + StepConfig( + name="cleanup", + command=cleanup, + phase="teardown", + finalizer_for="deploy", + ), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEARDOWN]) + + assert result.success is True + assert marker.is_file() + assert [(phase.name, phase.phase) for phase in result.phases] == [ + ("teardown", Phase.TEARDOWN), + ] + + def test_custom_phase_failure_blocks_later_phases_by_default(self) -> None: + """Without an opt-in, the existing stop-on-failure behavior is unchanged.""" + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two"], + steps=[ + StepConfig(name="case_one", command="false", phase="case-one"), + StepConfig( + name="case_two", + command="echo", + args=['{"success": true, "platform": "kubernetes"}'], + phase="case-two", + output_schema="generic", + ), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is False + assert [(phase.name, phase.message) for phase in result.phases] == [ + ("case-one", "case_one: failed"), + ("case-two", "SKIPPED: previous phase failed"), + ] + def test_platform_detection_missing(self) -> None: """Test error when platform cannot be detected. @@ -448,6 +765,29 @@ def test_config_without_commands_runs_live_validations_only(self, monkeypatch: p assert [entry.entry.name for entry in result.validations] == ["K8sCsiStorageTypesCheck"] assert result.validations[0].state is State.PASSED + def test_config_without_commands_reports_failed_live_validation(self, monkeypatch: pytest.MonkeyPatch) -> None: + """A failed commandless validation returns a failed result instead of reading command policy.""" + monkeypatch.setattr("isvctl.orchestrator.loop.load_released_test_filter", lambda: None) + config = RunConfig( + tests=ValidationConfig( + validations={ + "live_checks": { + "checks": { + "ExistingSystemFieldCheck": { + "compose": [{"FieldExistsCheck": {"field": "missing"}}], + } + }, + } + }, + ), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST], capability="kubernetes") + + assert result.success is False + assert result.validations[0].state is State.FAILED + assert "Missing fields: missing" in result.validations[0].message + def test_config_without_commands_or_validations_is_not_a_pass(self) -> None: """Validations are all a commandless run has, so wiring none asserts nothing.""" orchestrator = Orchestrator(RunConfig(tests=ValidationConfig(validations={}))) @@ -600,9 +940,80 @@ def test_validation_without_step_output_is_reported_as_skipped(self, monkeypatch "state": "skipped", "skip_reason": "step_no_output", "error_reason": None, + "subtest_summary": {"total": 0, "passed": 0, "failed": 0, "skipped": 0}, } ] + def test_failed_owned_step_is_reported_as_validation_error( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """An early workflow failure cannot become a harmless missing-output skip.""" + monkeypatch.setattr("isvctl.orchestrator.loop.load_released_test_filter", lambda: None) + failing_step = _write_script( + tmp_path, + "deploy.sh", + "#!/bin/sh\necho 'driver image not found' >&2\nexit 4\n", + ) + junit_path = tmp_path / "junit.xml" + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["use-case"], + steps=[ + StepConfig( + name="deploy_fixture", + command=failing_step, + phase="use-case", + requires_selected_validations=["ProbeSucceededCheck"], + ), + StepConfig( + name="validate_fixture", + command="true", + phase="use-case", + requires_selected_validations=["ProbeSucceededCheck"], + ), + ], + ) + }, + tests=ValidationConfig( + capability="kubernetes", + validations={ + "probe_checks": { + "step": "validate_fixture", + "checks": {"ProbeSucceededCheck": {"compose": ["StepSuccessCheck"]}}, + }, + }, + ), + ) + + result = Orchestrator(config).run( + phases=[Phase.TEST], + capability="kubernetes", + junitxml=str(junit_path), + ) + + assert result.success is False + validation = result.validations[0] + assert validation.state is State.ERROR + assert validation.error_reason is ErrorReason.STEP_FAILED + assert validation.message == ( + "workflow step 'deploy_fixture' failed: Command exited with code 4: driver image not found" + ) + + suite = ET.parse(junit_path).getroot().find("testsuite") + assert suite is not None + assert suite.get("errors") == "1" + assert suite.get("skipped") == "0" + case = suite.find("testcase") + assert case is not None + assert case.get("name") == "ProbeSucceededCheck" + error = case.find("error") + assert error is not None + assert error.get("type") == ErrorReason.STEP_FAILED.value + assert case.find("skipped") is None + def test_validation_template_error_is_reported_as_error( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/isvctl/tests/test_orchestrator_process.py b/isvctl/tests/test_orchestrator_process.py new file mode 100644 index 000000000..803d646e7 --- /dev/null +++ b/isvctl/tests/test_orchestrator_process.py @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for orchestration subprocess lifecycle handling.""" + +import os +import signal +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from isvctl.orchestrator.process import run_command_process + + +def test_run_command_process_captures_output(tmp_path: Path) -> None: + """Successful commands return captured text output.""" + completed = run_command_process( + [sys.executable, "-c", "print('ready')"], + cwd=tmp_path, + env=None, + timeout=5, + ) + + assert completed.returncode == 0 + assert completed.stdout == "ready\n" + assert completed.stderr == "" + + +def test_run_command_process_accepts_no_timeout(tmp_path: Path) -> None: + """A null step timeout waits for a command that owns its deadline.""" + completed = run_command_process( + [sys.executable, "-c", "print('tool-owned-timeout')"], + cwd=tmp_path, + env=None, + timeout=None, + ) + + assert completed.returncode == 0 + assert completed.stdout == "tool-owned-timeout\n" + assert completed.stderr == "" + + +@pytest.mark.skipif(os.name != "posix", reason="process-group behavior is POSIX-specific") +def test_timeout_terminates_descendant_process(tmp_path: Path) -> None: + """A timed-out wrapper must not leave its provider CLI child running.""" + child_pid_path = tmp_path / "child.pid" + wrapper = """ +import subprocess +import sys +import time +from pathlib import Path + +child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) +Path(sys.argv[1]).write_text(str(child.pid)) +print("wrapper-ready", flush=True) +time.sleep(60) +""" + child_pid: int | None = None + + try: + with pytest.raises(subprocess.TimeoutExpired) as exc_info: + run_command_process( + [sys.executable, "-c", wrapper, str(child_pid_path)], + cwd=tmp_path, + env=None, + timeout=0.5, + ) + + assert "wrapper-ready" in (exc_info.value.stdout or "") + child_pid = int(child_pid_path.read_text()) + + deadline = time.monotonic() + 2 + while _process_exists(child_pid) and time.monotonic() < deadline: + time.sleep(0.05) + assert not _process_exists(child_pid) + finally: + if child_pid is not None and _process_exists(child_pid): + os.kill(child_pid, signal.SIGKILL) + + +def _process_exists(pid: int) -> bool: + """Return whether a process currently exists.""" + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + return True diff --git a/isvctl/tests/test_schema.py b/isvctl/tests/test_schema.py index 371d57f1f..ad16a27dc 100644 --- a/isvctl/tests/test_schema.py +++ b/isvctl/tests/test_schema.py @@ -77,6 +77,7 @@ def test_minimal_step(self) -> None: assert step.skip is False assert step.requires == [] assert step.requires_available_validations == [] + assert step.requires_selected_validations == [] def test_full_step(self) -> None: """Test creating a fully specified step config.""" @@ -91,6 +92,7 @@ def test_full_step(self) -> None: skip=False, requires=["vm", "bare_metal"], requires_available_validations=["NewCheck"], + requires_selected_validations=["SelectedCheck"], continue_on_failure=True, output_schema="vpc", ) @@ -102,9 +104,16 @@ def test_full_step(self) -> None: assert step.phase == "setup" assert step.requires == ["vm", "bare_metal"] assert step.requires_available_validations == ["NewCheck"] + assert step.requires_selected_validations == ["SelectedCheck"] assert step.continue_on_failure is True assert step.output_schema == "vpc" + def test_null_timeout_disables_watchdog(self) -> None: + """A provider may delegate timeout ownership to the invoked tool.""" + step = StepConfig(name="validate", command="l8k", timeout=None) + + assert step.timeout is None + def test_step_rejects_unknown_or_duplicate_requires(self) -> None: """Step requirements use the declarable capability vocabulary.""" with pytest.raises(ValidationError, match="requires must be a list containing only"): @@ -113,6 +122,100 @@ def test_step_rejects_unknown_or_duplicate_requires(self) -> None: StepConfig(name="setup", command="echo", requires=["vm", "vm"]) +class TestPlatformCommands: + """Tests for ordered command phases.""" + + def test_phases_reject_duplicates(self) -> None: + """A duplicate phase would otherwise execute its steps more than once.""" + with pytest.raises(ValidationError, match="phases must not contain duplicate"): + PlatformCommands(phases=["setup", "test", "test"]) + + def test_continue_after_failure_rejects_unknown_phase(self) -> None: + """A typo must not silently restore stop-on-failure behavior.""" + with pytest.raises(ValidationError, match="phases not listed in phases"): + PlatformCommands( + phases=["setup", "use-case"], + continue_after_failure=["use-csae"], + ) + + @pytest.mark.parametrize("phase", ["setup", "teardown"]) + def test_continue_after_failure_rejects_lifecycle_phases(self, phase: str) -> None: + """Setup and teardown are never independent test-case phases.""" + with pytest.raises(ValidationError, match="cannot contain lifecycle phases"): + PlatformCommands( + phases=["setup", "use-case", "teardown"], + continue_after_failure=[phase], + ) + + def test_continue_after_failure_rejects_duplicates(self) -> None: + """Duplicate continuation entries are configuration errors, not useful policy.""" + with pytest.raises(ValidationError, match="must not contain duplicate"): + PlatformCommands( + phases=["setup", "use-case"], + continue_after_failure=["use-case", "use-case"], + ) + + def test_finalizer_requires_one_target_in_its_phase_or_teardown(self) -> None: + """A finalizer target must resolve unambiguously in a supported lifecycle position.""" + with pytest.raises(ValidationError, match="must name exactly one configured step"): + PlatformCommands( + phases=["test"], + steps=[ + StepConfig(name="cleanup", command="clean", phase="test", finalizer_for="deploy"), + ], + ) + + with pytest.raises(ValidationError, match="same phase or the finalizer must use phase 'teardown'"): + PlatformCommands( + phases=["setup", "test", "cleanup"], + steps=[ + StepConfig(name="deploy", command="deploy", phase="test"), + StepConfig(name="cleanup", command="clean", phase="cleanup", finalizer_for="deploy"), + ], + ) + + config = PlatformCommands( + phases=["test", "teardown"], + steps=[ + StepConfig(name="deploy", command="deploy", phase="test"), + StepConfig(name="cleanup", command="clean", phase="teardown", finalizer_for="deploy"), + ], + ) + assert config.steps[1].finalizer_for == "deploy" + + def test_teardown_finalizer_must_follow_target_and_match_its_gates(self) -> None: + """Linked teardown cannot precede its mutation or be filtered independently.""" + with pytest.raises(ValidationError, match="teardown must be ordered after target"): + PlatformCommands( + phases=["teardown", "test"], + steps=[ + StepConfig(name="deploy", command="deploy", phase="test"), + StepConfig(name="cleanup", command="clean", phase="teardown", finalizer_for="deploy"), + ], + ) + + with pytest.raises(ValidationError, match="must use the same gates as target"): + PlatformCommands( + phases=["test", "teardown"], + steps=[ + StepConfig(name="deploy", command="deploy", phase="test", requires=["kubernetes"]), + StepConfig(name="cleanup", command="clean", phase="teardown", finalizer_for="deploy"), + ], + ) + + def test_finalizer_cannot_target_another_finalizer(self) -> None: + """Finalizer chains have ambiguous activation and cleanup ordering.""" + with pytest.raises(ValidationError, match="cannot finalize finalizer step"): + PlatformCommands( + phases=["test"], + steps=[ + StepConfig(name="deploy", command="deploy", phase="test"), + StepConfig(name="cleanup", command="clean", phase="test", finalizer_for="deploy"), + StepConfig(name="verify_cleanup", command="verify", phase="test", finalizer_for="cleanup"), + ], + ) + + class TestCommandOutput: """Tests for CommandOutput model (setup command JSON output).""" diff --git a/isvctl/tests/test_stub_contracts.py b/isvctl/tests/test_stub_contracts.py index 686b29088..3c291c5a5 100644 --- a/isvctl/tests/test_stub_contracts.py +++ b/isvctl/tests/test_stub_contracts.py @@ -156,7 +156,7 @@ def _collect_yaml_checks() -> list[StepArgCheck]: checks: list[StepArgCheck] = [] yaml_paths = sorted( [ - *CONFIGS_DIR.glob("suites/*.yaml"), + *CONFIGS_DIR.glob("suites/**/*.yaml"), *CONFIGS_DIR.glob("providers/*.yaml"), # k3s.yaml, microk8s.yaml, minikube.yaml *CONFIGS_DIR.glob("providers/*/config/*.yaml"), # aws/config/*.yaml, my-isv/config/*.yaml ] diff --git a/isvctl/tests/test_suite_resolution.py b/isvctl/tests/test_suite_resolution.py index 86e253c10..0ba72c76e 100644 --- a/isvctl/tests/test_suite_resolution.py +++ b/isvctl/tests/test_suite_resolution.py @@ -52,6 +52,21 @@ def test_one_suite_flag_resolves_canonical_and_provider_suites(tmp_path: Path) - assert canonical_plain.platform is None +def test_canonical_suite_resolution_discovers_nested_suite_yaml(tmp_path: Path) -> None: + """Domain folders under suites remain selectable through the generic resolver.""" + _write_catalog(tmp_path) + domain = tmp_path / "suites" / "launch-kit" + domain.mkdir() + nested = domain / "network-operator.yaml" + nested.write_text("tests:\n validations: {}\n") + + resolved = resolve_suite(None, "network-operator", configs_root=tmp_path) + + assert resolved.config_path == nested + assert resolved.name == "network_operator" + assert resolved.platform is None + + def test_capability_uses_catalog_vocabulary(tmp_path: Path) -> None: """An unknown capability is rejected while omitted context disables filtering.""" _write_catalog(tmp_path) diff --git a/isvctl/tests/test_test_cli_labels.py b/isvctl/tests/test_test_cli_labels.py index f08803270..93ec47c4b 100644 --- a/isvctl/tests/test_test_cli_labels.py +++ b/isvctl/tests/test_test_cli_labels.py @@ -52,6 +52,66 @@ def _write_config(tmp_path: Path) -> Path: return config +def test_validation_result_detail_compacts_successful_subtests() -> None: + """Successful validations with probes render an aggregate instead of their long message.""" + detail = test_cli._validation_result_detail( + { + "passed": True, + "skipped": False, + "state": "passed", + "message": "long; member; output", + "subtest_summary": {"total": 6, "passed": 6, "failed": 0, "skipped": 0}, + } + ) + + assert detail == "6 subtests passed" + + +def test_validation_result_detail_preserves_failures() -> None: + """Failed validations keep their actionable message even when they report probes.""" + detail = test_cli._validation_result_detail( + { + "passed": False, + "skipped": False, + "state": "failed", + "message": "RdmaCheck: worker-a -> worker-b timed out", + "subtest_summary": {"total": 6, "passed": 5, "failed": 1, "skipped": 0}, + } + ) + + assert detail == "RdmaCheck: worker-a -> worker-b timed out" + + +def test_validation_result_detail_counts_successful_runs_with_skips() -> None: + """An allowed skipped probe remains visible in the concise success summary.""" + detail = test_cli._validation_result_detail( + { + "passed": True, + "skipped": False, + "state": "passed", + "message": "long output", + "subtest_summary": {"total": 6, "passed": 5, "failed": 0, "skipped": 1}, + } + ) + + assert detail == "6 subtests: 5 passed, 0 failed, 1 skipped" + + +def test_validation_result_detail_derives_total_for_legacy_summaries() -> None: + """Older result payloads remain concise when they omit the explicit total.""" + detail = test_cli._validation_result_detail( + { + "passed": True, + "skipped": False, + "state": "passed", + "message": "long output", + "subtest_summary": {"passed": 5, "failed": 0, "skipped": 1}, + } + ) + + assert detail == "6 subtests: 5 passed, 0 failed, 1 skipped" + + def _write_provider_config(root: Path, provider: str, name: str, suite: str, platform: str) -> Path: """Write a minimal provider config importing one suite.""" config_path = root / "providers" / provider / "config" / name @@ -129,6 +189,49 @@ def test_test_run_forwards_label_filters(monkeypatch: pytest.MonkeyPatch, tmp_pa assert _FakeOrchestrator.captured["include_labels"] == ["gpu", "slow"] +def test_orchestration_summary_hides_filtered_validations_by_default( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """The default summary omits phases containing only selection-filtered checks.""" + config = _write_config(tmp_path) + + class FilteredOrchestrator(_FakeOrchestrator): + """Return one resolution-only phase for a filtered validation.""" + + def run(self, **kwargs: Any) -> OrchestratorResult: + """Return the synthetic filtered result.""" + return OrchestratorResult( + success=True, + phases=[ + PhaseResult( + phase=Phase.TEST, + success=True, + message="test phase validations resolved without execution", + details={ + "validations": [ + { + "name": "InfiniBandCheck", + "skipped": True, + "state": "skipped", + "skip_reason": "test_excluded", + "message": "does not match selected label ethernet", + } + ] + }, + ) + ], + ) + + monkeypatch.setattr(test_cli, "Orchestrator", FilteredOrchestrator) + + result = runner.invoke(test_cli.app, ["run", "-f", str(config), "--label", "ethernet", "--no-upload"]) + + assert result.exit_code == 0, result.output + assert "InfiniBandCheck" not in result.output + assert "TEST : test phase validations resolved" not in result.output + + def test_test_run_uploads_the_complete_catalog_document( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/isvtest/src/isvtest/catalog.py b/isvtest/src/isvtest/catalog.py index 302df3e82..a8e3d1d3b 100644 --- a/isvtest/src/isvtest/catalog.py +++ b/isvtest/src/isvtest/catalog.py @@ -20,7 +20,7 @@ The catalog is version-keyed by the installed isvtest package version. Suite placement and capability requirements come only from canonical -``isvctl/configs/suites/*.yaml`` wiring. +``isvctl/configs/suites/**/*.yaml`` wiring. """ import logging @@ -209,7 +209,7 @@ def _iter_suite_docs() -> Iterator[tuple[Path, dict[str, Any]]]: if not configs_dir: logger.warning("Could not locate isvctl/configs/ directory") return - for config_path in sorted((configs_dir / "suites").glob("*.yaml")): + for config_path in sorted((configs_dir / "suites").rglob("*.yaml")): yield config_path, yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} diff --git a/isvtest/src/isvtest/core/composite.py b/isvtest/src/isvtest/core/composite.py index 833fd1774..0b08e4d26 100644 --- a/isvtest/src/isvtest/core/composite.py +++ b/isvtest/src/isvtest/core/composite.py @@ -42,6 +42,8 @@ from typing import Any, ClassVar +import pytest + from isvtest.core.validation import BaseValidation, get_validation_class COMPOSE_KEY = "compose" @@ -86,7 +88,7 @@ class CompositeCheck(BaseValidation): _exclude_from_discovery: ClassVar[bool] = True def run(self) -> None: - """Run every configured member and fail the composite on invalid or failed members.""" + """Run every member, retaining skips and failing on invalid or failed members.""" raw = self.config.get(COMPOSE_KEY) members = composed_members(raw) if not members: @@ -115,9 +117,23 @@ def run(self) -> None: member = member_class(runner=self.runner, config={**shared, **member_params}) member.name = member_name - result = member.execute() + try: + result = member.execute() + except pytest.skip.Exception as exc: + reason = str(exc) + self.report_subtest(member_name, False, reason, skipped=True) + outputs.append(f"{member_name}: skipped - {reason}") + continue message = result["output"] if result["passed"] else result["error"] self.report_subtest(member_name, result["passed"], message, duration=result["duration"]) + for nested in result.get("subtests", []): + self.report_subtest( + f"{member_name}/{nested['name']}", + bool(nested.get("passed")), + str(nested.get("message", "")), + skipped=bool(nested.get("skipped")), + duration=nested.get("duration"), + ) if result["passed"]: outputs.append(f"{member_name}: {message}" if message else member_name) else: diff --git a/isvtest/src/isvtest/core/resolution.py b/isvtest/src/isvtest/core/resolution.py index bafa0144f..d5ff41b21 100644 --- a/isvtest/src/isvtest/core/resolution.py +++ b/isvtest/src/isvtest/core/resolution.py @@ -20,7 +20,7 @@ import logging from collections.abc import Iterable, Mapping from collections.abc import Set as AbstractSet -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import StrEnum from functools import cache from typing import Any @@ -93,6 +93,7 @@ class ErrorReason(StrEnum): INVALID_CONFIG = "invalid_config" RUNTIME_EXCEPTION = "runtime_exception" + STEP_FAILED = "step_failed" TEMPLATE_RENDER_FAILED = "template_render_failed" @@ -109,6 +110,20 @@ class ValidationEntry: requires: tuple[str, ...] = () +@dataclass(frozen=True) +class SubtestSummary: + """Aggregate counts for the subtests reported by one validation.""" + + passed: int = 0 + failed: int = 0 + skipped: int = 0 + + @property + def total(self) -> int: + """Return the total number of reported subtests.""" + return self.passed + self.failed + self.skipped + + @dataclass class ResolvedEntry: """Lifecycle record for a single validation entry.""" @@ -120,6 +135,7 @@ class ResolvedEntry: error_reason: ErrorReason | None = None message: str = "" duration_seconds: float = 0.0 + subtest_summary: SubtestSummary = field(default_factory=SubtestSummary) @property def is_ready(self) -> bool: @@ -240,6 +256,67 @@ def parse_validations(raw_config: Mapping[str, Any]) -> list[ValidationEntry]: return entries +def resolve_entry_selection( + entry: ValidationEntry, + *, + include_labels: AbstractSet[str], + exclude_labels: AbstractSet[str], + exclude_tests: AbstractSet[str], + released_tests: AbstractSet[str] | None, + capability: str | None = None, +) -> ResolvedEntry | None: + """Return a terminal result when selection excludes an entry, otherwise ``None``. + + This is the provider-neutral selection boundary shared by validation + execution and lifecycle steps gated with ``requires_selected_validations``. + It deliberately stops before phase, step-output, and template resolution. + """ + config_error = _validate_entry_shape(entry) + if config_error: + return _error(entry, ErrorReason.INVALID_CONFIG, config_error) + + # Variant-aware match: a configured ``ClassName-Variant`` is considered + # released when the bare ``ClassName`` is in the manifest, mirroring the + # pytest-discovery path (``_is_released_validation`` in test_validations). + if released_tests is not None and resolve_class_key(entry.name, released_tests) is None: + return _skip( + entry, + SkipReason.UNRELEASED, + f"validation '{entry.name}' is not in released_tests.json", + ) + + if entry.name in exclude_tests: + return _skip(entry, SkipReason.EXCLUDED, f"validation '{entry.name}' is excluded by name") + + if capability is not None and not requirements_satisfied(entry.requires, capability): + requirement_list = ", ".join(entry.requires) or "(none)" + return _skip( + entry, + SkipReason.CAPABILITY_REQUIREMENT, + f"requires {requirement_list} (context: {capability})", + ) + + missing_include_labels = sorted(set(include_labels).difference(entry.labels)) + if missing_include_labels: + label_list = ", ".join(sorted(include_labels)) + return _skip( + entry, + SkipReason.EXCLUDED, + f"validation '{entry.name}' does not match all selected labels: {label_list}", + ) + + label_matches = sorted(set(entry.labels).intersection(exclude_labels)) + if label_matches: + label_list = ", ".join(label_matches) + return _skip( + entry, + SkipReason.EXCLUDED, + f"validation '{entry.name}' is excluded by label: {label_list}", + ) + + return None + + def resolve_entries( entries: list[ValidationEntry], *, @@ -278,62 +355,16 @@ def resolve_entries( env = _create_jinja_env() for entry in entries: - config_error = _validate_entry_shape(entry) - if config_error: - resolved.append(_error(entry, ErrorReason.INVALID_CONFIG, config_error)) - continue - - # Variant-aware match: a configured ``ClassName-Variant`` is considered - # released when the bare ``ClassName`` is in the manifest, mirroring the - # pytest-discovery path (``_is_released_validation`` in test_validations). - if released_tests is not None and resolve_class_key(entry.name, released_tests) is None: - resolved.append( - _skip( - entry, - SkipReason.UNRELEASED, - f"validation '{entry.name}' is not in released_tests.json", - ) - ) - continue - - if entry.name in exclude_tests: - resolved.append(_skip(entry, SkipReason.EXCLUDED, f"validation '{entry.name}' is excluded by name")) - continue - - if capability is not None and not requirements_satisfied(entry.requires, capability): - requirement_list = ", ".join(entry.requires) or "(none)" - context_list = capability - resolved.append( - _skip( - entry, - SkipReason.CAPABILITY_REQUIREMENT, - f"requires {requirement_list} (context: {context_list})", - ) - ) - continue - - missing_include_labels = sorted(set(include_labels).difference(entry.labels)) - if missing_include_labels: - label_list = ", ".join(sorted(include_labels)) - resolved.append( - _skip( - entry, - SkipReason.EXCLUDED, - f"validation '{entry.name}' does not match all selected labels: {label_list}", - ) - ) - continue - - label_matches = sorted(set(entry.labels).intersection(exclude_labels)) - if label_matches: - label_list = ", ".join(label_matches) - resolved.append( - _skip( - entry, - SkipReason.EXCLUDED, - f"validation '{entry.name}' is excluded by label: {label_list}", - ) - ) + selection_result = resolve_entry_selection( + entry, + include_labels=include_labels, + exclude_labels=exclude_labels, + exclude_tests=exclude_tests, + released_tests=released_tests, + capability=capability, + ) + if selection_result is not None: + resolved.append(selection_result) continue if entry.step and entry.step in skipped_steps: diff --git a/isvtest/src/isvtest/main.py b/isvtest/src/isvtest/main.py index 44806cadd..f2b723946 100644 --- a/isvtest/src/isvtest/main.py +++ b/isvtest/src/isvtest/main.py @@ -36,7 +36,7 @@ from isvtest.config.loader import ConfigLoader from isvtest.core import runners as reframe_runner from isvtest.core.logger import setup_logger -from isvtest.core.resolution import ErrorReason, ResolvedEntry, SkipReason, State +from isvtest.core.resolution import ErrorReason, ResolvedEntry, SkipReason, State, SubtestSummary from isvtest.tests.test_validations import ( clear_validation_results, get_validation_results, @@ -223,6 +223,12 @@ def _result_to_resolved_entry(entry: ResolvedEntry, result: dict[str, Any]) -> R """Convert a captured pytest validation result to a terminal resolved entry.""" message = str(result.get("message", "")) duration = float(result.get("duration", 0.0) or 0.0) + raw_subtests = result.get("subtest_summary", {}) + subtest_summary = SubtestSummary( + passed=int(raw_subtests.get("passed", 0)) if isinstance(raw_subtests, dict) else 0, + failed=int(raw_subtests.get("failed", 0)) if isinstance(raw_subtests, dict) else 0, + skipped=int(raw_subtests.get("skipped", 0)) if isinstance(raw_subtests, dict) else 0, + ) if result.get("skipped"): return ResolvedEntry( entry=entry.entry, @@ -231,6 +237,7 @@ def _result_to_resolved_entry(entry: ResolvedEntry, result: dict[str, Any]) -> R skip_reason=SkipReason.RUNTIME_SKIP, message=message, duration_seconds=duration, + subtest_summary=subtest_summary, ) if result.get("passed", False): return ResolvedEntry( @@ -239,6 +246,7 @@ def _result_to_resolved_entry(entry: ResolvedEntry, result: dict[str, Any]) -> R state=State.PASSED, message=message, duration_seconds=duration, + subtest_summary=subtest_summary, ) if result.get("error_reason") == ErrorReason.RUNTIME_EXCEPTION.value: return ResolvedEntry( @@ -248,6 +256,7 @@ def _result_to_resolved_entry(entry: ResolvedEntry, result: dict[str, Any]) -> R error_reason=ErrorReason.RUNTIME_EXCEPTION, message=message, duration_seconds=duration, + subtest_summary=subtest_summary, ) return ResolvedEntry( entry=entry.entry, @@ -255,6 +264,7 @@ def _result_to_resolved_entry(entry: ResolvedEntry, result: dict[str, Any]) -> R state=State.FAILED, message=message, duration_seconds=duration, + subtest_summary=subtest_summary, ) diff --git a/isvtest/src/isvtest/testing/subtests.py b/isvtest/src/isvtest/testing/subtests.py index 09f889fbb..e7cd5fb83 100644 --- a/isvtest/src/isvtest/testing/subtests.py +++ b/isvtest/src/isvtest/testing/subtests.py @@ -509,11 +509,6 @@ def _inject_subtests_into_junit(junit_path: Path, reports: list[SubTestReport]) name = tc.get("name", "") testcase_indices[name] = idx - # Track counts for updating testsuite attributes - added_tests = 0 - added_failures = 0 - added_skipped = 0 - # Insert subtests after their parent, in reverse order of parent index # to avoid index shifting issues insertions: list[tuple[int, list[ET.Element]]] = [] @@ -540,16 +535,12 @@ def _inject_subtests_into_junit(junit_path: Path, reports: list[SubTestReport]) testcase.set("classname", "") # Match pytest's default testcase.set("time", f"{report.duration:.3f}") - added_tests += 1 - if report.failed: - added_failures += 1 failure = ET.SubElement(testcase, "failure") failure.set("message", f"Subtest {subtest_desc} failed") if report.longrepr: failure.text = str(report.longrepr)[:2000] # Limit length elif report.skipped: - added_skipped += 1 skipped_elem = ET.SubElement(testcase, "skipped") # longrepr from both our skipped-flag path and pytest.skip() # is a (file, line, msg) tuple; surface the msg so the @@ -578,14 +569,15 @@ def _inject_subtests_into_junit(junit_path: Path, reports: list[SubTestReport]) for i, elem in enumerate(subtest_elements): testsuite.insert(insert_pos + i, elem) - # Update testsuite counts - current_tests = int(testsuite.get("tests", "0")) - current_failures = int(testsuite.get("failures", "0")) - current_skipped = int(testsuite.get("skipped", "0")) - - testsuite.set("tests", str(current_tests + added_tests)) - testsuite.set("failures", str(current_failures + added_failures)) - testsuite.set("skipped", str(current_skipped + added_skipped)) + # pytest counts subtest reports in the testsuite attributes even though + # its JUnit plugin does not serialize them as testcase elements. Adding + # our testcase nodes and incrementing those attributes would therefore + # double-count every subtest. Reconcile counters with the serialized XML. + serialized_cases = testsuite.findall("testcase") + testsuite.set("tests", str(len(serialized_cases))) + testsuite.set("failures", str(sum(case.find("failure") is not None for case in serialized_cases))) + testsuite.set("errors", str(sum(case.find("error") is not None for case in serialized_cases))) + testsuite.set("skipped", str(sum(case.find("skipped") is not None for case in serialized_cases))) # Write back tree.write(junit_path, encoding="utf-8", xml_declaration=True) diff --git a/isvtest/src/isvtest/tests/test_validations.py b/isvtest/src/isvtest/tests/test_validations.py index f95db81e0..d430bcda0 100644 --- a/isvtest/src/isvtest/tests/test_validations.py +++ b/isvtest/src/isvtest/tests/test_validations.py @@ -313,6 +313,18 @@ def test_validation( "category": category, "duration": result.get("duration", 0.0), "error_reason": result.get("error_reason"), + "subtest_summary": { + "total": len(result.get("subtests", [])), + "passed": sum( + 1 for subtest in result.get("subtests", []) if subtest.get("passed") and not subtest.get("skipped") + ), + "failed": sum( + 1 + for subtest in result.get("subtests", []) + if not subtest.get("passed") and not subtest.get("skipped") + ), + "skipped": sum(1 for subtest in result.get("subtests", []) if subtest.get("skipped")), + }, } ) diff --git a/isvtest/src/isvtest/validations/k8s_launch_kit/__init__.py b/isvtest/src/isvtest/validations/k8s_launch_kit/__init__.py new file mode 100644 index 000000000..953b56014 --- /dev/null +++ b/isvtest/src/isvtest/validations/k8s_launch_kit/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Kubernetes Launch Kit validation checks.""" diff --git a/isvtest/src/isvtest/validations/k8s_launch_kit/checks.py b/isvtest/src/isvtest/validations/k8s_launch_kit/checks.py new file mode 100644 index 000000000..8597d9088 --- /dev/null +++ b/isvtest/src/isvtest/validations/k8s_launch_kit/checks.py @@ -0,0 +1,719 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Assertions over unmodified Kubernetes Launch Kit command output. + +Cluster interaction and Launch Kit command execution stay in the provider. +These checks interpret the real discover and validate documents and expose +resource or matrix rows as pytest subtests for actionable reporting. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, ClassVar + +import pytest + +from isvtest.core.validation import BaseValidation + +_CONNECTIVITY_FAMILIES: dict[str, set[int]] = { + "icmp": {0, 1}, + "rping": {2, 3}, + "ib_write_bw": {4, 5}, + "gpudirect_dmabuf": {6, 7}, +} + +_PROFILE_NETWORK_KINDS: dict[tuple[str, str], str] = { + ("ethernet", "sriov"): "SriovNetwork", + ("infiniband", "sriov"): "SriovIBNetwork", + ("ethernet", "rdma_shared"): "MacvlanNetwork", + ("infiniband", "rdma_shared"): "IPoIBNetwork", + ("ethernet", "host_device"): "HostDeviceNetwork", + ("infiniband", "host_device"): "HostDeviceNetwork", +} + + +def _object(value: Any) -> dict[str, Any]: + """Return ``value`` as a JSON object or an empty object.""" + return value if isinstance(value, dict) else {} + + +def _list(value: Any) -> list[Any]: + """Return ``value`` as a list or an empty list.""" + return value if isinstance(value, list) else [] + + +def _profile_network_kind(profile: dict[str, Any]) -> str | None: + """Return the expected secondary-network resource for a resolved profile.""" + fabric = profile.get("fabric") + deployment = profile.get("deployment") + if not isinstance(fabric, str) or not isinstance(deployment, str): + return None + return _PROFILE_NETWORK_KINDS.get((fabric, deployment)) + + +class _LaunchKitCheck(BaseValidation): + """Shared parsing and subtest reporting for Launch Kit checks.""" + + _exclude_from_discovery: ClassVar[bool] = True + + def _step_output(self, operation: str | None = None) -> dict[str, Any] | None: + """Return the bound provider envelope and validate its operation.""" + output = self.config.get("step_output") + if not isinstance(output, dict): + self.set_failed("Missing Launch Kit step_output") + return None + if operation is not None and output.get("operation") != operation: + self.set_failed(f"Expected Launch Kit operation {operation!r}, got {output.get('operation')!r}") + return None + return output + + def _configured_output(self, key: str) -> dict[str, Any]: + """Decode another step envelope passed through validation configuration.""" + value = self.config.get(key) + if isinstance(value, dict): + return value + if not isinstance(value, str) or not value: + return {} + try: + parsed = json.loads(value) + except json.JSONDecodeError: + return {} + return parsed if isinstance(parsed, dict) else {} + + def _documents(self, output: dict[str, Any]) -> list[dict[str, Any]]: + """Return only object documents from a provider envelope.""" + return [document for document in _list(output.get("documents")) if isinstance(document, dict)] + + def _profile(self) -> dict[str, Any] | None: + """Return the resolved profile from the real discover JSONResult.""" + discover = self._configured_output("discover_output") + documents = self._documents(discover) + profile = documents[0].get("profile") if documents else None + if not isinstance(profile, dict): + self.set_failed("Launch Kit discover output has no resolved profile") + return None + return profile + + def _static_document(self, output: dict[str, Any]) -> dict[str, Any] | None: + """Find the manifest/version validation document.""" + for document in self._documents(output): + if {"versionCheck", "manifests", "summary"}.issubset(document): + return document + provider_error = output.get("error") + suffix = f": {provider_error}" if isinstance(provider_error, str) and provider_error else "" + self.set_failed(f"Launch Kit validate output has no static validation document{suffix}") + return None + + def _connectivity(self, output: dict[str, Any]) -> dict[str, Any] | None: + """Find the source-bound connectivity matrix document.""" + for document in self._documents(output): + connectivity = document.get("connectivity") + if isinstance(connectivity, dict): + return connectivity + provider_error = output.get("error") + suffix = f": {provider_error}" if isinstance(provider_error, str) and provider_error else "" + self.set_failed(f"Launch Kit validate output has no connectivity matrix{suffix}") + return None + + def _finish_probes(self, title: str, probes: list[dict[str, Any]]) -> None: + """Report every probe and aggregate its failures after the final row.""" + if not probes: + if not self._error: + self.set_failed(f"{title} produced no probes") + return + failures: list[str] = [] + for index, probe in enumerate(probes, start=1): + name = str(probe.get("name") or f"probe-{index}") + passed = probe.get("passed") is True + skipped = probe.get("skipped") is True + message = str(probe.get("message") or probe.get("error") or "") + self.report_subtest(name, passed=passed, skipped=skipped, message=message) + if not passed and not skipped: + failures.append(f"{name}: {message or 'failed without a diagnostic'}") + if failures: + self.set_failed(f"{title} failed: {'; '.join(failures)}") + return + self.set_passed(f"{title} passed ({len(probes)} probes)") + + def _manifest_probes( + self, + output: dict[str, Any], + *, + required_kinds: set[str] | None = None, + ) -> list[dict[str, Any]]: + """Build probes from Launch Kit manifest validation rows.""" + static = self._static_document(output) + if static is None: + return [] + manifests = [item for item in _list(static.get("manifests")) if isinstance(item, dict)] + if required_kinds is not None: + manifests = [item for item in manifests if item.get("Kind") in required_kinds] + probes = [] + for item in manifests: + kind = str(item.get("Kind") or "unknown-kind") + namespace = str(item.get("Namespace") or "cluster") + name = str(item.get("Name") or "unknown") + passed = item.get("State") == "success" and item.get("Missing") is not True + probes.append( + { + "name": f"{kind}/{namespace}/{name}", + "passed": passed, + "message": str(item.get("Reason") or item.get("Detail") or item.get("State") or ""), + } + ) + return probes + + def _matrix_probes(self, output: dict[str, Any], families: set[str]) -> list[dict[str, Any]]: + """Build one informative subtest for every selected connectivity row.""" + connectivity = self._connectivity(output) + if connectivity is None: + return [] + probes: list[dict[str, Any]] = [] + for row in _list(connectivity.get("PingResults")): + if not isinstance(row, dict): + continue + test = _object(row.get("Test")) + kind = test.get("Kind") + explicit_family = row.get("Family") + family = explicit_family if isinstance(explicit_family, str) else None + if family not in _CONNECTIVITY_FAMILIES: + family = next( + (name for name, family_kinds in _CONNECTIVITY_FAMILIES.items() if kind in family_kinds), + None, + ) + if family not in families: + continue + source = str(test.get("SrcNode") or test.get("SrcPod") or "unknown-source") + destination = str(test.get("DstNode") or test.get("DstPod") or "unknown-destination") + source_rail = str(test.get("SrcRail") or test.get("Rail") or "unknown-rail") + destination_rail = str(test.get("DstRail") or test.get("Rail") or "unknown-rail") + expectation = str(row.get("Expectation") or test.get("Expectation") or "required") + passed = row.get("OK") is True + stderr = str(row.get("Stderr") or "").strip() + error = str(row.get("Error") or "").strip() + bandwidth = row.get("BandwidthGbps") + minimum = row.get("MinBandwidthGbps") + details = [f"expectation={expectation}", f"observedOK={row.get('ObservedOK')}"] + if family in {"ib_write_bw", "gpudirect_dmabuf"}: + details.extend([f"bandwidthGbps={bandwidth}", f"minimumGbps={minimum}"]) + if family == "gpudirect_dmabuf": + source_gpu = test.get("SrcGPUIndex") + destination_gpu = test.get("DstGPUIndex") + valid_gpu_indices = all( + isinstance(index, int) and not isinstance(index, bool) and index >= 0 + for index in (source_gpu, destination_gpu) + ) + passed = passed and valid_gpu_indices + details.append(f"gpuIndices={source_gpu}->{destination_gpu}") + source_gpu_pci = test.get("SrcGPUPCIAddress") + destination_gpu_pci = test.get("DstGPUPCIAddress") + if source_gpu_pci: + details.append(f"sourceGpuPci={source_gpu_pci}") + if destination_gpu_pci: + details.append(f"destinationGpuPci={destination_gpu_pci}") + if not valid_gpu_indices: + details.append("invalid or missing endpoint GPU index") + if stderr: + details.append(f"stderr={stderr}") + if error and error != stderr: + details.append(f"error={error}") + probes.append( + { + "name": f"{family}/{source}->{destination}/{source_rail}->{destination_rail}", + "passed": passed, + "message": ", ".join(details), + "source_rail": source_rail, + "destination_rail": destination_rail, + } + ) + return probes + + def _kind_coverage_probes( + self, + output: dict[str, Any], + expected_kinds: set[str], + ) -> list[dict[str, Any]]: + """Require every applicable manifest kind to appear in Launch Kit output.""" + static = self._static_document(output) + if static is None: + return [] + observed = { + str(item.get("Kind")) + for item in _list(static.get("manifests")) + if isinstance(item, dict) and item.get("Kind") + } + return [ + { + "name": f"kind-coverage/{kind}", + "passed": kind in observed, + "message": f"observed kinds: {', '.join(sorted(observed)) or '(none)'}", + } + for kind in sorted(expected_kinds) + ] + + +class LaunchKitKubernetesPrerequisiteCheck(_LaunchKitCheck): + """Require a reachable Kubernetes API and a non-empty Ready-node inventory.""" + + description: ClassVar[str] = "Check Kubernetes prerequisites before Launch Kit execution" + + def run(self) -> None: + """Report every provider preflight probe.""" + output = self._configured_output("preflight_output") or self._step_output("kubernetes-preflight") + if output is None: + return + probes = [probe for probe in _list(output.get("checks")) if isinstance(probe, dict)] + self._finish_probes("Kubernetes prerequisite", probes) + + +class LaunchKitTopologyDiscoveryCheck(_LaunchKitCheck): + """Validate that Launch Kit completed discovery and resolved a profile.""" + + description: ClassVar[str] = "Check cluster topology discovery with Kubernetes Launch Kit" + + def run(self) -> None: + """Check the real discover JSONResult without inventing topology fields.""" + output = self._configured_output("discover_output") or self._step_output("discover") + if output is None: + return + documents = self._documents(output) + document = documents[0] if len(documents) == 1 else {} + profile = _object(document.get("profile")) + probes = [ + { + "name": "discover-command", + "passed": output.get("success") is True and document.get("success") is True, + "message": str(output.get("error") or f"phase={document.get('phase')!r}"), + }, + { + "name": "resolved-profile", + "passed": bool(profile.get("fabric") and profile.get("deployment")), + "message": f"fabric={profile.get('fabric')}, deployment={profile.get('deployment')}", + }, + ] + self._finish_probes("Launch Kit topology discovery", probes) + + +class LaunchKitDeploymentHealthCheck(_LaunchKitCheck): + """Validate the Network Operator release and every generated resource.""" + + description: ClassVar[str] = "Check Network Operator deployment health with Kubernetes Launch Kit" + + def run(self) -> None: + """Report version and manifest readiness independently of connectivity.""" + output = self._step_output("validate") + if output is None: + return + static = self._static_document(output) + if static is None: + return + version = _object(static.get("versionCheck")) + summary = _object(static.get("summary")) + version_skipped = version.get("Skipped") is True + probes = [ + { + "name": "launch-kit-validate-command", + "passed": output.get("success") is True and output.get("exit_code") == 0, + "message": str( + output.get("error") or f"success={output.get('success')!r}, exitCode={output.get('exit_code')!r}" + ), + }, + { + "name": "network-operator-version", + "passed": not version_skipped and version.get("Match") is True, + "skipped": version_skipped, + "message": ( + str(version.get("Reason")) + if version_skipped + else ( + f"selected={version.get('SelectedRelease')}, expected={version.get('ExpectedVersion')}, " + f"deployed={_object(version.get('DeployedRelease')).get('ChartVersion')}" + ) + ), + }, + { + "name": "static-summary", + "passed": summary.get("success") is True, + "message": ( + f"success={summary.get('successManifests')}/{summary.get('totalManifests')}, " + f"errors={summary.get('errorManifests')}, missing={summary.get('missingManifests')}" + ), + }, + { + "name": "manifest-inventory", + "passed": bool(_list(static.get("manifests"))), + "message": f"rows={len(_list(static.get('manifests')))}", + }, + *self._manifest_probes(output), + ] + self._finish_probes("Network Operator deployment health", probes) + + +class LaunchKitSriovReadinessCheck(_LaunchKitCheck): + """Validate SR-IOV policies and secondary-network resources.""" + + description: ClassVar[str] = "Check SR-IOV Network RDMA readiness with Kubernetes Launch Kit" + + def run(self) -> None: + """Check applicable validated resources for an SR-IOV profile.""" + output = self._step_output("validate") + if output is None: + return + profile = self._profile() + if profile is None: + return + if profile.get("deployment") != "sriov": + pytest.skip(f"selected Launch Kit deployment is {profile.get('deployment')}, not sriov") + fabric = profile.get("fabric") + network_kind = "SriovIBNetwork" if fabric == "infiniband" else "SriovNetwork" + expected_kinds = {"SriovNetworkNodePolicy", network_kind} + probes = self._kind_coverage_probes(output, expected_kinds) + probes.extend( + self._manifest_probes( + output, + required_kinds=expected_kinds, + ) + ) + self._finish_probes("SR-IOV readiness", probes) + + +class LaunchKitRdmaConnectivityCheck(_LaunchKitCheck): + """Validate every rping matrix result.""" + + description: ClassVar[str] = "Check pod-to-pod RDMA connectivity with Kubernetes Launch Kit" + + def run(self) -> None: + """Report all same-rail and cross-rail RDMA-CM observations.""" + output = self._step_output("validate") + if output is not None: + self._finish_probes("RDMA-CM connectivity", self._matrix_probes(output, {"rping"})) + + +class LaunchKitRoceCheck(_LaunchKitCheck): + """Validate the selected Ethernet/RoCE profile resources.""" + + description: ClassVar[str] = "Check RoCE secondary networking with Kubernetes Launch Kit" + + def run(self) -> None: + """Skip non-Ethernet profiles and report the selected network resources.""" + output = self._step_output("validate") + if output is None: + return + profile = self._profile() + if profile is None: + return + if profile.get("fabric") != "ethernet": + pytest.skip(f"selected Launch Kit fabric is {profile.get('fabric')}, not ethernet") + network_kind = _profile_network_kind(profile) + if network_kind is None: + self.set_failed(f"Launch Kit returned an unsupported profile: {profile}") + return + probes = self._kind_coverage_probes(output, {network_kind}) + probes.extend(self._manifest_probes(output, required_kinds={network_kind})) + self._finish_probes("Ethernet/RoCE profile", probes) + + +class LaunchKitInfiniBandCheck(_LaunchKitCheck): + """Validate the selected InfiniBand profile resources.""" + + description: ClassVar[str] = "Check InfiniBand networking with Kubernetes Launch Kit" + + def run(self) -> None: + """Skip non-IB profiles and report IB network resources.""" + output = self._step_output("validate") + if output is None: + return + profile = self._profile() + if profile is None: + return + if profile.get("fabric") != "infiniband": + pytest.skip(f"selected Launch Kit fabric is {profile.get('fabric')}, not infiniband") + network_kind = _profile_network_kind(profile) + if network_kind is None: + self.set_failed(f"Launch Kit returned an unsupported profile: {profile}") + return + probes = self._kind_coverage_probes(output, {network_kind}) + probes.extend(self._manifest_probes(output, required_kinds={network_kind})) + self._finish_probes("InfiniBand profile", probes) + + +class LaunchKitHostDeviceCheck(_LaunchKitCheck): + """Validate an applicable host-device profile.""" + + description: ClassVar[str] = "Check host-device networking with Kubernetes Launch Kit" + + def run(self) -> None: + """Skip other deployment types and report HostDeviceNetwork rows.""" + output = self._step_output("validate") + if output is None: + return + profile = self._profile() + if profile is None: + return + if profile.get("deployment") != "host_device": + pytest.skip(f"selected Launch Kit deployment is {profile.get('deployment')}, not host_device") + probes = self._kind_coverage_probes(output, {"HostDeviceNetwork"}) + probes.extend(self._manifest_probes(output, required_kinds={"HostDeviceNetwork"})) + self._finish_probes( + "host-device networking", + probes, + ) + + +class LaunchKitSecondaryNetworkCheck(_LaunchKitCheck): + """Validate secondary-network resources and test DaemonSet readiness.""" + + description: ClassVar[str] = "Check secondary-network and IPAM readiness with Kubernetes Launch Kit" + + def run(self) -> None: + """Report network/IPPool manifests and test-pod rollout state.""" + output = self._step_output("validate") + if output is None: + return + profile = self._profile() + if profile is None: + return + network_kind = _profile_network_kind(profile) + if network_kind is None: + self.set_failed(f"Launch Kit returned an unsupported profile: {profile}") + return + network_kinds = {"SriovNetwork", "SriovIBNetwork", "MacvlanNetwork", "IPoIBNetwork", "HostDeviceNetwork"} + probes = self._kind_coverage_probes(output, {"IPPool", network_kind}) + probes.extend( + self._manifest_probes( + output, + required_kinds={"IPPool", *network_kinds}, + ) + ) + static = self._static_document(output) + if static is None: + return + observed_network_kinds = { + str(item.get("Kind")) + for item in _list(static.get("manifests")) + if isinstance(item, dict) and item.get("Kind") in network_kinds + } + probes.append( + { + "name": "kind-coverage/secondary-network", + "passed": bool(observed_network_kinds), + "message": f"observed kinds: {', '.join(sorted(observed_network_kinds)) or '(none)'}", + } + ) + connectivity = self._connectivity(output) + if connectivity is None: + return + for daemonset in _list(connectivity.get("DaemonSets")): + if not isinstance(daemonset, dict): + continue + ref = _object(daemonset.get("Ref")) + rollout = _object(daemonset.get("Rollout")) + desired = rollout.get("Desired") + ready = rollout.get("Ready") + not_ready = rollout.get("NotReady") + valid_counts = all(type(value) is int and value >= 0 for value in (desired, ready, not_ready)) + rollout_detail = f"ready={ready}/{desired}, notReady={not_ready}" + if not valid_counts: + rollout_detail += ", invalid or missing integer rollout counts" + probes.append( + { + "name": f"DaemonSet/{ref.get('Namespace')}/{ref.get('Name')}", + "passed": valid_counts and desired > 0 and ready == desired and not_ready == 0, + "message": rollout_detail, + } + ) + self._finish_probes("secondary-network readiness", probes) + + +class LaunchKitRdmaSharedCheck(_LaunchKitCheck): + """Validate an applicable RDMA Shared profile.""" + + description: ClassVar[str] = "Check RDMA Shared networking with Kubernetes Launch Kit" + + def run(self) -> None: + """Skip other profiles and report Macvlan or IPoIB resources.""" + output = self._step_output("validate") + if output is None: + return + profile = self._profile() + if profile is None: + return + if profile.get("deployment") != "rdma_shared": + pytest.skip(f"selected Launch Kit deployment is {profile.get('deployment')}, not rdma_shared") + network_kind = _profile_network_kind(profile) + if network_kind is None: + self.set_failed(f"Launch Kit returned an unsupported profile: {profile}") + return + probes = self._kind_coverage_probes(output, {network_kind}) + probes.extend(self._manifest_probes(output, required_kinds={network_kind})) + self._finish_probes( + "RDMA Shared networking", + probes, + ) + + +class LaunchKitIcmpConnectivityCheck(_LaunchKitCheck): + """Validate every source-bound ICMP matrix result.""" + + description: ClassVar[str] = "Check source-bound ICMP connectivity with Kubernetes Launch Kit" + + def run(self) -> None: + """Report all same-rail and expected-isolation ICMP observations.""" + output = self._step_output("validate") + if output is not None: + self._finish_probes("source-bound ICMP", self._matrix_probes(output, {"icmp"})) + + +class LaunchKitRdmaBandwidthCheck(_LaunchKitCheck): + """Validate every ib_write_bw matrix result and its Launch Kit threshold.""" + + description: ClassVar[str] = "Check RDMA bandwidth with Kubernetes Launch Kit" + + def run(self) -> None: + """Use the observed and minimum bandwidth emitted by Launch Kit.""" + output = self._step_output("validate") + if output is not None: + self._finish_probes("RDMA bandwidth", self._matrix_probes(output, {"ib_write_bw"})) + + +class LaunchKitGpuDirectRdmaCheck(_LaunchKitCheck): + """Validate every Launch Kit GPUDirect DMA-BUF bandwidth result.""" + + description: ClassVar[str] = "Check GPUDirect RDMA DMA-BUF bandwidth with Kubernetes Launch Kit" + + def run(self) -> None: + """Report endpoint GPU topology and Launch Kit's bandwidth verdict.""" + output = self._step_output("validate") + if output is None: + return + probes = self._matrix_probes(output, {"gpudirect_dmabuf"}) + if not probes: + if self._error: + return + pytest.skip( + "Launch Kit emitted no gpudirect_dmabuf results; validation.gpuDirect is disabled " + "or ib_write_bw is not selected" + ) + self._finish_probes("GPUDirect RDMA DMA-BUF bandwidth", probes) + + +class LaunchKitMultirailCheck(_LaunchKitCheck): + """Validate same-rail reachability and expected cross-rail isolation.""" + + description: ClassVar[str] = "Check multi-rail connectivity behavior with Kubernetes Launch Kit" + + def run(self) -> None: + """Require same-rail and cross-rail coverage when multiple rails exist.""" + output = self._step_output("validate") + if output is None: + return + profile = self._profile() + if profile is None: + return + multirail = profile.get("multirail") + if multirail not in {True, "true"}: + pytest.skip(f"selected Launch Kit profile is not multirail: {multirail!r}") + probes = self._matrix_probes(output, set(_CONNECTIVITY_FAMILIES)) + rails = {rail for probe in probes for rail in (probe["source_rail"], probe["destination_rail"])} + if len(rails) == 1 and "unknown-rail" not in rails: + pytest.skip(f"Launch Kit connectivity matrix contains only one rail: {next(iter(rails))}") + same_rail = [probe for probe in probes if probe["source_rail"] == probe["destination_rail"]] + cross_rail = [probe for probe in probes if probe not in same_rail] + probes.extend( + [ + {"name": "same-rail-coverage", "passed": bool(same_rail), "message": f"rows={len(same_rail)}"}, + {"name": "cross-rail-coverage", "passed": bool(cross_rail), "message": f"rows={len(cross_rail)}"}, + ] + ) + self._finish_probes("multi-rail behavior", probes) + + +def _artifact_paths(value: Any) -> list[Path]: + """Recursively collect paths from provider artifact mappings.""" + if isinstance(value, dict): + paths: list[Path] = [] + for child in value.values(): + paths.extend(_artifact_paths(child)) + return paths + if isinstance(value, list): + paths = [] + for child in value: + paths.extend(_artifact_paths(child)) + return paths + if isinstance(value, str) and value: + return [Path(value)] + return [] + + +def _evidence_path(value: str, output: dict[str, Any]) -> Path: + """Resolve a Launch Kit-emitted path against its command working directory.""" + path = Path(value) + if path.is_absolute(): + return path + working_directory = output.get("working_directory") + if isinstance(working_directory, str) and working_directory: + return Path(working_directory) / path + return path + + +class LaunchKitEvidenceCaptureCheck(_LaunchKitCheck): + """Validate raw command logs and the Launch Kit HTML report.""" + + description: ClassVar[str] = "Check Launch Kit evidence capture" + + def run(self) -> None: + """Require fresh evidence for every real workflow command.""" + outputs = { + "verify": self._configured_output("verify_output"), + "preflight": self._configured_output("preflight_output"), + "discover": self._configured_output("discover_output"), + "generate": self._configured_output("generate_output"), + "deploy": self._configured_output("deploy_output"), + "validate": self._step_output("validate") or {}, + } + prepare = self._configured_output("prepare_output") + if prepare: + outputs = {"prepare": prepare, **outputs} + probes: list[dict[str, Any]] = [] + for operation, output in outputs.items(): + paths = _artifact_paths(output.get("artifacts")) + existing = [path for path in paths if path.is_file()] + probes.append( + { + "name": f"{operation}-artifacts", + "passed": bool(paths) and len(existing) == len(paths), + "message": f"found {len(existing)}/{len(paths)} files", + } + ) + generated_paths = [ + _evidence_path(path, outputs["generate"]) + for document in self._documents(outputs["generate"]) + for path in _list(document.get("generatedFiles")) + if isinstance(path, str) + ] + probes.append( + { + "name": "generated-files", + "passed": bool(generated_paths) and all(path.is_file() for path in generated_paths), + "message": ( + f"found {sum(path.is_file() for path in generated_paths)}/{len(generated_paths)} generated files" + ), + } + ) + validate = outputs["validate"] + report_paths = [ + _evidence_path(document["reportPath"], validate) + for document in self._documents(validate) + if isinstance(document.get("reportPath"), str) + ] + probes.append( + { + "name": "launch-kit-html-report", + "passed": bool(report_paths) and all(path.is_file() for path in report_paths), + "message": ", ".join(str(path) for path in report_paths) or "no reportPath document", + } + ) + self._finish_probes("Launch Kit evidence capture", probes) diff --git a/isvtest/tests/k8s_launch_kit/test_checks.py b/isvtest/tests/k8s_launch_kit/test_checks.py new file mode 100644 index 000000000..1a1d233e7 --- /dev/null +++ b/isvtest/tests/k8s_launch_kit/test_checks.py @@ -0,0 +1,521 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for Kubernetes Launch Kit result interpretation.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from isvtest.core.validation import BaseValidation +from isvtest.validations.k8s_launch_kit.checks import ( + LaunchKitDeploymentHealthCheck, + LaunchKitEvidenceCaptureCheck, + LaunchKitGpuDirectRdmaCheck, + LaunchKitHostDeviceCheck, + LaunchKitIcmpConnectivityCheck, + LaunchKitInfiniBandCheck, + LaunchKitKubernetesPrerequisiteCheck, + LaunchKitMultirailCheck, + LaunchKitRdmaBandwidthCheck, + LaunchKitRdmaConnectivityCheck, + LaunchKitRdmaSharedCheck, + LaunchKitRoceCheck, + LaunchKitSecondaryNetworkCheck, + LaunchKitSriovReadinessCheck, + LaunchKitTopologyDiscoveryCheck, +) + +pytestmark = pytest.mark.unit + +_NETWORK_KIND = { + ("ethernet", "sriov"): "SriovNetwork", + ("infiniband", "sriov"): "SriovIBNetwork", + ("ethernet", "rdma_shared"): "MacvlanNetwork", + ("infiniband", "rdma_shared"): "IPoIBNetwork", + ("ethernet", "host_device"): "HostDeviceNetwork", + ("infiniband", "host_device"): "HostDeviceNetwork", +} + + +def _discover( + fabric: str = "ethernet", + deployment: str = "sriov", + *, + multirail: bool = True, +) -> dict[str, Any]: + """Build the provider envelope around a real discover-shaped document.""" + return { + "success": True, + "platform": "kubernetes", + "operation": "discover", + "exit_code": 0, + "documents": [ + { + "success": True, + "phase": "discover", + "profile": { + "fabric": fabric, + "deployment": deployment, + "multirail": "true" if multirail else "false", + }, + "deployed": False, + "messages": [], + } + ], + "artifacts": {}, + } + + +def _manifest(kind: str, *, state: str = "success") -> dict[str, Any]: + """Build one exported manifest validation row.""" + return { + "Kind": kind, + "APIVersion": "example.nvidia.com/v1", + "Name": f"mock-{kind.lower()}", + "Namespace": "default", + "State": state, + "Reason": "resource exists and is Ready" if state == "success" else "rollout has 1 unavailable pod", + "Found": True, + "Missing": False, + } + + +def _matrix_row(kind: int, *, same_rail: bool, passed: bool = True) -> dict[str, Any]: + """Build one exported connectivity row with source and destination detail.""" + destination_rail = "rail-0" if same_rail else "rail-1" + family = "icmp" if kind < 2 else "rping" if kind < 4 else "ib_write_bw" if kind < 6 else "gpudirect_dmabuf" + bandwidth_family = family in {"ib_write_bw", "gpudirect_dmabuf"} + return { + "Test": { + "Kind": kind, + "SrcNode": "worker-a", + "DstNode": "worker-b", + "SrcRail": "rail-0", + "DstRail": destination_rail, + "Expectation": "required" if same_rail else "forbidden", + **( + { + "SrcGPUIndex": 2, + "DstGPUIndex": 5, + "SrcGPUPCIAddress": "0000:41:00.0", + "DstGPUPCIAddress": "0000:71:00.0", + } + if family == "gpudirect_dmabuf" + else {} + ), + }, + "Family": family, + "OK": passed, + "ObservedOK": same_rail if passed else False, + "Expectation": "required" if same_rail else "forbidden", + "BandwidthGbps": 187.6 if bandwidth_family and passed else 42.5, + "MinBandwidthGbps": 100.0 if bandwidth_family else 0.0, + "Stderr": "" if passed else f"{family}: connection refused on rail-0", + **({"Error": f"{family} validation failed"} if not passed else {}), + } + + +def _validate( + fabric: str = "ethernet", + deployment: str = "sriov", + *, + failed_kind: str | None = None, + failed_family: str | None = None, + include_sriov_policy: bool = True, +) -> dict[str, Any]: + """Build a validate transport envelope with static and matrix documents.""" + network_kind = _NETWORK_KIND[(fabric, deployment)] + kinds = ["NicClusterPolicy", "NicNodePolicy", "IPPool"] + if deployment == "sriov" and include_sriov_policy: + kinds.append("SriovNetworkNodePolicy") + kinds.append(network_kind) + manifests = [_manifest(kind, state="error" if kind == failed_kind else "success") for kind in kinds] + rows: list[dict[str, Any]] = [] + for family, pair in { + "icmp": (0, 1), + "rping": (2, 3), + "ib_write_bw": (4, 5), + "gpudirect_dmabuf": (6, 7), + }.items(): + rows.append(_matrix_row(pair[0], same_rail=True, passed=family != failed_family)) + rows.append(_matrix_row(pair[1], same_rail=False)) + failed_manifests = sum(item["State"] != "success" for item in manifests) + failed_rows = sum(row["OK"] is not True for row in rows) + return { + "success": failed_rows == 0, + "platform": "kubernetes", + "operation": "validate", + "exit_code": 0 if failed_rows == 0 else 4, + "documents": [ + { + "versionCheck": { + "Skipped": False, + "SelectedRelease": "26.4", + "ExpectedVersion": "v26.4.1", + "DeployedRelease": {"ChartVersion": "26.4.1"}, + "Match": True, + }, + "manifests": manifests, + "presetDeviations": [], + "summary": { + "totalManifests": len(manifests), + "successManifests": len(manifests) - failed_manifests, + "errorManifests": failed_manifests, + "missingManifests": 0, + "success": failed_manifests == 0, + }, + }, + { + "connectivity": { + "DaemonSets": [ + { + "Ref": {"Namespace": "default", "Name": "l8k-network-test"}, + "Rollout": {"Desired": 2, "Ready": 2, "NotReady": 0}, + } + ], + "PingResults": rows, + "Summary": {"TotalTests": len(rows), "Failed": failed_rows}, + } + }, + ], + "artifacts": {}, + **({"error": "one or more connectivity rows failed"} if failed_rows else {}), + } + + +def _config( + output: dict[str, Any], + *, + discover: dict[str, Any] | None = None, + **extra: Any, +) -> dict[str, Any]: + """Bind a command envelope and optional earlier step outputs to a check.""" + config = {"step_output": output, **extra} + if discover is not None: + config["discover_output"] = json.dumps(discover) + return config + + +def test_kubernetes_prerequisite_reports_every_probe() -> None: + """A failed prerequisite retains successful checks and its remediation detail.""" + output = { + "success": False, + "platform": "kubernetes", + "operation": "kubernetes-preflight", + "checks": [ + {"name": "api-version", "passed": True, "message": "server v1.34.1"}, + {"name": "nodes", "passed": False, "message": "Forbidden: cannot list nodes"}, + {"name": "non-empty-cluster", "passed": False, "message": "cluster contains no nodes"}, + ], + } + + result = LaunchKitKubernetesPrerequisiteCheck(config=_config(output)).execute() + + assert result["passed"] is False + assert [probe["name"] for probe in result["subtests"]] == ["api-version", "nodes", "non-empty-cluster"] + assert "Forbidden: cannot list nodes" in result["error"] + + +def test_topology_discovery_uses_the_real_profile_document() -> None: + """Discovery succeeds only when l8k resolves both fabric and deployment.""" + result = LaunchKitTopologyDiscoveryCheck(config=_config(_discover())).execute() + + assert result["passed"] is True + assert [probe["name"] for probe in result["subtests"]] == ["discover-command", "resolved-profile"] + + +def test_deployment_health_reports_all_resources_before_failing() -> None: + """One unhealthy manifest is named without hiding later manifest rows.""" + output = _validate(failed_kind="SriovNetworkNodePolicy") + + result = LaunchKitDeploymentHealthCheck(config=_config(output)).execute() + + assert result["passed"] is False + names = [probe["name"] for probe in result["subtests"]] + assert "SriovNetworkNodePolicy/default/mock-sriovnetworknodepolicy" in names + assert names[-1] == "SriovNetwork/default/mock-sriovnetwork" + assert "rollout has 1 unavailable pod" in result["error"] + + +def test_deployment_health_honors_the_launch_kit_exit_verdict() -> None: + """A validate-level drift failure cannot be hidden by green manifest rows.""" + output = _validate() + output["success"] = False + output["exit_code"] = 4 + output["error"] = "l8k validate exited with code 4: component versions diverge" + + result = LaunchKitDeploymentHealthCheck(config=_config(output)).execute() + + assert result["passed"] is False + assert result["subtests"][0]["name"] == "launch-kit-validate-command" + assert "component versions diverge" in result["error"] + + +def test_deployment_health_allows_an_unconfigured_version_expectation() -> None: + """An optional Launch Kit version check is reported as skipped, not failed.""" + output = _validate() + output["documents"][0]["versionCheck"] = { + "Skipped": True, + "Reason": "cluster config has no selectedRelease", + } + + result = LaunchKitDeploymentHealthCheck(config=_config(output)).execute() + + assert result["passed"] is True + assert result["subtests"][1] == { + "name": "network-operator-version", + "passed": False, + "skipped": True, + "message": "cluster config has no selectedRelease", + "duration": None, + } + + +def test_sriov_readiness_requires_policy_and_network_kinds() -> None: + """A non-vacuous SR-IOV result requires both policy and attachment resources.""" + discover = _discover("infiniband", "sriov") + output = _validate("infiniband", "sriov", include_sriov_policy=False) + + result = LaunchKitSriovReadinessCheck(config=_config(output, discover=discover)).execute() + + assert result["passed"] is False + assert "kind-coverage/SriovNetworkNodePolicy" in result["error"] + assert any(probe["name"].startswith("SriovIBNetwork/") for probe in result["subtests"]) + + +@pytest.mark.parametrize( + ("check_class", "fabric", "deployment", "expected_kind"), + [ + (LaunchKitRoceCheck, "ethernet", "sriov", "SriovNetwork"), + (LaunchKitInfiniBandCheck, "infiniband", "sriov", "SriovIBNetwork"), + (LaunchKitHostDeviceCheck, "ethernet", "host_device", "HostDeviceNetwork"), + (LaunchKitRdmaSharedCheck, "infiniband", "rdma_shared", "IPoIBNetwork"), + ], +) +def test_profile_checks_require_the_applicable_network_kind( + check_class: type[BaseValidation], + fabric: str, + deployment: str, + expected_kind: str, +) -> None: + """Profile-specific checks select the exact resource implied by discover.""" + result = check_class( + config=_config(_validate(fabric, deployment), discover=_discover(fabric, deployment)) + ).execute() + + assert result["passed"] is True + names = [probe["name"] for probe in result["subtests"]] + assert names[0] == f"kind-coverage/{expected_kind}" + assert len(names) == len(set(names)) + + +def test_non_applicable_profile_is_skipped() -> None: + """An individually selectable check is skipped when the selected profile does not apply.""" + check = LaunchKitInfiniBandCheck(config=_config(_validate(), discover=_discover())) + + with pytest.raises(pytest.skip.Exception, match="not infiniband"): + check.execute() + + +@pytest.mark.parametrize( + ("check_class", "family"), + [ + (LaunchKitIcmpConnectivityCheck, "icmp"), + (LaunchKitRdmaConnectivityCheck, "rping"), + (LaunchKitRdmaBandwidthCheck, "ib_write_bw"), + (LaunchKitGpuDirectRdmaCheck, "gpudirect_dmabuf"), + ], +) +def test_connectivity_checks_create_source_bound_subtests( + check_class: type[BaseValidation], + family: str, +) -> None: + """Each matrix row becomes an independently named report item.""" + result = check_class(config=_config(_validate())).execute() + + assert result["passed"] is True + assert [probe["name"] for probe in result["subtests"]] == [ + f"{family}/worker-a->worker-b/rail-0->rail-0", + f"{family}/worker-a->worker-b/rail-0->rail-1", + ] + + +def test_connectivity_failure_preserves_stderr_and_bandwidth() -> None: + """A bandwidth failure includes endpoints, rails, observation, and threshold.""" + result = LaunchKitRdmaBandwidthCheck(config=_config(_validate(failed_family="ib_write_bw"))).execute() + + assert result["passed"] is False + assert len(result["subtests"]) == 2 + assert "bandwidthGbps=42.5" in result["error"] + assert "minimumGbps=100.0" in result["error"] + assert "connection refused on rail-0" in result["error"] + + +def test_gpudirect_failure_preserves_endpoint_gpu_and_bandwidth_evidence() -> None: + """A DMA-BUF failure identifies both endpoint GPUs and the failed threshold.""" + result = LaunchKitGpuDirectRdmaCheck(config=_config(_validate(failed_family="gpudirect_dmabuf"))).execute() + + assert result["passed"] is False + assert "gpuIndices=2->5" in result["error"] + assert "sourceGpuPci=0000:41:00.0" in result["error"] + assert "destinationGpuPci=0000:71:00.0" in result["error"] + assert "bandwidthGbps=42.5" in result["error"] + assert "minimumGbps=100.0" in result["error"] + assert "error=gpudirect_dmabuf validation failed" in result["error"] + + +def test_gpudirect_prefers_the_exported_family_contract() -> None: + """The stable Family field selects GPUDirect even if numeric kinds evolve.""" + output = _validate() + rows = output["documents"][1]["connectivity"]["PingResults"] + gpudirect_rows = [row for row in rows if row["Family"] == "gpudirect_dmabuf"] + for row in gpudirect_rows: + row["Test"]["Kind"] = 999 + output["documents"][1]["connectivity"]["PingResults"] = gpudirect_rows + + result = LaunchKitGpuDirectRdmaCheck(config=_config(output)).execute() + + assert result["passed"] is True + assert len(result["subtests"]) == 2 + + +def test_gpudirect_rejects_missing_endpoint_gpu_indices() -> None: + """A green row without explicit endpoint GPU topology is not accepted.""" + output = _validate() + row = next( + row for row in output["documents"][1]["connectivity"]["PingResults"] if row["Family"] == "gpudirect_dmabuf" + ) + row["Test"].pop("DstGPUIndex") + + result = LaunchKitGpuDirectRdmaCheck(config=_config(output)).execute() + + assert result["passed"] is False + assert "invalid or missing endpoint GPU index" in result["error"] + + +def test_gpudirect_is_skipped_when_launch_kit_does_not_emit_the_family() -> None: + """A discovery-disabled GPUDirect family is inapplicable, not failed.""" + output = _validate() + output["documents"][1]["connectivity"]["PingResults"] = [ + row for row in output["documents"][1]["connectivity"]["PingResults"] if row["Family"] != "gpudirect_dmabuf" + ] + check = LaunchKitGpuDirectRdmaCheck(config=_config(output)) + + with pytest.raises(pytest.skip.Exception, match=r"validation\.gpuDirect is disabled"): + check.execute() + + +def test_secondary_network_requires_ipam_network_and_ready_test_pods() -> None: + """Secondary-network coverage combines static resources with workload readiness.""" + discover = _discover("ethernet", "rdma_shared") + result = LaunchKitSecondaryNetworkCheck( + config=_config(_validate("ethernet", "rdma_shared"), discover=discover) + ).execute() + + assert result["passed"] is True + names = {probe["name"] for probe in result["subtests"]} + assert {"kind-coverage/IPPool", "kind-coverage/MacvlanNetwork", "DaemonSet/default/l8k-network-test"} <= names + + +@pytest.mark.parametrize( + "rollout", + [ + {}, + {"Desired": 2, "Ready": 2}, + {"Desired": 0, "Ready": 0, "NotReady": 0}, + {"Desired": True, "Ready": True, "NotReady": 0}, + ], +) +def test_secondary_network_rejects_incomplete_or_empty_daemonset_rollout(rollout: dict[str, Any]) -> None: + """Missing, invalid, or zero-sized rollout counts cannot pass vacuously.""" + discover = _discover("ethernet", "rdma_shared") + output = _validate("ethernet", "rdma_shared") + output["documents"][1]["connectivity"]["DaemonSets"][0]["Rollout"] = rollout + + result = LaunchKitSecondaryNetworkCheck(config=_config(output, discover=discover)).execute() + + assert result["passed"] is False + rollout_probe = next(probe for probe in result["subtests"] if probe["name"].startswith("DaemonSet/")) + assert rollout_probe["passed"] is False + + +def test_multirail_requires_same_and_cross_rail_coverage() -> None: + """Multi-rail validation distinguishes same-rail reachability from isolation rows.""" + result = LaunchKitMultirailCheck(config=_config(_validate(), discover=_discover())).execute() + + assert result["passed"] is True + assert result["subtests"][-2]["name"] == "same-rail-coverage" + assert result["subtests"][-1]["name"] == "cross-rail-coverage" + + +def test_multirail_is_skipped_when_matrix_contains_one_rail() -> None: + """A single-rail topology is inapplicable rather than a coverage failure.""" + output = _validate() + connectivity = output["documents"][1]["connectivity"] + connectivity["PingResults"] = [ + row for row in connectivity["PingResults"] if row["Test"]["SrcRail"] == row["Test"]["DstRail"] + ] + check = LaunchKitMultirailCheck(config=_config(output, discover=_discover())) + + with pytest.raises(pytest.skip.Exception, match="only one rail: rail-0"): + check.execute() + + assert check._subtest_results == [] + + +def test_structured_launch_kit_error_is_actionable() -> None: + """A failed l8k invocation surfaces its structured error when documents are absent.""" + output = { + "success": False, + "platform": "kubernetes", + "operation": "validate", + "documents": [{"error": {"message": "failed to create Kubernetes client"}}], + "error": "failed to create Kubernetes client; verify kubeconfig access", + } + + result = LaunchKitDeploymentHealthCheck(config=_config(output)).execute() + + assert result["passed"] is False + assert "failed to create Kubernetes client; verify kubeconfig access" in result["error"] + + +def test_evidence_check_requires_every_command_artifact_and_html_report(tmp_path: Path) -> None: + """Command evidence and the Launch Kit report are verified as real files.""" + outputs: dict[str, dict[str, Any]] = {} + for operation in ("prepare", "verify", "preflight", "discover", "generate", "deploy", "validate"): + artifact = tmp_path / f"{operation}.log" + artifact.write_text(f"{operation} evidence\n", encoding="utf-8") + outputs[operation] = { + "success": True, + "platform": "kubernetes", + "operation": "kubernetes-preflight" if operation == "preflight" else operation, + "working_directory": str(tmp_path), + "artifacts": {"stderr": str(artifact)}, + "documents": [], + } + generated = tmp_path / "generated" / "network-operator.yaml" + generated.parent.mkdir() + generated.write_text("kind: NicClusterPolicy\n", encoding="utf-8") + outputs["generate"]["documents"] = [{"generatedFiles": ["generated/network-operator.yaml"]}] + report = tmp_path / "k8s-launch-kit-validation-report.html" + report.write_text("passed\n", encoding="utf-8") + outputs["validate"]["documents"] = [{"reportPath": str(report)}] + + config = _config( + outputs["validate"], + prepare_output=json.dumps(outputs["prepare"]), + verify_output=json.dumps(outputs["verify"]), + preflight_output=json.dumps(outputs["preflight"]), + discover_output=json.dumps(outputs["discover"]), + generate_output=json.dumps(outputs["generate"]), + deploy_output=json.dumps(outputs["deploy"]), + ) + result = LaunchKitEvidenceCaptureCheck(config=config).execute() + + assert result["passed"] is True + assert len(result["subtests"]) == 9 diff --git a/isvtest/tests/test_catalog.py b/isvtest/tests/test_catalog.py index bbf33ff13..bbb289acb 100644 --- a/isvtest/tests/test_catalog.py +++ b/isvtest/tests/test_catalog.py @@ -52,6 +52,8 @@ def test_derives_suite_vocabulary_from_plain_suites(self) -> None: """Plain suite YAML files are listed separately from platform suites.""" suites = build_suite_vocabulary() assert "iam" in suites + assert "network_operator" in suites + assert "network_operator_use_cases" in suites assert "storage" in suites assert "kubernetes" not in suites assert "vm" not in suites @@ -109,6 +111,7 @@ def test_entries_have_suite_contract(self) -> None: assert isinstance(entry["requires"], list) if entry["capability"]: assert entry["requires"] == [] + assert "EastWestNetworkRoceSriovCheck" in names def test_extract_checks_supports_direct_dict_category_form(self, tmp_path) -> None: """Direct dict category wiring is included in catalog config scans.""" diff --git a/isvtest/tests/test_composite.py b/isvtest/tests/test_composite.py index 0d8c16f75..283fd1684 100644 --- a/isvtest/tests/test_composite.py +++ b/isvtest/tests/test_composite.py @@ -19,9 +19,11 @@ import pytest +import isvtest.core.composite as composite_module from isvtest.core.composite import CompositeCheck, composed_members, is_composite from isvtest.core.discovery import discover_all_tests from isvtest.core.resolution import parse_validations +from isvtest.core.validation import BaseValidation from isvtest.validations.generic import ( CrudOperationsCheck, FieldExistsCheck, @@ -144,6 +146,52 @@ def test_reports_each_member_as_a_subtest(self) -> None: ("FieldExistsCheck", True), ] + def test_forwards_member_subtests_with_member_qualified_names(self, monkeypatch: pytest.MonkeyPatch) -> None: + """A composite preserves member probes for terminal and JUnit diagnostics.""" + + class NestedCheck(BaseValidation): + def run(self) -> None: + self.report_subtest("probe-a", True, "probe passed") + self.report_subtest("probe-b", False, "not applicable", skipped=True) + self.set_passed("nested check passed") + + monkeypatch.setattr(composite_module, "get_validation_class", lambda name: NestedCheck) + composite = CompositeCheck(config=_config(["NestedCheck"])) + + result = composite.execute() + + assert result["passed"] is True + assert [(sub["name"], sub["skipped"]) for sub in result["subtests"]] == [ + ("NestedCheck", False), + ("NestedCheck/probe-a", False), + ("NestedCheck/probe-b", True), + ] + + def test_skipped_member_does_not_skip_or_fail_the_composite(self, monkeypatch: pytest.MonkeyPatch) -> None: + """An inapplicable member is skipped while later members still run.""" + + class SkippedCheck(BaseValidation): + def run(self) -> None: + pytest.skip("only one rail discovered") + + class PassingCheck(BaseValidation): + def run(self) -> None: + self.set_passed("later member passed") + + classes = {"SkippedCheck": SkippedCheck, "PassingCheck": PassingCheck} + monkeypatch.setattr(composite_module, "get_validation_class", classes.get) + composite = CompositeCheck(config=_config(["SkippedCheck", "PassingCheck"])) + + result = composite.execute() + + assert result["passed"] is True + assert [(sub["name"], sub["passed"], sub["skipped"]) for sub in result["subtests"]] == [ + ("SkippedCheck", False, True), + ("PassingCheck", True, False), + ] + assert "SkippedCheck: skipped - only one rail discovered" in result["output"] + assert "PassingCheck: later member passed" in result["output"] + def test_fails_naming_the_failing_member(self) -> None: """A failing member fails the composite and is named in the error.""" result = CompositeCheck( diff --git a/isvtest/tests/test_main.py b/isvtest/tests/test_main.py index efbe688c7..4b833835d 100644 --- a/isvtest/tests/test_main.py +++ b/isvtest/tests/test_main.py @@ -22,6 +22,7 @@ from isvtest.main import ( _entries_with_pytest_names, _resolved_entries_to_pytest_validations, + _result_to_resolved_entry, run_validations_via_pytest, ) @@ -102,6 +103,27 @@ def test_run_validations_via_pytest_updates_ready_entries() -> None: assert results[1].message == "NIM was not deployed" +def test_result_to_resolved_entry_preserves_subtest_summary() -> None: + """Subtest counts remain structured until the presentation layer renders them.""" + entry = _ready("StepSuccessCheck", "setup_checks", {"step_output": {"success": True}}) + + result = _result_to_resolved_entry( + entry, + { + "passed": True, + "skipped": False, + "message": "detailed validation output", + "subtest_summary": {"passed": 4, "failed": 0, "skipped": 2}, + }, + ) + + assert result.message == "detailed validation output" + assert result.subtest_summary.total == 6 + assert result.subtest_summary.passed == 4 + assert result.subtest_summary.failed == 0 + assert result.subtest_summary.skipped == 2 + + def test_run_validations_via_pytest_skips_structured_step_skips() -> None: """A step-level structured skip should skip all dependent validations.""" step_output = {"success": True, "skipped": True, "skip_reason": "No VPCs found at site"} diff --git a/isvtest/tests/test_subtests_junit.py b/isvtest/tests/test_subtests_junit.py index 3d83d20e6..767e333f6 100644 --- a/isvtest/tests/test_subtests_junit.py +++ b/isvtest/tests/test_subtests_junit.py @@ -23,9 +23,12 @@ from isvtest.testing.subtests import SubTestReport, _inject_subtests_into_junit -def _parent_junit(tmp_path: Path, parent_name: str) -> Path: +def _parent_junit(tmp_path: Path, parent_name: str, *, reported_tests: int = 1) -> Path: """Write a minimal pytest-style JUnit with a single parent testcase.""" - suite = ET.Element("testsuite", attrib={"name": "phase", "tests": "1", "skipped": "0", "failures": "0"}) + suite = ET.Element( + "testsuite", + attrib={"name": "phase", "tests": str(reported_tests), "skipped": "0", "failures": "0"}, + ) ET.SubElement(suite, "testcase", attrib={"name": parent_name, "classname": "", "time": "0.000"}) path = tmp_path / "junit.xml" ET.ElementTree(suite).write(path, encoding="utf-8", xml_declaration=True) @@ -92,3 +95,18 @@ def test_skipped_subtest_falls_back_when_longrepr_is_missing(tmp_path: Path) -> skipped = subtest_case.find("skipped") assert skipped is not None assert skipped.get("message") == "Subtest noisy-subtest skipped" + + +def test_injected_subtests_reconcile_precounted_junit_totals(tmp_path: Path) -> None: + """pytest's pre-counted subtest report must not be counted again after injection.""" + junit = _parent_junit(tmp_path, "ParentCheck", reported_tests=2) + report = _stub_report("::ParentCheck", "probe", failed=True, longrepr="probe failed") + + _inject_subtests_into_junit(junit, cast(list[SubTestReport], [report])) + + suite = ET.parse(junit).getroot() + assert len(suite.findall("testcase")) == 2 + assert suite.get("tests") == "2" + assert suite.get("failures") == "1" + assert suite.get("errors") == "0" + assert suite.get("skipped") == "0" diff --git a/isvtest/tests/test_validation.py b/isvtest/tests/test_validation.py index a42954d36..4685f8055 100644 --- a/isvtest/tests/test_validation.py +++ b/isvtest/tests/test_validation.py @@ -76,6 +76,16 @@ def run(self) -> None: self.set_failed("Test failed", "Error output") +class SubtestValidation(BaseValidation): + """Validation that reports passing and skipped probes.""" + + def run(self) -> None: + """Report two probe outcomes and pass the parent validation.""" + self.report_subtest("ready", True, "ready") + self.report_subtest("optional", False, "not applicable", skipped=True) + self.set_passed("All required probes passed") + + class ExceptionValidation(BaseValidation): """Validation that raises an exception.""" @@ -1778,6 +1788,16 @@ def test_passed_validation_captured(self) -> None: assert r["skipped"] is False assert r["passed"] is True + def test_subtest_summary_captured(self) -> None: + """Probe counts cross the pytest bridge without replacing the parent message.""" + subtests = MagicMock() + + run_validation_entry_point(SubtestValidation, {"_category": "test_cat"}, "SubtestValidation", subtests) + + result = _validation_results[0] + assert result["message"] == "All required probes passed" + assert result["subtest_summary"] == {"total": 2, "passed": 1, "failed": 0, "skipped": 1} + def test_failed_validation_captured(self) -> None: """Failed validations must appear with passed=False.""" config = {"_category": "test_cat"} diff --git a/scripts/requirements_source_to_md.py b/scripts/requirements_source_to_md.py index 3c5ca3923..3653cb5f6 100644 --- a/scripts/requirements_source_to_md.py +++ b/scripts/requirements_source_to_md.py @@ -17,7 +17,8 @@ """Render a source-requirements YAML to a publishable Markdown listing. YAML is the source of record (queryable, key-clean); Markdown is the published, -Google-Docs-friendly view. Handles both the `offtake` and `reference` sources. +Google-Docs-friendly view. Handles the registered offtake, reference, storage, +and project-PRD sources. Usage: python3 scripts/requirements_source_to_md.py docs/requirements/offtake-requirements.yaml @@ -38,6 +39,7 @@ REQ_DIR / "offtake-requirements.yaml", REQ_DIR / "software-reference-requirements.yaml", REQ_DIR / "storage-acceptance-requirements.yaml", + REQ_DIR / "network-operator-readiness-requirements.yaml", ] GENERATED_BANNER = "" @@ -149,7 +151,40 @@ def render_storage(doc: dict[str, Any], src_name: str) -> str: return "\n".join(out) + "\n" -RENDERERS = {"offtake": render_offtake, "reference": render_reference, "storage": render_storage} +def render_project_prd(doc: dict[str, Any], src_name: str) -> str: + """Render a project PRD listing, grouped by section.""" + out = [ + GENERATED_BANNER.format(src=src_name), + "", + f"# {doc.get('title', 'Project Requirements')}", + "", + f"> Structured source of record: `{src_name}` (version {doc.get('version', 'n/a')}).", + f"> Owner: {doc.get('owner', 'not specified')}.", + "> Edit the YAML, not this file.", + "", + ] + section = None + for requirement in doc.get("requirements", []): + if requirement.get("section") != section: + section = requirement.get("section") + heading(out, f"## {section}") + out += [ + "| Req ID | Requirement Area | Description | Status |", + "| :----- | :--------------- | :---------- | :----- |", + ] + out.append( + f"| {cell(requirement.get('req_id'))} | {cell(requirement.get('area'))} " + f"| {cell(requirement.get('description'))} | {cell(requirement.get('status', 'active'))} |" + ) + return "\n".join(out) + "\n" + + +RENDERERS = { + "offtake": render_offtake, + "project-prd": render_project_prd, + "reference": render_reference, + "storage": render_storage, +} def render(path: Path) -> None: @@ -158,9 +193,14 @@ def render(path: Path) -> None: if not isinstance(doc, dict): raise ValueError(f"{path} must contain a single mapping document") source = doc.get("source") - renderer = RENDERERS.get(source) + if not isinstance(source, str): + raise ValueError(f"{path} must declare a string source") + document_format = doc.get("format", source) + if not isinstance(document_format, str): + raise ValueError(f"{path} format must be a string") + renderer = RENDERERS.get(document_format) if renderer is None: - raise ValueError(f"{path} has unsupported source {source!r} (expected one of {sorted(RENDERERS)})") + raise ValueError(f"{path} has unsupported format {document_format!r} (expected one of {sorted(RENDERERS)})") out_path = path.with_suffix(".md") out_path.write_text(renderer(doc, path.name), encoding="utf-8") print(f"Wrote {out_path}") diff --git a/scripts/test_plan_coverage.py b/scripts/test_plan_coverage.py index 33d720ff4..d46fd8293 100644 --- a/scripts/test_plan_coverage.py +++ b/scripts/test_plan_coverage.py @@ -131,7 +131,7 @@ def config_test_id_map(suites_dir: Path = SUITES_DIR) -> dict[str, list[str]]: both bare_metal and vm), so values still aggregate to a set across configs. """ out: dict[str, set[str]] = defaultdict(set) - for path in sorted(suites_dir.glob("*.yaml")): + for path in sorted(suites_dir.rglob("*.yaml")): for name, params in iter_config_checks(path): tid = params.get("test_id") if isinstance(tid, str) and tid: @@ -148,7 +148,7 @@ def config_label_map(suites_dir: Path = SUITES_DIR) -> dict[str, list[str]]: union of its labels. """ out: dict[str, set[str]] = defaultdict(set) - for path in sorted(suites_dir.glob("*.yaml")): + for path in sorted(suites_dir.rglob("*.yaml")): for name, params in iter_config_checks(path): out[name].update(_normalize_labels(params.get("labels"))) return {name: sorted(labels) for name, labels in out.items()} @@ -166,7 +166,7 @@ def _normalize_labels(value: Any) -> list[str]: def config_test_label_instances(suites_dir: Path = SUITES_DIR) -> list[tuple[str, str, str, list[str]]]: """Return ``(source, check_name, test_id, labels)`` for each mapped suite check.""" instances: list[tuple[str, str, str, list[str]]] = [] - for path in sorted(suites_dir.glob("*.yaml")): + for path in sorted(suites_dir.rglob("*.yaml")): try: source = path.relative_to(REPO_ROOT).as_posix() except ValueError: diff --git a/scripts/tests/test_requirements_source_to_md.py b/scripts/tests/test_requirements_source_to_md.py new file mode 100644 index 000000000..d5d8f7093 --- /dev/null +++ b/scripts/tests/test_requirements_source_to_md.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the structured-requirements Markdown renderer.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest +import yaml + +_SCRIPT = Path(__file__).resolve().parent.parent / "requirements_source_to_md.py" +_spec = importlib.util.spec_from_file_location("requirements_source_to_md", _SCRIPT) +assert _spec and _spec.loader +requirements_source_to_md = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(requirements_source_to_md) + + +def test_project_prd_format_renders_a_uniquely_named_source() -> None: + """A project PRD keeps a unique trace source while sharing a generic renderer.""" + source_path = requirements_source_to_md.REQ_DIR / "network-operator-readiness-requirements.yaml" + document = yaml.safe_load(source_path.read_text(encoding="utf-8")) + + rendered = requirements_source_to_md.RENDERERS[document["format"]](document, source_path.name) + + assert source_path in requirements_source_to_md.DEFAULT_SOURCES + assert document["source"] == "network-operator-prd" + assert document["format"] == "project-prd" + assert "# Enterprise RA Network Operator Self-Validation Integration PRD" in rendered + assert "> Owner: NVIDIA Network Operator team." in rendered + assert "## Network Validation" in rendered + assert "| ENT-REQ-008 | GPUDirect RDMA |" in rendered + + +def test_render_rejects_a_non_string_source(tmp_path: Path) -> None: + """A malformed source fails clearly before renderer dispatch.""" + source_path = tmp_path / "malformed-requirements.yaml" + source_path.write_text("source: null\nrequirements: []\n", encoding="utf-8") + + with pytest.raises(ValueError, match="must declare a string source"): + requirements_source_to_md.render(source_path) + + +def test_render_rejects_an_unknown_document_format(tmp_path: Path) -> None: + """A unique source may select only a documented reusable renderer format.""" + source_path = tmp_path / "malformed-requirements.yaml" + source_path.write_text( + "source: team-prd\nformat: not-a-renderer\nrequirements: []\n", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="unsupported format 'not-a-renderer'"): + requirements_source_to_md.render(source_path) diff --git a/scripts/tests/test_validate_suite_wiring.py b/scripts/tests/test_validate_suite_wiring.py index ac6ef32b5..ee4536070 100644 --- a/scripts/tests/test_validate_suite_wiring.py +++ b/scripts/tests/test_validate_suite_wiring.py @@ -95,6 +95,19 @@ def test_wiring_errors_reports_yaml_parse_failures(tmp_path: Path) -> None: assert "failed to read/parse" in errors[0] +def test_wiring_errors_scans_nested_suite_directories(tmp_path: Path) -> None: + """Domain-organized suites receive the same metadata guardrails as root suites.""" + nested = tmp_path / "launch-kit" + nested.mkdir() + (nested / "network-operator.yaml").write_text( + "tests:\n validations:\n sample:\n checks:\n MissingMetadata: {}\n" + ) + + errors = validate_suite_wiring.wiring_errors(tmp_path) + + assert any("launch-kit/network-operator.yaml" in error and "MissingMetadata" in error for error in errors) + + def test_find_check_line_numbers_supports_list_form() -> None: """List-form wiring reports each repeated check at its own line.""" lines = """ diff --git a/scripts/validate_suite_wiring.py b/scripts/validate_suite_wiring.py index a07cc007e..2fdfa5a4b 100644 --- a/scripts/validate_suite_wiring.py +++ b/scripts/validate_suite_wiring.py @@ -16,7 +16,7 @@ """Validate suite identity and check resolution in canonical and provider YAML. -Suite configs under ``isvctl/configs/suites/`` are the source of truth for +Suite configs recursively under ``isvctl/configs/suites/`` are the source of truth for validation metadata on this branch. Each wired check must declare: * ``test_id`` - a plan id from ``docs/test-plan.yaml``, or ``"N/A"`` when the @@ -190,7 +190,7 @@ def wiring_errors(suites_dir: Path = SUITES_DIR) -> list[str]: # Read and parse each suite once; both the dead-requirement pre-pass and the # per-check loop below work off these parsed documents. parsed: list[tuple[Path, list[str], dict[str, Any]]] = [] - for path in sorted(suites_dir.glob("*.yaml")): + for path in sorted(suites_dir.rglob("*.yaml")): try: text = path.read_text() parsed.append((path, text.splitlines(), yaml.safe_load(text) or {}))