diff --git a/isvctl/configs/providers/gb300/config/bare_metal.yaml b/isvctl/configs/providers/gb300/config/bare_metal.yaml new file mode 100644 index 000000000..9615c8b38 --- /dev/null +++ b/isvctl/configs/providers/gb300/config/bare_metal.yaml @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Direct GB300 NVSwitch firmware validation. +# +# This provider runs on a GB300 BCM head where Bright Cluster Manager and +# cm-nvfwupd are installed. It reads existing NVSwitch inventory and executes +# only `nvfwupd show_version -j`; it never performs a firmware update. +# +# Prerequisites: +# - passwordless sudo for the read-only cmsh and nvfwupd commands +# - GB300_RACK set to a rack prefix (for example, a05-p01), or +# GB300_SWITCH_HOSTS set to a comma-separated list of NVSwitch hostnames +# - GB300_SWITCH_LIMIT optionally controls the number inspected (default: 1) +# +# Usage: +# GB300_RACK=a05-p01 \ +# uv run isvctl test run \ +# -f isvctl/configs/providers/gb300/config/bare_metal.yaml \ +# -- -k NvSwitchFirmwareCheck + +import: + - ../../../suites/bare_metal.yaml + +version: "1.0" + +commands: + bare_metal: + phases: ["test"] + steps: + - name: query_switch_firmware + phase: test + continue_on_failure: true + command: "python ../scripts/breakfix/query_switch_firmware.py" + timeout: 180 + +tests: + cluster_name: "gb300-direct-firmware-validation" + description: "Inspect GB300 NVSwitch tray firmware directly with nvfwupd" + + settings: + region: "" + instance_type: "" + teardown_flag: "" diff --git a/isvctl/configs/providers/gb300/scripts/breakfix/query_switch_firmware.py b/isvctl/configs/providers/gb300/scripts/breakfix/query_switch_firmware.py new file mode 100644 index 000000000..57f27b4a5 --- /dev/null +++ b/isvctl/configs/providers/gb300/scripts/breakfix/query_switch_firmware.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Inspect GB300 NVSwitch tray firmware with read-only ``nvfwupd`` (BFX03-02). + +The script is intended to run on a GB300 BCM head. It discovers dedicated +``nvswitch`` devices through read-only ``cmsh`` inventory and invokes only +``nvfwupd show_version -j`` against the selected tray BMCs. Credentials remain +inside the privileged subprocess and are never included in the JSON result. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +from collections.abc import Sequence +from typing import Any + +_HOST_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +_NVFWUPD_PATH = "/cm/local/apps/cm-nvfwupd/nvfwupd" +_SERVER_TYPE = "gb200switch" +_MODULE_SETUP = """\ +export MODULEPATH=/cm/local/modulefiles:/cm/shared/modulefiles +source /cm/local/apps/environment-modules/current/init/bash +module load shared cmsh cm-nvfwupd >/dev/null 2>&1 +""" +_DISCOVER_SCRIPT = ( + _MODULE_SETUP + + """\ +exec cmsh-lazy-load -c 'device; list -t switch -f hostname:64,category:32,status:32' +""" +) +_QUERY_SCRIPT = ( + _MODULE_SETUP + + f"""\ +switch_host="$1" +bmc_creds=$(cmsh-lazy-load -c "device; use $switch_host; get ip; accesssettings; get username; get password") +set -- $bmc_creds +if [ "$#" -lt 3 ]; then + exit 20 +fi +bmc_ip="$1" +bmc_user="$2" +bmc_pass="$3" +exec {_NVFWUPD_PATH} \\ + -t ip="$bmc_ip" user="$bmc_user" password="$bmc_pass" servertype={_SERVER_TYPE} \\ + show_version -j +""" +) + + +class InspectionError(RuntimeError): + """Raised when direct NVSwitch firmware evidence cannot be obtained.""" + + +def _emit(result: dict[str, Any]) -> int: + """Print the provider-neutral JSON result and return its exit status.""" + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 if result.get("success") else 1 + + +def _run_privileged(script: str, *args: str, timeout: int) -> subprocess.CompletedProcess[str]: + """Run one fixed read-only helper as root without exposing BMC credentials.""" + return subprocess.run( + ["sudo", "-n", "bash", "-s", "--", *args], + input=script, + text=True, + capture_output=True, + check=False, + timeout=timeout, + ) + + +def _validate_host(host: str) -> str: + """Reject values that cannot be safely used as BCM device names.""" + value = host.strip() + if not _HOST_RE.fullmatch(value): + raise InspectionError("invalid NVSwitch hostname") + return value + + +def _discover_switches(rack: str) -> list[str]: + """Return dedicated NVSwitch hostnames from BCM's read-only inventory.""" + completed = _run_privileged(_DISCOVER_SCRIPT, timeout=30) + if completed.returncode != 0: + raise InspectionError("unable to read NVSwitch inventory from BCM") + + rack_prefix = rack.strip().lower() + switches: list[str] = [] + for line in completed.stdout.splitlines(): + fields = line.split() + if len(fields) < 2 or "nvswitch" not in fields[1].lower(): + continue + host = _validate_host(fields[0]) + if rack_prefix and not host.lower().startswith(f"{rack_prefix}-"): + continue + if host not in switches: + switches.append(host) + return switches + + +def _query_tray(host: str) -> dict[str, Any]: + """Run ``nvfwupd show_version -j`` against one dedicated switch tray.""" + completed = _run_privileged(_QUERY_SCRIPT, _validate_host(host), timeout=120) + if completed.returncode != 0: + raise InspectionError("nvfwupd show_version failed") + try: + inventory = json.loads(completed.stdout) + except json.JSONDecodeError as exc: + raise InspectionError("nvfwupd returned invalid JSON") from exc + if not isinstance(inventory, dict) or str(inventory.get("Error Code", 0)) != "0": + raise InspectionError("nvfwupd returned a firmware inventory error") + + devices = inventory.get("Firmware Devices") + if not isinstance(devices, list) or not devices: + raise InspectionError("nvfwupd returned no firmware devices") + + versions: dict[str, str] = {} + for device in devices: + if not isinstance(device, dict): + raise InspectionError("nvfwupd returned a malformed firmware device") + name = str(device.get("AP Name") or "").strip() + version = str(device.get("Sys Version") or "").strip() + if not name or not version: + raise InspectionError("nvfwupd returned an incomplete firmware device") + versions[name] = version + + primary_version = versions.get("BMC") or versions.get("ASIC") or next(iter(versions.values())) + return { + "tray_id": host, + "firmware_version": primary_version, + "firmware_versions": versions, + } + + +def _configured_switches(cli_switches: Sequence[str]) -> list[str]: + """Combine repeatable CLI targets with the comma-separated environment value.""" + configured = list(cli_switches) + configured.extend(os.environ.get("GB300_SWITCH_HOSTS", "").split(",")) + switches: list[str] = [] + for candidate in configured: + if not candidate.strip(): + continue + host = _validate_host(candidate) + if host not in switches: + switches.append(host) + return switches + + +def _positive_int(value: str) -> int: + """Parse a strictly positive tray limit for argparse.""" + parsed = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("must be at least 1") + return parsed + + +def main() -> int: + """Inspect selected GB300 NVSwitch trays and emit BFX03-02 evidence.""" + parser = argparse.ArgumentParser(description="Inspect GB300 NVSwitch tray firmware with nvfwupd") + parser.add_argument("--rack", default=os.environ.get("GB300_RACK", ""), help="BCM rack hostname prefix") + parser.add_argument("--switch-host", action="append", default=[], help="NVSwitch hostname; repeat as needed") + parser.add_argument( + "--limit", + type=_positive_int, + default=os.environ.get("GB300_SWITCH_LIMIT", "1"), + help="Maximum number of trays to inspect (default: 1)", + ) + args = parser.parse_args() + + result: dict[str, Any] = { + "success": False, + "platform": "gb300", + "source": "nvfwupd show_version -j", + "trays": [], + } + try: + switches = _configured_switches(args.switch_host) + if not switches: + switches = _discover_switches(args.rack) + if not switches: + raise InspectionError("no NVSwitch trays discovered on the GB300 system") + result["trays"] = [_query_tray(host) for host in switches[: args.limit]] + except (InspectionError, subprocess.TimeoutExpired) as exc: + result["error_type"] = "firmware_inspection" + result["error"] = str(exc) if isinstance(exc, InspectionError) else "NVSwitch firmware inspection timed out" + return _emit(result) + + result["success"] = True + return _emit(result) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/tests/providers/gb300/test_gb300_provider.py b/isvctl/tests/providers/gb300/test_gb300_provider.py new file mode 100644 index 000000000..e34a8410c --- /dev/null +++ b/isvctl/tests/providers/gb300/test_gb300_provider.py @@ -0,0 +1,208 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for direct GB300 NVSwitch tray firmware inspection.""" + +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest +import yaml +from isvtest.validations.breakfix import NvSwitchFirmwareCheck + +ISVCTL_ROOT = Path(__file__).resolve().parents[3] +GB300_ROOT = ISVCTL_ROOT / "configs" / "providers" / "gb300" +SCRIPT_PATH = GB300_ROOT / "scripts" / "breakfix" / "query_switch_firmware.py" +CONFIG_PATH = GB300_ROOT / "config" / "bare_metal.yaml" + +LIVE_SHAPED_INVENTORY = { + "Error Code": 0, + "Firmware Devices": [ + {"AP Name": "ASIC", "Sys Version": "35.2014.4784"}, + {"AP Name": "BIOS", "Sys Version": "0ACTV_00.01.020"}, + {"AP Name": "BMC", "Sys Version": "88.0002.1961"}, + {"AP Name": "CPLD1", "Sys Version": "CPLD000420_REV0300"}, + {"AP Name": "CPLD2", "Sys Version": "CPLD000419_REV0301"}, + {"AP Name": "CPLD3", "Sys Version": "CPLD000418_REV0200"}, + {"AP Name": "EROT", "Sys Version": "01.04.0031.0000_n04"}, + {"AP Name": "FPGA", "Sys Version": "0.24"}, + {"AP Name": "SSD", "Sys Version": "CE00A450"}, + ], +} + + +def _load_script() -> ModuleType: + """Load the provider script as an isolated module.""" + spec = importlib.util.spec_from_file_location("test_gb300_switch_firmware", SCRIPT_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _completed(stdout: str, *, returncode: int = 0, stderr: str = "") -> subprocess.CompletedProcess[str]: + """Build a subprocess result for privileged-helper mocks.""" + return subprocess.CompletedProcess([], returncode, stdout=stdout, stderr=stderr) + + +def test_config_wires_only_read_only_firmware_query() -> None: + """The GB300 provider binds BFX03-02 to the direct query script.""" + config = yaml.safe_load(CONFIG_PATH.read_text(encoding="utf-8")) + steps = config["commands"]["bare_metal"]["steps"] + + assert steps == [ + { + "name": "query_switch_firmware", + "phase": "test", + "continue_on_failure": True, + "command": "python ../scripts/breakfix/query_switch_firmware.py", + "timeout": 180, + } + ] + + +def test_discovers_dedicated_nvswitches_in_requested_rack(monkeypatch: pytest.MonkeyPatch) -> None: + """BCM compute and other-rack devices never become firmware targets.""" + module = _load_script() + inventory = """\ +a05-p01-dgx-01-c01 compute [ UP ] +a05-p01-nvsw-01 nvswitch [ UP ] +a05-p01-nvsw-02 nvswitch [ UP ] +b04-p01-nvsw-01 nvswitch [ UP ] +""" + monkeypatch.setattr(module, "_run_privileged", lambda *args, **kwargs: _completed(inventory)) + + assert module._discover_switches("a05-p01") == ["a05-p01-nvsw-01", "a05-p01-nvsw-02"] + + +def test_queries_live_shaped_nvfwupd_inventory_read_only(monkeypatch: pytest.MonkeyPatch) -> None: + """The observed GB300 switch shape produces complete BFX03-02 evidence.""" + module = _load_script() + observed: dict[str, Any] = {} + + def fake_run(script: str, *args: str, timeout: int) -> subprocess.CompletedProcess[str]: + observed.update(script=script, args=args, timeout=timeout) + return _completed(json.dumps(LIVE_SHAPED_INVENTORY)) + + monkeypatch.setattr(module, "_run_privileged", fake_run) + + tray = module._query_tray("a05-p01-nvsw-01") + + assert tray["tray_id"] == "a05-p01-nvsw-01" + assert tray["firmware_version"] == "88.0002.1961" + assert tray["firmware_versions"]["ASIC"] == "35.2014.4784" + assert tray["firmware_versions"]["CPLD3"] == "CPLD000418_REV0200" + assert observed["args"] == ("a05-p01-nvsw-01",) + assert observed["timeout"] == 120 + assert "show_version -j" in observed["script"] + assert all(token not in observed["script"] for token in ("update_fw", "update_firmware", "activate_fw")) + + +def test_main_passes_with_direct_gb300_tray_evidence( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """One inspected GB300 NVSwitch tray satisfies the suite's minimum.""" + module = _load_script() + monkeypatch.delenv("GB300_SWITCH_HOSTS", raising=False) + monkeypatch.setattr(module, "_discover_switches", lambda rack: ["a05-p01-nvsw-01"]) + monkeypatch.setattr( + module, + "_query_tray", + lambda host: { + "tray_id": host, + "firmware_version": "88.0002.1961", + "firmware_versions": {"BMC": "88.0002.1961", "ASIC": "35.2014.4784"}, + }, + ) + monkeypatch.setattr(sys, "argv", ["query_switch_firmware.py", "--rack", "a05-p01"]) + + assert module.main() == 0 + + payload = json.loads(capsys.readouterr().out) + assert payload == { + "platform": "gb300", + "source": "nvfwupd show_version -j", + "success": True, + "trays": [ + { + "tray_id": "a05-p01-nvsw-01", + "firmware_version": "88.0002.1961", + "firmware_versions": {"BMC": "88.0002.1961", "ASIC": "35.2014.4784"}, + } + ], + } + + check = NvSwitchFirmwareCheck(config={"step_output": payload}) + check.run() + assert check.passed + assert "1 NV switch tray(s)" in check.message + + +def test_assumed_gb300_without_switch_inventory_fails( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """Applicable GB300 hardware cannot skip or pass without tray evidence.""" + module = _load_script() + monkeypatch.delenv("GB300_SWITCH_HOSTS", raising=False) + monkeypatch.setattr(module, "_discover_switches", lambda rack: []) + monkeypatch.setattr(sys, "argv", ["query_switch_firmware.py", "--rack", "a05-p01"]) + + assert module.main() == 1 + + payload = json.loads(capsys.readouterr().out) + assert payload["success"] is False + assert payload["trays"] == [] + assert payload["error_type"] == "firmware_inspection" + assert "no NVSwitch trays discovered" in payload["error"] + assert "skipped" not in payload + + +def test_incomplete_nvfwupd_device_is_rejected(monkeypatch: pytest.MonkeyPatch) -> None: + """A reported firmware component without a version is not a PASS.""" + module = _load_script() + incomplete = { + "Error Code": 0, + "Firmware Devices": [ + {"AP Name": "BMC", "Sys Version": "88.0002.1961"}, + {"AP Name": "ASIC", "Sys Version": ""}, + ], + } + monkeypatch.setattr( + module, + "_run_privileged", + lambda *args, **kwargs: _completed(json.dumps(incomplete)), + ) + + with pytest.raises(module.InspectionError, match="incomplete firmware device"): + module._query_tray("a05-p01-nvsw-01") + + +def test_privileged_failure_does_not_leak_stderr( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """BMC credentials or raw command diagnostics never enter provider JSON.""" + module = _load_script() + monkeypatch.delenv("GB300_SWITCH_HOSTS", raising=False) + monkeypatch.setattr(module, "_discover_switches", lambda rack: ["a05-p01-nvsw-01"]) + monkeypatch.setattr( + module, + "_run_privileged", + lambda *args, **kwargs: _completed("", returncode=1, stderr="password=do-not-print"), + ) + monkeypatch.setattr(sys, "argv", ["query_switch_firmware.py", "--rack", "a05-p01"]) + + assert module.main() == 1 + + output = capsys.readouterr().out + assert "do-not-print" not in output + assert "password=" not in output diff --git a/isvtest/tests/test_breakfix.py b/isvtest/tests/test_breakfix.py index 485c8cbcb..daf254604 100644 --- a/isvtest/tests/test_breakfix.py +++ b/isvtest/tests/test_breakfix.py @@ -17,6 +17,7 @@ HostReplacementCheck, MaintenanceEventsCheck, NodeHealthAgentCheck, + NvSwitchFirmwareCheck, PlannedMaintenanceNotificationCheck, RepairHistoryCheck, RetirementNoticesCheck, @@ -146,6 +147,42 @@ def test_node_maintenance_reports_mode(self) -> None: assert "maintenance_mode=hw" in check.message +class TestNvSwitchFirmwareCheck: + """Cover BFX03-02 firmware evidence for every returned switch tray.""" + + def test_fails_when_no_trays_are_returned(self) -> None: + """An applicable GB300 system cannot pass without NVSwitch tray evidence.""" + check = _run(NvSwitchFirmwareCheck, {"success": True, "trays": []}) + assert not check.passed + assert "Expected at least 1 NV switch tray(s), got 0" in check.message + + def test_passes_when_every_tray_has_firmware(self) -> None: + """Every discovered NVSwitch tray must report a non-empty version.""" + step_output = { + "success": True, + "trays": [ + {"tray_id": "switch-1", "firmware_version": "1.2.3"}, + {"tray_id": "switch-2", "firmware_version": "2.0.0"}, + ], + } + check = _run(NvSwitchFirmwareCheck, step_output) + assert check.passed + assert "2 NV switch tray(s)" in check.message + + def test_fails_when_any_tray_has_no_firmware(self) -> None: + """One missing version keeps partial inventory from passing BFX03-02.""" + step_output = { + "success": True, + "trays": [ + {"tray_id": "switch-1", "firmware_version": "1.2.3"}, + {"tray_id": "switch-2", "firmware_version": ""}, + ], + } + check = _run(NvSwitchFirmwareCheck, step_output) + assert not check.passed + assert "1 switch tray(s) missing firmware_version" in check.message + + class TestNodeHealthAgentCheck: """Cover the BFX04-01 GPUd/Sentinel health-agent check."""