Add NICo and Kubernetes node maintenance validation

docs/guides/remote-deployment.md

@@ -30,6 +30,8 @@ Set these on the local machine. The upload variables are read here, since
 | `ISV_CLIENT_SECRET` | Required for result upload to ISV Lab Service | locally |
 | `NGC_API_KEY` | Required for NIM model benchmarks | forwarded |
 | `ISVTEST_INCLUDE_UNRELEASED` | Include checks not yet in `released_tests.json` | forwarded |
+| `ISVTEST_BREAKFIX_ALLOW_MUTATION` | Explicitly allow a mutating break-fix validation | forwarded |
+| `ISVTEST_BREAKFIX_NODE` | Exact Kubernetes node selected for break-fix validation | forwarded |
 
 Anything else the tests need has to reach the target another way - a config file
 under `isvctl/` travels in the deployment archive, so `-f` overrides are the
@@ -85,6 +87,16 @@ Pass extra pytest arguments after `--`:
 uv run isvctl deploy run <target-ip> -f isvctl/configs/suites/slurm.yaml -- -v -s -k "test_name"
 ```
 
+### Node Maintenance Validation
+
+The BFX01-02 reference uses the NVIDIA Maintenance Operator API on the
+target's active Kubernetes context. It never selects a node implicitly and
+drains only its uniquely labelled probe workload.
+
+```bash
+ISVTEST_INCLUDE_UNRELEASED=1 ISVTEST_BREAKFIX_ALLOW_MUTATION=1 ISVTEST_BREAKFIX_NODE=<dedicated-test-node> uv run isvctl deploy run <target-ip> -f isvctl/configs/providers/kubernetes-node-maintenance.yaml -- -v -s -k ReturnNodeMaintenanceCheck
+```
+
 ### With ISV Lab Service Integration
 
 Upload results to the ISV Lab Service:

isvctl/configs/providers/kubernetes-node-maintenance.yaml

@@ -0,0 +1,40 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Explicitly mutating Kubernetes BFX01-02 configuration.
+#
+# The active Kubernetes cluster must provide the NVIDIA Maintenance Operator
+# NodeMaintenance CRD. This run creates a uniquely owned probe Deployment,
+# requests maintenance for one explicit node, verifies the operator cordons the
+# node and evicts only that probe, then deletes the request and verifies the
+# node and workload recover.
+#
+#   export ISVTEST_INCLUDE_UNRELEASED=1
+#   export ISVTEST_BREAKFIX_ALLOW_MUTATION=1
+#   export ISVTEST_BREAKFIX_NODE=<dedicated-test-node>
+#   uv run isvctl test run \
+#     -f isvctl/configs/providers/kubernetes-node-maintenance.yaml \
+#     --label breakfix -- -v -s -k ReturnNodeMaintenanceCheck
+
+import: ../suites/bare_metal.yaml
+
+version: "1.0"
+
+commands:
+  bare_metal:
+    phases: ["test"]
+    steps:
+      - name: return_node_maintenance
+        phase: test
+        command: "python shared/breakfix/return_node_maintenance.py"
+        args:
+          - "--node={{ env.ISVTEST_BREAKFIX_NODE | default('', true) }}"
+        timeout: 1200
+        requires_available_validations:
+          - ReturnNodeMaintenanceCheck
+
+tests:
+  description: "Bare-metal BFX01-02 validation through the Kubernetes Maintenance Operator API"
+
+  settings:
+    show_skipped_tests: true

isvctl/configs/providers/my-isv/scripts/breakfix/return_node_maintenance.py

@@ -30,7 +30,8 @@ def main() -> int:
             "requested": True,
             "accepted": True,
             "machine_id": machine_id,
-            "maintenance_mode": True,
+            "maintenance_mode": "Maintenance",
+            "restored": True,
         },
     )
 

isvctl/configs/providers/nico/config/bare_metal.yaml

@@ -85,6 +85,9 @@
 #   - Expected machines pre-registered; machines ingested and DPUs initialized
 #   - NICO_ORGANIZATION and NICO_SITE_ID environment variables set
 #   - Optional NICO_INSTANCE_ID narrows inventory checks to a known instance
+#   - BFX01-02 is staging-only and mutating. It structured-skips unless both
+#     NICO_BREAKFIX_MACHINE_ID names a dedicated fixture and
+#     NICO_BREAKFIX_ALLOW_MUTATION=1 explicitly opts in.
 #   - ssh-keygen on the host (used by query_key_access to mint the throwaway key)
 #
 # Usage:
@@ -439,17 +442,19 @@ commands:
       - name: return_node_maintenance
         phase: test
         continue_on_failure: true
-        command: "python ../scripts/breakfix/gap_stub.py"
+        command: "python ../scripts/breakfix/return_node_maintenance.py"
         args:
           - "--org"
           - "{{org}}"
           - "--site-id"
           - "{{site_id}}"
           - "--api-base"
           - "{{nico_api_base}}"
-          - "--gap"
-          - "BFX01-02"
-        timeout: 600
+          - "--machine-id={{breakfix_machine_id}}"
+          - "--allow-mutation={{breakfix_allow_mutation}}"
+        # The subprocess must outlive both bounded PATCH calls and restoration
+        # polling so its finally block cannot be killed before cleanup.
+        timeout: 1200
 
       - name: return_rack_maintenance
         phase: test
@@ -592,6 +597,11 @@ tests:
     site_id: "{{env.NICO_SITE_ID}}"
     nico_api_base: "{{env.NICO_API_BASE}}"
     instance_id: "{{env.NICO_INSTANCE_ID}}"
+    # BFX01-02 is mutating and never selects a Machine automatically. Set this
+    # only to a dedicated staging fixture that the test may enter and leave
+    # maintenance mode.
+    breakfix_machine_id: "{{env.NICO_BREAKFIX_MACHINE_ID}}"
+    breakfix_allow_mutation: "{{env.NICO_BREAKFIX_ALLOW_MUTATION}}"
     # The imported bare_metal suite defines these for the full instance
     # lifecycle, which NICo does not run yet. Blank them so the suite's
     # {{instance_type}} self-reference does not emit missing-variable warnings.

isvctl/configs/providers/nico/scripts/breakfix/gap_stub.py

@@ -5,7 +5,7 @@
 """Emit the documented NICo gap payload for a break-fix requirement.
 
 Several break-fix requirements have no NICo tenant REST surface to exercise:
-the mutating BFX01 workflows run through Maestro/repair fixtures, and the
+the remaining mutating BFX01 workflows run through Maestro/repair fixtures, and the
 BFX02-02/BFX03-02/BFX04-01/BFX05/BFX06 signals are not exposed at all. Each of
 those steps emits a structured skip naming the gap rather than a hard failure,
 so the suite reports "not available on this platform" instead of "broken".
@@ -28,10 +28,6 @@
 
 # gap id -> (skip reason, contract fields the bound validation still expects)
 GAPS: dict[str, tuple[str, dict[str, Any]]] = {
-    "BFX01-02": (
-        "Return-node-for-maintenance is a mutating NICo repair workflow requiring lab fixtures (BFX01-02 gap)",
-        {"operation": {"requested": False, "accepted": False}},
-    ),
     "BFX01-03": (
         "Rack-level maintenance return API is not exposed on NICo tenant REST (BFX01-03 gap)",
         {"operation": {"requested": False, "accepted": False}},

isvctl/configs/providers/nico/scripts/breakfix/return_node_maintenance.py

@@ -0,0 +1,214 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Return one explicit NICo machine for maintenance and restore it (BFX01-02).
+
+The caller must name a dedicated fixture. The script never discovers or selects
+a mutation target on its own. It verifies NICo reports the Machine in
+``Maintenance`` after the request, then disables maintenance mode in ``finally``
+so failed assertions do not strand the fixture.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import time
+from pathlib import Path
+from typing import Any
+from urllib.error import URLError
+from urllib.parse import quote
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from breakfix._common import emit
+from common.nico_client import NicoAuthError, forge_get, forge_patch, resolve_auth
+
+MUTATION_TIMEOUT_SECONDS = 300
+RESTORE_TIMEOUT_SECONDS = 120
+RESTORE_POLL_INTERVAL_SECONDS = 2.0
+MAINTENANCE_MESSAGE = "ISV BFX01-02 validation fixture; automatically restored"
+
+
+def _operation(machine_id: str) -> dict[str, Any]:
+    """Build the provider-neutral operation result with safe defaults."""
+    return {
+        "requested": False,
+        "accepted": False,
+        "machine_id": machine_id,
+        "maintenance_mode": "",
+        "restored": False,
+    }
+
+
+def _api_error(exc: Exception) -> str:
+    """Return a concise API error without response payloads or credentials."""
+    return f"{type(exc).__name__}: {exc}"
+
+
+def _machine_status(machine: dict[str, Any]) -> str:
+    """Return a normalized Machine status string."""
+    return str(machine.get("status") or "").strip()
+
+
+def _has_binding(machine: dict[str, Any], key: str) -> bool:
+    """Return whether a Machine has a non-empty allocation identifier."""
+    value = machine.get(key)
+    return value is not None and bool(str(value).strip())
+
+
+def _wait_for_status(
+    org: str,
+    machine_path: str,
+    token: str,
+    *,
+    base_url: str,
+    expected_status: str,
+) -> dict[str, Any]:
+    """Poll NICo until the Machine returns to its exact initial status."""
+    deadline = time.monotonic() + RESTORE_TIMEOUT_SECONDS
+    while True:
+        current = forge_get(org, machine_path, token, base_url=base_url)
+        if _machine_status(current) == expected_status or time.monotonic() >= deadline:
+            return current
+        time.sleep(RESTORE_POLL_INTERVAL_SECONDS)
+
+
+def main() -> int:
+    """Request and verify maintenance mode, then restore the explicit fixture."""
+    parser = argparse.ArgumentParser(description="Return an explicit NICo machine for maintenance")
+    parser.add_argument("--org", required=True)
+    parser.add_argument("--site-id", required=True)
+    parser.add_argument("--api-base", required=True)
+    parser.add_argument("--machine-id", default="", help="Dedicated staging Machine ID; no automatic selection")
+    parser.add_argument("--allow-mutation", default="", help="Must be exactly 1 to mutate the staging fixture")
+    args = parser.parse_args()
+
+    machine_id = args.machine_id.strip()
+    result: dict[str, Any] = {
+        "success": False,
+        "platform": "nico",
+        "site_id": args.site_id,
+        "operation": _operation(machine_id),
+    }
+    operation = result["operation"]
+
+    if not machine_id:
+        result.update(
+            {
+                "success": True,
+                "skipped": True,
+                "skip_reason": (
+                    "No dedicated maintenance fixture configured; set NICO_BREAKFIX_MACHINE_ID "
+                    "to a staging Machine that may be mutated and restored"
+                ),
+            }
+        )
+        return emit(result)
+    if args.allow_mutation != "1":
+        result.update(
+            {
+                "success": True,
+                "skipped": True,
+                "skip_reason": (
+                    "NICo maintenance mutation is disabled; set NICO_BREAKFIX_ALLOW_MUTATION=1 "
+                    "only for an approved staging fixture"
+                ),
+            }
+        )
+        return emit(result)
+
+    machine_path = f"machine/{quote(machine_id, safe='')}"
+    try:
+        auth = resolve_auth()
+        initial = forge_get(args.org, machine_path, auth.token, base_url=args.api_base)
+    except (NicoAuthError, URLError, ValueError) as exc:
+        result["error"] = _api_error(exc)
+        return emit(result)
+
+    if initial.get("id") != machine_id:
+        result["error"] = "NICo returned a different Machine than the configured maintenance fixture"
+        return emit(result)
+    if initial.get("siteId") != args.site_id:
+        result["error"] = "Configured maintenance fixture does not belong to the configured Site"
+        return emit(result)
+
+    initial_status = _machine_status(initial)
+    if not initial_status:
+        result["error"] = "Configured maintenance fixture has no observable status"
+        return emit(result)
+    if initial_status == "Maintenance":
+        result["error"] = "Configured maintenance fixture is already in Maintenance; refusing to take ownership"
+        return emit(result)
+    if initial_status != "Ready":
+        result["error"] = "Configured maintenance fixture must be Ready before validation"
+        return emit(result)
+    if _has_binding(initial, "instanceId") or _has_binding(initial, "tenantId"):
+        result["error"] = "Configured maintenance fixture is allocated; refusing to mutate it"
+        return emit(result)
+
+    maintenance_attempted = False
+    cleanup_errors: list[str] = []
+    try:
+        operation["requested"] = True
+        maintenance_attempted = True
+        updated = forge_patch(
+            args.org,
+            machine_path,
+            auth.token,
+            base_url=args.api_base,
+            body={"setMaintenanceMode": True, "maintenanceMessage": MAINTENANCE_MESSAGE},
+            timeout=MUTATION_TIMEOUT_SECONDS,
+        )
+        current = forge_get(args.org, machine_path, auth.token, base_url=args.api_base)
+        updated_status = _machine_status(updated)
+        current_status = _machine_status(current)
+        operation["maintenance_mode"] = current_status
+        operation["accepted"] = updated_status == "Maintenance" and current_status == "Maintenance"
+        if not operation["accepted"]:
+            result["error"] = (
+                "NICo did not confirm Maintenance state "
+                f"(response={updated_status or 'missing'}, current={current_status or 'missing'})"
+            )
+    except (NicoAuthError, URLError, ValueError) as exc:
+        result["error"] = _api_error(exc)
+    finally:
+        if maintenance_attempted:
+            try:
+                forge_patch(
+                    args.org,
+                    machine_path,
+                    auth.token,
+                    base_url=args.api_base,
+                    body={"setMaintenanceMode": False},
+                    timeout=MUTATION_TIMEOUT_SECONDS,
+                )
+                restored_current = _wait_for_status(
+                    args.org,
+                    machine_path,
+                    auth.token,
+                    base_url=args.api_base,
+                    expected_status=initial_status,
+                )
+                current_status = _machine_status(restored_current)
+                operation["restored"] = current_status == initial_status
+                if not operation["restored"]:
+                    cleanup_errors.append(
+                        "NICo did not restore the fixture to its initial Ready state "
+                        f"(current={current_status or 'missing'})"
+                    )
+            except (NicoAuthError, URLError, ValueError) as exc:
+                cleanup_errors.append(_api_error(exc))
+
+    if cleanup_errors:
+        result["cleanup_errors"] = cleanup_errors
+        result.setdefault("error", "Failed to restore the maintenance fixture")
+
+    result["success"] = bool(operation["requested"] and operation["accepted"] and operation["restored"])
+    if not result["success"]:
+        result.setdefault("error", "Node maintenance validation did not complete")
+    return emit(result)
+
+
+if __name__ == "__main__":
+    sys.exit(main())