From 07311d8bf23799a8ba95f0076da2728ceafc661c Mon Sep 17 00:00:00 2001 From: thxCode Date: Sun, 25 Oct 2026 16:41:03 +0800 Subject: [PATCH] refactor(agents): consolidate the e2e case corpus without row loss The 122-case corpus carried whole families split across files that shared one fixture, one precondition and one dry-run channel, plus repeated blocks no helper owned. A three-agent pipeline consolidated it: an analysis pass (agy) ranked 24 zero-row-loss case merges, 11 code extractions and 8 dead-weight cleanups and rejected 11 merges that would lose rows; an implementation pass (mcode) executed the approved plan; a review pass (agy) audited the diff against the plan; an adversarial crosscheck (codex) then found one blocker and three should-fix escapes - a poller forwarded through a replaceable global, a teardown that stopped sweeping unnamed bindings, a node-JSON parse failure reading as an environmental skip, and a section-only dependency promoted to a whole-case gate - all fixed here. The corpus lands at 83 cases and 13 libraries: 39 case files retired into their hosts, about 3.9k lines removed, every logical STATUS row preserved (1,051 distinct CHECK labels before, 1,058 after - the growth is the crosscheck-era un-hiding of rows the old counter skipped and the failover suite running its labels per kill mode), and the catalogue keeps its 1:1 bijection with the disk, naming each retired number in its host row. Signed-off-by: thxCode --- .agents/skills/gpustack-operator-e2e/SKILL.md | 124 +- .../gpustack-operator-e2e/cases/_epd-lib.sh | 558 +++++ .../cases/_kvcache-ha-lib.sh | 220 ++ .../cases/_kvcache-inject-lib.sh | 104 +- .../gpustack-operator-e2e/cases/_md-lib.sh | 389 ++++ .../cases/_model-hub-lib.sh | 51 +- .../cases/_partition-lib.sh | 55 +- .../gpustack-operator-e2e/cases/_pool-lib.sh | 147 ++ .../gpustack-operator-e2e/cases/_rows-lib.sh | 184 +- .../gpustack-operator-e2e/cases/_ssh-lib.sh | 90 + .../cases/_topology-tas-lib.sh | 107 +- .../gpustack-operator-e2e/cases/case-1.sh | 12 +- .../gpustack-operator-e2e/cases/case-100.sh | 6 - .../gpustack-operator-e2e/cases/case-101.sh | 291 ++- .../gpustack-operator-e2e/cases/case-102.sh | 291 --- .../gpustack-operator-e2e/cases/case-103.sh | 180 +- .../gpustack-operator-e2e/cases/case-104.sh | 8 +- .../gpustack-operator-e2e/cases/case-105.sh | 342 --- .../gpustack-operator-e2e/cases/case-106.sh | 226 -- .../gpustack-operator-e2e/cases/case-107.sh | 20 +- .../gpustack-operator-e2e/cases/case-108.sh | 744 +++++-- .../gpustack-operator-e2e/cases/case-109.sh | 181 -- .../gpustack-operator-e2e/cases/case-11.sh | 31 +- .../gpustack-operator-e2e/cases/case-110.sh | 290 --- .../gpustack-operator-e2e/cases/case-111.sh | 98 +- .../gpustack-operator-e2e/cases/case-112.sh | 10 +- .../gpustack-operator-e2e/cases/case-113.sh | 24 +- .../gpustack-operator-e2e/cases/case-114.sh | 241 --- .../gpustack-operator-e2e/cases/case-116.sh | 660 ++++-- .../gpustack-operator-e2e/cases/case-117.sh | 889 ++++++-- .../gpustack-operator-e2e/cases/case-118.sh | 221 +- .../gpustack-operator-e2e/cases/case-119.sh | 570 ----- .../gpustack-operator-e2e/cases/case-12.sh | 158 -- .../gpustack-operator-e2e/cases/case-120.sh | 242 +-- .../gpustack-operator-e2e/cases/case-121.sh | 258 +-- .../gpustack-operator-e2e/cases/case-122.sh | 344 --- .../gpustack-operator-e2e/cases/case-123.sh | 315 --- .../gpustack-operator-e2e/cases/case-124.sh | 334 --- .../gpustack-operator-e2e/cases/case-125.sh | 621 ------ .../gpustack-operator-e2e/cases/case-126.sh | 439 ++-- .../gpustack-operator-e2e/cases/case-127.sh | 92 +- .../gpustack-operator-e2e/cases/case-128.sh | 67 +- .../gpustack-operator-e2e/cases/case-129.sh | 41 +- .../gpustack-operator-e2e/cases/case-13.sh | 29 +- .../gpustack-operator-e2e/cases/case-130.sh | 241 --- .../gpustack-operator-e2e/cases/case-14.sh | 8 +- .../gpustack-operator-e2e/cases/case-15.sh | 29 +- .../gpustack-operator-e2e/cases/case-16.sh | 217 +- .../gpustack-operator-e2e/cases/case-18.sh | 187 -- .../gpustack-operator-e2e/cases/case-19.sh | 35 +- .../gpustack-operator-e2e/cases/case-2.sh | 356 ++- .../gpustack-operator-e2e/cases/case-20.sh | 189 -- .../gpustack-operator-e2e/cases/case-21.sh | 45 +- .../gpustack-operator-e2e/cases/case-22.sh | 8 +- .../gpustack-operator-e2e/cases/case-23.sh | 441 +--- .../gpustack-operator-e2e/cases/case-24.sh | 6 +- .../gpustack-operator-e2e/cases/case-25.sh | 322 ++- .../gpustack-operator-e2e/cases/case-26.sh | 196 -- .../gpustack-operator-e2e/cases/case-27.sh | 4 +- .../gpustack-operator-e2e/cases/case-28.sh | 10 +- .../gpustack-operator-e2e/cases/case-29.sh | 467 +++- .../gpustack-operator-e2e/cases/case-3.sh | 135 -- .../gpustack-operator-e2e/cases/case-30.sh | 176 -- .../gpustack-operator-e2e/cases/case-31.sh | 166 -- .../gpustack-operator-e2e/cases/case-32.sh | 4 +- .../gpustack-operator-e2e/cases/case-34.sh | 4 +- .../gpustack-operator-e2e/cases/case-35.sh | 31 +- .../gpustack-operator-e2e/cases/case-36.sh | 62 +- .../gpustack-operator-e2e/cases/case-37.sh | 251 ++- .../gpustack-operator-e2e/cases/case-38.sh | 32 +- .../gpustack-operator-e2e/cases/case-39.sh | 29 +- .../gpustack-operator-e2e/cases/case-4.sh | 623 ++++-- .../gpustack-operator-e2e/cases/case-40.sh | 274 --- .../gpustack-operator-e2e/cases/case-41.sh | 37 +- .../gpustack-operator-e2e/cases/case-42.sh | 52 +- .../gpustack-operator-e2e/cases/case-43.sh | 25 +- .../gpustack-operator-e2e/cases/case-44.sh | 1112 ++++++---- .../gpustack-operator-e2e/cases/case-45.sh | 579 +++-- .../gpustack-operator-e2e/cases/case-47.sh | 781 +++++-- .../gpustack-operator-e2e/cases/case-48.sh | 458 ---- .../gpustack-operator-e2e/cases/case-49.sh | 507 +++-- .../gpustack-operator-e2e/cases/case-5.sh | 8 +- .../gpustack-operator-e2e/cases/case-50.sh | 1144 ++++++---- .../gpustack-operator-e2e/cases/case-51.sh | 489 ----- .../gpustack-operator-e2e/cases/case-52.sh | 23 +- .../gpustack-operator-e2e/cases/case-53.sh | 1900 +++++++++++++++-- .../gpustack-operator-e2e/cases/case-54.sh | 174 -- .../gpustack-operator-e2e/cases/case-55.sh | 283 --- .../gpustack-operator-e2e/cases/case-56.sh | 187 -- .../gpustack-operator-e2e/cases/case-57.sh | 358 ---- .../gpustack-operator-e2e/cases/case-59.sh | 425 ---- .../gpustack-operator-e2e/cases/case-6.sh | 341 --- .../gpustack-operator-e2e/cases/case-60.sh | 493 ----- .../gpustack-operator-e2e/cases/case-61.sh | 438 ---- .../gpustack-operator-e2e/cases/case-62.sh | 541 ++--- .../gpustack-operator-e2e/cases/case-63.sh | 133 +- .../gpustack-operator-e2e/cases/case-64.sh | 412 ---- .../gpustack-operator-e2e/cases/case-65.sh | 112 +- .../gpustack-operator-e2e/cases/case-66.sh | 4 - .../gpustack-operator-e2e/cases/case-67.sh | 288 +-- .../gpustack-operator-e2e/cases/case-68.sh | 78 +- .../gpustack-operator-e2e/cases/case-7.sh | 103 - .../gpustack-operator-e2e/cases/case-70.sh | 410 ++-- .../gpustack-operator-e2e/cases/case-71.sh | 192 -- .../gpustack-operator-e2e/cases/case-73.sh | 137 +- .../gpustack-operator-e2e/cases/case-74.sh | 153 -- .../gpustack-operator-e2e/cases/case-75.sh | 400 +++- .../gpustack-operator-e2e/cases/case-76.sh | 340 --- .../gpustack-operator-e2e/cases/case-77.sh | 402 ---- .../gpustack-operator-e2e/cases/case-78.sh | 1096 +++++++--- .../gpustack-operator-e2e/cases/case-79.sh | 479 ----- .../gpustack-operator-e2e/cases/case-8.sh | 538 ++++- .../gpustack-operator-e2e/cases/case-86.sh | 17 +- .../gpustack-operator-e2e/cases/case-87.sh | 5 +- .../gpustack-operator-e2e/cases/case-88.sh | 10 +- .../gpustack-operator-e2e/cases/case-89.sh | 12 +- .../gpustack-operator-e2e/cases/case-9.sh | 12 +- .../gpustack-operator-e2e/cases/case-90.sh | 12 +- .../gpustack-operator-e2e/cases/case-93.sh | 363 ---- .../gpustack-operator-e2e/cases/case-95.sh | 341 --- .../gpustack-operator-e2e/cases/case-96.sh | 33 +- .../gpustack-operator-e2e/cases/case-97.sh | 43 +- .../gpustack-operator-e2e/cases/case-98.sh | 367 +++- .../gpustack-operator-e2e/cases/case-99.sh | 6 - .../cases/run-partition-block.sh | 2 +- .../references/drain-recycle.md | 14 +- 126 files changed, 13705 insertions(+), 18016 deletions(-) create mode 100644 .agents/skills/gpustack-operator-e2e/cases/_epd-lib.sh create mode 100644 .agents/skills/gpustack-operator-e2e/cases/_kvcache-ha-lib.sh create mode 100644 .agents/skills/gpustack-operator-e2e/cases/_md-lib.sh create mode 100644 .agents/skills/gpustack-operator-e2e/cases/_pool-lib.sh create mode 100644 .agents/skills/gpustack-operator-e2e/cases/_ssh-lib.sh delete mode 100644 .agents/skills/gpustack-operator-e2e/cases/case-102.sh delete mode 100644 .agents/skills/gpustack-operator-e2e/cases/case-105.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-106.sh delete mode 100644 .agents/skills/gpustack-operator-e2e/cases/case-109.sh delete mode 100644 .agents/skills/gpustack-operator-e2e/cases/case-110.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-114.sh delete mode 100644 .agents/skills/gpustack-operator-e2e/cases/case-119.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-12.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-122.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-123.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-124.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-125.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-130.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-18.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-20.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-26.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-3.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-30.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-31.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-40.sh delete mode 100644 .agents/skills/gpustack-operator-e2e/cases/case-48.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-51.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-54.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-55.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-56.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-57.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-59.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-6.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-60.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-61.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-64.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-7.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-71.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-74.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-76.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-77.sh delete mode 100755 .agents/skills/gpustack-operator-e2e/cases/case-79.sh delete mode 100644 .agents/skills/gpustack-operator-e2e/cases/case-93.sh delete mode 100644 .agents/skills/gpustack-operator-e2e/cases/case-95.sh diff --git a/.agents/skills/gpustack-operator-e2e/SKILL.md b/.agents/skills/gpustack-operator-e2e/SKILL.md index 7c7a460b1..bb9f38c2b 100644 --- a/.agents/skills/gpustack-operator-e2e/SKILL.md +++ b/.agents/skills/gpustack-operator-e2e/SKILL.md @@ -41,14 +41,19 @@ Run as a **test-orchestration lead** (main agent) coordinating read-only **domai - The `cases/_*.sh` helpers are not cases: never run one directly. - `../_e2e-lib/scripts/` — `preflight.sh`, `build-load.sh `, `deploy.sh ` (**installs only** — it refuses an existing release; upgrade in place with `helm upgrade`, or `teardown.sh` first), `assert-core.sh `, `teardown.sh ` (custom-resource drain, test artifacts, release uninstall and a CRD-drain verdict; the drain and the cleanup are both delegated to the chart's `files/drain.sh` and `files/cleanup.sh`), plus the cluster-lifecycle trio `cluster-auth.sh ` (read-only), `provision.sh ` and `destroy.sh `, and `kube-context.sh ` (read-only; targets a context that is not the current one). - `cases/case-N.sh ` — one scenario each; ends in a `STATUS | CHECK | OBJECT` table, exits non-zero on any FAIL. -- `cases/_quota-lib.sh` — sourced by the cases that short a pool by occupying it (50, 95, 118): `workload_charge` reads what Kueue charged one replica's Workload, and `pool_replica_bound` turns it into an upper bound on the replicas the queue's quota holds, summed over every flavor. -- `cases/_rows-lib.sh` — `print_rows`, the printer for that table. It splits a row on its first two `|` only, so an OBJECT that carries `|` prints whole; source it rather than writing another loop. -- `cases/_partition-lib.sh` — sourced by the hardware-partition cases (24–32, 34) for node correlation, the `MIG_NODE_SSH` gate, profile/key discovery and pod plumbing. +- `cases/_quota-lib.sh` — sourced by the cases that short a pool by occupying it (50, 118): `workload_charge` reads what Kueue charged one replica's Workload, and `pool_replica_bound` turns it into an upper bound on the replicas the queue's quota holds, summed over every flavor. +- `cases/_rows-lib.sh` — the shared plumbing every case builds on: `print_rows`, the printer for that table (it splits a row on its first two `|` only, so an OBJECT that carries `|` prints whole), `record` and `results` with the `FAILS`/`SKIPS`/`ROWS` counters, the kubectl-shim PATH injection, `wait_for` (the jsonpath poller that reports the last value it saw), the `gpustack-settings` readers and writers (`setting_get`, `setting_set`, `setting_unset`, `settings_save`, `settings_restore`) and `kvi_release_binding_finalizer`. Source it rather than writing another loop or another copy of any of these. +- `cases/_md-lib.sh` — sourced by the ModelDeployment cases (45, 47, 49, 50, 68, 70, 78, 97, 98, 116, 118, 126, 127, 128) for the ModelDeployment Kueue discovery and rendering (`md_usable_instance_type`, `md_usable_type_for`, `md_cq_of_it`, `md_role_block`, `md_apply`), the workload and verdict readers (`md_workloads`, `md_workload_verdicts`), the force-release, and the server-side dry-run harness that asserts a manifest is refused with a substring or accepted (`md_manifest`, `md_dryrun_refuses`, `md_dryrun_accepts`). +- `cases/_epd-lib.sh` — sourced by the live EPD cases (116–121) for the CPU-vendor `NodeFeature` and instance-type-aware cpu-manufacturer fixture, the router-sidecar image override and its restore, and the live serving readers (`epd_wait_pods`, `epd_router_pods`, `epd_wait_endpoint`, `epd_ec_document`, `epd_role_pod_claim`). +- `cases/_kvcache-ha-lib.sh` — sourced by the HA KV cache cases (62, 63, 75) for the Mooncake leader lease, the rollout, the member snapshot and the failover verdict (`kvha_ready_leader_pod`, `kvha_lease_holder`, `kvha_member_snapshot`). +- `cases/_pool-lib.sh` — sourced by the pool cases (11, 23, 35, 38, 39, 41, 42): `discover_node_pool` matches the node's identity labels against the pool's InstanceTypes, which is the one place that mapping is written. +- `cases/_ssh-lib.sh` — sourced by the SSH cases (13, 15, 21, 28, 36) for the keypair generation, the Secret and the port-forward harness (`ssh_setup_key_secret`, `ssh_open_port_forward`, `ssh_exec`, `ssh_sftp`). +- `cases/_partition-lib.sh` — sourced by the hardware-partition cases (23, 24, 25, 27, 28, 29, 32, 34) for node correlation, the `MIG_NODE_SSH` gate, profile/key discovery and pod plumbing. - `cases/run-partition-block.sh [NS] [CASES...]` — runs that family in its required order, writing each case's raw log and exit code. **Drive 24–32/34 with it rather than by hand**: the ordering is a real constraint and the block outlives a context, so a step held only in conversation is one a compaction drops. - `cases/_rdma-lib.sh` — sourced by the RDMA cases (80–85) for the precondition gate, the host-probe Pod and the ledger facts. - `cases/run-rdma-block.sh [--report-only] [NS] [CASES...]` — `--report-only` prints, without touching the cluster, which of those readings this cluster can answer and what each unmet requirement is missing. A run prints the same matrix, then each case's exit code **and whether its output carries `NOTHING WAS VERIFIED`** — which is how a skip is told from a pass, since both exit 0. - `cases/_topology-tas-lib.sh` — sourced by the TAS cases (87) for the zone-shape measurement, the apply and name-scoped teardown of a case-prefixed Topology/ResourceFlavor/ClusterQueue/LocalQueue set, the Workload admission readers, and the Workload→owner-Pod→ModelDeployment join. -- `cases/_model-hub-lib.sh` — sourced by the node-delivery cases (101–104, 107–111, 113, 114): deploys `cases/_model-hub.py` as a test Hugging Face hub (revision, tree, whoami and ranged resolve endpoints, with throttle, corrupt and deny switches and a request record) or as a forward proxy on the stock python image, and holds the Setting, consumer-Pod and plugin readers, a raw-watch recorder of one NodeModelStore's writes with their threshold statistics, and an impersonated reader of an artifact's `progress`. +- `cases/_model-hub-lib.sh` — sourced by the node-delivery cases (101, 103, 104, 107, 108, 111, 113): deploys `cases/_model-hub.py` as a test Hugging Face hub (revision, tree, whoami and ranged resolve endpoints, with throttle, corrupt and deny switches and a request record) or as a forward proxy on the stock python image, and holds the Setting, consumer-Pod and plugin readers, a raw-watch recorder of one NodeModelStore's writes with their threshold statistics, and an impersonated reader of an artifact's `progress`. - `cases/_serving-lib.sh` — sourced by the serving cases (91, 94, 129) and delegates to `../_e2e-lib/scripts/serving-lib.sh` (shared with `gpustack-operator-benchmark`): `serving_profile ` is the one place a vendor or engine difference is written — where a transfer is counted, how the leg is pinned, which routers render it — so aligning a vendor is an edit there, not in each case. - `references/` — `drain-recycle.md` (per-case rationale + mock recipes), `packaged-image-deploy.md` (image-ref ↔ chart-values contract), `rdma-host-shapes.md` (which host shape answers which RDMA reading, and the one nothing answers), `manual-ssh-verification.md` (verifying on a host the cases cannot reach through kubectl), `kind-two-zones.yaml` (a local cluster of four workers, two per zone, for cases 87 and 107); shared `../_e2e-lib/references/{orchestration,troubleshooting}.md`. @@ -56,80 +61,57 @@ Run as a **test-orchestration lead** (main agent) coordinating read-only **domai Each case is self-contained; its header (see **Case header contract**) states goal / environment (incl. auto-skip) / inputs (MOCKED vs real) / expected / cleanup. `Needs` = the hardware/tooling the case requires; the exact auto-skip conditions live in the case's own `Environment:` field, which is what the script actually prints. +In every row below, the first number is the surviving script's number; the numbers after it were absorbed into that script, so a retired number maps to its host row. + | Case | Title | Run when these change (`git diff --name-only origin/main...HEAD`) | Mutates | Needs | |---|---|---|---|---| | 1 | CPU-only scheduling chain materializes — zero Cohort, InstanceType Active | always (mandatory) | no | any | -| 2 | Running Instance admits, then drain stops it (not recreate) | `pkg/worker/controllers/worker/instance.go`, `pkg/worker/webhooks/worker/instance.go`, `deploy/gpustack-operator/chart/values.yaml` (Kueue's `quotaCheckStrategy`) | yes (confirm) | any | -| 3 | Managed-toggle scopes node onboarding | `pkg/worker/controllers/worker/{node_flavor,instance_type}.go`, `pkg/nodefeature/helper.go` | yes (confirm) | any | -| 4 | AdmissionCheck holds exclusive over-admit | `pkg/worker/controllers/worker/{node_devices_admission,node_devices_inflight,node_devices,instance_type}.go`, `pkg/worker/kuberess/apps_kueue_admission_check.go` | yes (confirm) | one Ready, untainted, schedulable node with no Devices ledger and no `nvidia.com/gpu` | +| 2, 3, 7 | Running Instance admits, then drain stops it (not recreate); the managed toggle scopes node onboarding; and the portless Instance reaches Ready and creates no Service | `pkg/worker/controllers/worker/{instance,node_flavor,instance_type}.go`, `pkg/worker/webhooks/worker/instance.go`, `pkg/nodefeature/helper.go`, `deploy/gpustack-operator/chart/values.yaml` (Kueue's `quotaCheckStrategy`) | yes (confirm) | any | +| 4, 6 | AdmissionCheck holds exclusive over-admit, and the pooled four-view stays fresh: the NFD label projection, the ResourceFlavor, the ClusterQueue, the LocalQueue and the InstanceType all materialize from one Devices ledger | `pkg/worker/controllers/worker/{node_devices_admission,node_devices_inflight,node_devices,instance_type}.go`, `pkg/worker/kuberess/apps_kueue_admission_check.go`, `pkg/worker/webhooks/worker/instance_type.go`, `api/worker/v1alpha1/{instance_type,devices}.go` | yes (confirm) | one Ready, untainted, schedulable node with no Devices ledger and no `nvidia.com/gpu` | | 5 | Pod webhook folds slice-by-memory-% into units | `pkg/worker/webhooks/worker/pod.go`, `pkg/nodefeature/knowns.go` | yes (confirm) | any | -| 6 | Pooled four-view + watch freshness | `pkg/worker/controllers/worker/instance_type.go`, `pkg/worker/webhooks/worker/instance_type.go`, `api/worker/v1alpha1/{instance_type,devices}.go` | yes (confirm) | one Ready node with no Devices ledger | -| 7 | Portless Instance reaches Ready, creates no Service | `pkg/worker/controllers/worker/instance.go` | yes (confirm) | any | -| 8 | Real accelerator slicing runtime isolation | `pkg/deviceplugin/**`, `pkg/devicemanager/**`, `pkg/worker/webhooks/worker/pod.go` | yes (confirm) | real GPU, logical slicing | +| 8, 12, 20 | Real accelerator slicing runtime isolation; the logically sliceable Instance webhook sizes CPU/RAM as slice-% and pins the accelerator to 1; and the sibling InstanceTypes on one pool stay status-consistent | `pkg/deviceplugin/**`, `pkg/devicemanager/**`, `pkg/worker/webhooks/worker/pod.go`, `pkg/worker/webhooks/worker/instance.go`, `pkg/worker/controllers/worker/instance_type.go`, `pkg/utils/quantityx/quantity.go` | yes (confirm) | real GPU, logical slicing | | 9 | Instance lifecycle survives an InstanceType unit-spec change | `pkg/worker/webhooks/worker/instance.go`, `pkg/worker/webhooks/worker/instance_type.go` | yes (confirm) | any. The running-Instance half always SKIPs, because its state cannot be entered (both routes are refused, and the case asks rather than assumes); the stopped-Instance half moves to a sibling type with a smaller unit RAM and needs the general unit RAM to be at least 2Gi | | 11 | Per-card logical-slice accounting: slices pack, and no card is over-committed (SL view + per-card OnceMax) | `pkg/deviceplugin/{server,resource,budget}.go`, `pkg/worker/controllers/worker/{node_devices_admission,instance_type}.go` | yes (confirm) | real GPU, >=2 logically sliceable cards | -| 12 | Logically sliceable Instance webhook: slice-% sizes CPU/RAM, accelerator pinned to 1 | `pkg/worker/webhooks/worker/instance.go`, `pkg/utils/quantityx/quantity.go` | yes (confirm) | real GPU, logically sliceable pool | | 13 | SSH-enabled sliced Instance: slice visible over SSH + confined shell | `pkg/worker/controllers/worker/instance.go`, `pack/ssh-server/rootfs/chroot.sh`, `pkg/deviceplugin/**`, `pkg/devicemanager/allocator/**` | yes (confirm) | real GPU, logical slicing + ssh | | 14 | Multiple slices coexist on one physical card within budget, each reporting its own share | `pkg/worker/controllers/worker/{node_devices_admission,instance_type}.go`, `pkg/deviceplugin/**`, `pkg/devicemanager/detector/slice.go`, `pkg/devicemanager/procattr/**`, `pkg/kubemetrics/**` | yes (confirm) | real GPU, logical slicing | | 15 | Exclusive whole-card SSH Instance still works (regression) | `pkg/worker/controllers/worker/instance.go`, `pack/ssh-server/rootfs/chroot.sh` | yes (confirm) | real GPU, logical slicing + ssh | -| 16 | InstanceTypeFlavor catalog + declarative queue ownership (recreate-on-delete, delete-then-wait teardown) | `pkg/worker/controllers/worker/{instance_type,node_queue,node_flavor}.go`, `pkg/worker/extensionapis/worker/instance_type_flavor.go`, `pkg/worker/settings/value.go` | yes (confirm) | any | -| 18 | CPU-manufacturer awareness reshapes the catalog (finest RF + cpuDetail; collapse↔split by setting) | `pkg/nodefeature/helper.go`, `pkg/worker/settings/value.go`, `pkg/worker/extensionapis/worker/instance_type_flavor.go`, `pkg/worker/webhooks/worker/instance_type.go`, `pkg/worker/controllers/worker/node_flavor.go` | yes (confirm) | any | +| 16, 18 | InstanceTypeFlavor catalog + declarative queue ownership (recreate-on-delete, delete-then-wait teardown); and CPU-manufacturer awareness reshapes the catalog (finest RF + cpuDetail; collapse and split by setting) | `pkg/worker/controllers/worker/{instance_type,node_queue,node_flavor}.go`, `pkg/worker/extensionapis/worker/instance_type_flavor.go`, `pkg/worker/webhooks/worker/instance_type.go`, `pkg/worker/settings/value.go`, `pkg/nodefeature/helper.go` | yes (confirm) | any | | 19 | Awareness on: accelerated type carries real GPU + folded CPU descriptors; a real GPU Instance runs on it | `pkg/worker/controllers/worker/{node_flavor,instance_type}.go`, `pkg/worker/webhooks/worker/instance_type.go`, `pkg/nodefeature/helper.go` | yes (confirm) | real GPU; also checks a two-card NVIDIA Instance when two cards are free | -| 20 | Sibling InstanceTypes on one pool stay status-consistent (Devices-watch re-enqueues all) | `pkg/worker/controllers/worker/instance_type.go` | yes (confirm) | real GPU, logically sliceable pool | | 21 | SSH Instance serves non-interactive SSH (exec channel) + interactive login unchanged | `pack/ssh-server/rootfs/chroot.sh`, `pack/ssh-server/Dockerfile`, `pkg/worker/settings/value.go` | yes (confirm) | ssh client (ssh-keygen, sftp) | | 22 | Cross-mode claims never co-locate on one physical card (exclusive/shared/sliced; free-card placement + held-when-full) | `pkg/deviceplugin/{server,controller,resource}.go`, `pkg/devicemanager/allocator/**`, `pkg/worker/webhooks/worker/pod.go`, `pkg/worker/controllers/worker/node_devices_admission.go` | yes (confirm) | real accelerator, exclusive + shared (C/D also sliced) | | 23 | NVIDIA MIG dynamic-allocation lifecycle (logical→enable→carve→exclusion→reuse→reclaim→disable) | `pkg/devicemanager/allocator/nvidia/**`, `pkg/devicemanager/detector/nvidia/device.go`, `binding/nvml/**`, `pkg/deviceplugin/**`, `pkg/device/population.go`, `pkg/nodefeature/knowns.go`, `pkg/worker/controllers/worker/node_capacity.go`, `pkg/worker/webhooks/worker/pod.go` | yes (confirm) | MIG-capable NVIDIA card + node SSH | | 24 | Mixed node: a partition lands on a partitioned card, a logical slice on a whole one (zero `UnexpectedAdmissionError`) | `pkg/deviceplugin/{server,resource,controller}.go`, `pkg/device/{population,physical_placement}.go`, `pkg/devicemanager/allocator/nvidia/**`, `pkg/nodefeature/knowns.go` | yes (confirm) | partition-capable NVIDIA node, >=2 cards + node SSH | -| 25 | Per-profile capacity is derived from the live ledger, not from a static ceiling (+ node-status write volume) | `pkg/worker/controllers/worker/node_capacity.go`, `pkg/device/physical_placement.go`, `pkg/nodefeature/knowns.go` | yes (confirm) | partition-capable NVIDIA card + node SSH | -| 26 | Partition token health is a node-level count: allocated + remaining | `pkg/deviceplugin/{server,controller}.go`, `pkg/device/physical_placement.go` | yes (confirm) | partition-capable NVIDIA card + node SSH | +| 25, 26 | Per-profile capacity is derived from the live ledger, not from a static ceiling (+ node-status write volume); and partition token health is a node-level count: allocated + remaining | `pkg/worker/controllers/worker/node_capacity.go`, `pkg/device/physical_placement.go`, `pkg/nodefeature/knowns.go`, `pkg/deviceplugin/{server,controller}.go` | yes (confirm) | partition-capable NVIDIA card + node SSH | | 27 | A partitioned card is never judged feasible for an exclusive or shared claim | `pkg/deviceplugin/resource.go`, `pkg/device/population.go`, `pkg/worker/controllers/worker/{instance_type,node_devices_admission}.go` | yes (confirm) | partition-capable NVIDIA node, >=2 cards none partitioned at start + node SSH | | 28 | The SSH sidecar of a partition-backed workload is confined to that same partition | `pkg/deviceplugin/server.go`, `pkg/devicemanager/allocator/nvidia/**`, `pkg/worker/controllers/worker/instance.go`, `pack/ssh-server/**` | yes (confirm) | partition-capable NVIDIA card + node SSH | -| 29 | Two concurrent requests for different profiles each get their own instance | `pkg/deviceplugin/{controller,server}.go`, `pkg/device/physical_placement.go` | yes (confirm) | partition-capable NVIDIA card + node SSH | -| 30 | A terminated init container still charges the card its instance occupies | `pkg/deviceplugin/controller.go`, `pkg/worker/controllers/worker/node_capacity.go`, `pkg/worker/webhooks/worker/pod.go` | yes (confirm) | partition-capable NVIDIA card + node SSH | -| 31 | A same-profile replacement scheduled inside the reclaim window (observation) | `pkg/deviceplugin/reclaim.go`, `pkg/devicemanager/allocator/nvidia/mig.go`, `pkg/worker/controllers/worker/node_capacity.go` | yes (confirm) | partition-capable NVIDIA card + node SSH | +| 29, 30, 31 | Two concurrent requests for different profiles each get their own instance; a terminated init container still charges the card its instance occupies; and a same-profile replacement scheduled inside the reclaim window is observed | `pkg/deviceplugin/{controller,server,reclaim}.go`, `pkg/device/physical_placement.go`, `pkg/devicemanager/allocator/nvidia/mig.go`, `pkg/worker/controllers/worker/node_capacity.go`, `pkg/worker/webhooks/worker/pod.go` | yes (confirm) | partition-capable NVIDIA card + node SSH | | 32 | An instance carved outside GPUStack: placement sees it, the node keys never do (observation) | `pkg/device/physical_placement.go`, `pkg/devicemanager/allocator/nvidia/mig.go`, `pkg/worker/controllers/worker/node_capacity.go` | yes (confirm) | partition-capable NVIDIA card, exactly one partitioned + node SSH (MIG instance mgmt) | | 34 | `single-numa-node` topology with partition capacity only on the far socket (observation) | `pkg/deviceplugin/server.go` (the NUMA topology reported per family) | yes (confirm) | dual-socket node, `single-numa-node` policy, `static` CPU manager, partition-capable card + node SSH | | 35 | Ascend logical-slice placement: claims pack, spill only on a misfit, and never cross into an exclusive card | `pkg/deviceplugin/{server,resource,budget}.go`, `pkg/devicemanager/allocator/ascend/**`, `pkg/worker/webhooks/worker/pod.go` | yes (confirm) | real Ascend, >=3 logically sliceable cards (>=1 free) + CANN-family image | | 36 | Node-pinned Instance with additional volumes, and the host-access gates | `pkg/worker/controllers/worker/instance.go`, `pkg/worker/webhooks/worker/instance.go`, `pkg/worker/settings/value.go`, `api/worker/v1alpha1/instance.go` | yes (confirm) | any + default StorageClass | -| 37 | Instance metrics subresource serves the current instance-scoped utilization, and answers a stopped Instance | `pkg/worker/extensionapis/worker/**`, `pkg/kubemetrics/**`, `pkg/devicemanager/**`, `pkg/utils/datax/snapshot.go`, `api/worker/v1/instance.metrics.go` | yes (confirm) | any (chain materialized) | +| 37, 40 | Instance metrics subresource serves the current instance-scoped utilization and answers a stopped Instance; and the device manager exports this node's Instances as Prometheus gauges, from exactly one target | `pkg/worker/extensionapis/worker/**`, `pkg/kubemetrics/**`, `pkg/devicemanager/**`, `pkg/utils/datax/snapshot.go`, `api/worker/v1/instance.metrics.go` | yes (confirm) | any (chain materialized); the exporter half needs a node running a device manager | | 38 | AMD accelerator claims over both carriers: exclusive whole cards, logical slices, and the Instance metrics array | `pkg/devicemanager/allocator/amd/**`, `pkg/devicemanager/detector/amd/device.go`, `binding/amdsmi/**`, `csrc/amd/rocm-slicing-shim/**`, `pkg/worker/extensionapis/worker/instance.metrics.go` | yes (confirm) | real AMD, logical slicing | | 39 | T-Head PPU: a pinned claim lands exactly where it was told, and a logical slice is capped inside the container | `pkg/devicemanager/allocator/thead/**`, `pkg/devicemanager/detector/thead/device.go`, `binding/hgml/**`, `pkg/deviceplugin/{controller,server}.go`, `csrc/thead/ppu-slicing-shim/**` | yes (confirm) | real T-Head, >=2 accelerators idle + node host context (else `PPU_NODE_SSH`) | -| 40 | The device manager exports this node's Instances as Prometheus gauges, from exactly one target | `pkg/devicemanager/exporter/**`, `pkg/devicemanager/detector/snapshot.go`, `pkg/kubemetrics/**`, `pkg/manager/metrics.go` | yes (confirm) | any node running a device manager | | 41 | The slice pass reads only the carved cards: a whole card on the same node is never queried | `pkg/devicemanager/detector/slice.go`, `pkg/devicemanager/detector/snapshot.go`, `pkg/devicemanager/snapshot.go` | yes (confirm) | real GPU, >=2 logically sliceable cards on one node | | 42 | Hygon DCU partitions: what a grant carries, what one card holds at once, and what is refused | `pkg/devicemanager/allocator/hygon/**`, `pkg/deviceplugin/**`, `pkg/device/physical_placement.go` | yes (confirm) | REAL Hygon DCU hardware on one node ALREADY in Multi-Instance mode + SSH to that node (`MIG_NODE_SSH`); auto-skips otherwise | | 43 | Two namespaces share one KV cache pool: proportional quota, and both backend preconditions fail loudly | `pkg/worker/controllers/worker/kv_cache_pool.go`, `pkg/worker/webhooks/worker/kv_cache_pool{,_binding}.go`, `pkg/worker/kvcache/**`, `api/worker/v1alpha1/kv_cache_pool{,_binding}.go` | yes (confirm) | any (no GPU, no RDMA) + a registry the cluster can pull the Mooncake image from | -| 44 | What an effective quota enforces: a refusal when nothing can be evicted, an eviction of the domain's own objects when something can | `pkg/worker/kvcache/mooncake/tenant_metrics.go`, `pkg/worker/controllers/worker/kv_cache_pool.go` (the Binding pass) | yes (confirm) | CASE 43 passing + a registry the cluster can pull the Mooncake image from | -| 45 | The ModelDeployment admission surface: every refusal fires, and each one from the layer that owns it (schema / webhook / controller) | `pkg/worker/webhooks/worker/model_deployment.go`, `pkg/worker/controllers/worker/model_deployment{,_binding}.go`, `api/worker/v1alpha1/model_deployment.go` | yes | any (no GPU), and deliberately **no** KVCachePoolBinding — the case names one that does not exist. Optionally `E2E_MD_INSTANCE_TYPE` to pick the InstanceType | -| 47 | Two ModelDeployments on one Binding are rendered one reuse domain and a third on another Binding a different one, and the rendered value is what the store partitions on. Uses **sglang** because its tenant is one environment variable on the Pod spec, while the vLLM family carries the same domain as `tenant_id` inside a projected client config (case 55 reads both carriers) | `pkg/worker/controllers/worker/model_deployment_connector.go`, `pkg/worker/controllers/worker/model_deployment_binding.go`, `pkg/worker/kvcache/inject/**` | yes (confirm) | any (no GPU, no engine image) + a registry the cluster can pull the Mooncake image from; the replicas are never expected to start, every asserted value is on the Pod spec | -| 48 | The failure side: a Binding whose pool cannot be reached renders no connector AT ALL — measured against a control on a Ready Binding — and a rendered connector is never reported as an attached cache. Uses **vllm** deliberately: its connector travels on five separate carriers, so "no connector" is five independent absences and a PARTIAL render is visible. `CacheAttached != True` is true of every deployment on a cluster with no engine image, so no row asserts it alone | `pkg/worker/controllers/worker/model_deployment_binding.go` (`resolveModelDeploymentConnection`), `pkg/worker/controllers/worker/model_deployment_cache_attached.go`, `pkg/worker/kvcache/inject/vllm.go` | yes (confirm) | any (no GPU, no engine image) + a registry the cluster can pull the Mooncake image from + an InstanceType (`E2E_MD_INSTANCE_TYPE` to pick one); the replicas are never expected to start, and one row SKIPs — which shape the engine takes when the connector cannot initialize needs an image that serves | -| 49 | Every replica of a multi-role ModelDeployment becomes its own Kueue Workload, and deleting the deployment completes | `pkg/worker/controllers/worker/model_deployment{,_pod_group}.go`, `api/worker/v1alpha1/model_deployment.go` | yes (confirm) | any (no GPU, no engine image) + an InstanceType and, in ``, the pool's entrance LocalQueue. The replicas are placeholders that never serve, so the two serving rows SKIP; deleting one replica's Workload is asserted to leave its sibling's Pod and Workload admitted, and SKIPs only when the group never reaches two replicas on two Workloads. Optionally `E2E_MD_INSTANCE_TYPE` / `E2E_MD_IMAGE` | -| 50 | A short pool starves a P/D deployment WHOLE: the role that would have fit alone waits too | `pkg/worker/controllers/worker/model_deployment_pod_group.go`, `pkg/worker/controllers/worker/model_deployment.go` | yes (confirm) | as CASE 49, on a pool of any number of flavors whose queue is in no cohort. The shortage is made by OCCUPYING the quota, never by editing the ClusterQueue: a probe measures how many replicas Kueue reserves across every flavor, the filler leaves one, and the case SKIPs, printing the numbers, when the probe would exceed `E2E_C50_MAX_REPLICAS` (default 128) or the subject does not show exactly one role holding quota | -| 51 | Every P/D refusal fires, from the layer that owns it, and a replicas change replaces nobody | `pkg/worker/webhooks/worker/model_deployment.go`, `pkg/worker/controllers/worker/model_deployment{,_pod_group}.go`, `api/worker/v1alpha1/model_deployment.go` | yes | as CASE 49; the two-instanceType row SKIPs unless a second InstanceType exists that a deployment can name (not inactive, not terminating, and holding a whole card if accelerated) | +| 44, 77 | What an effective quota enforces: a refusal when nothing can be evicted, an eviction of the domain's own objects when something can, and the multi-tenant ledger gate - an unregistered tenant's put is refused `-1701` while the client stays healthy, and a Pool+Binding whose `domain.name` is the tenant id admits the identical put | `pkg/worker/kvcache/mooncake/tenant_metrics.go`, `pkg/worker/kvcache/mooncake/**`, `pkg/worker/controllers/worker/kv_cache_pool.go` (the Binding and domain-registration passes) | yes (confirm) | CASE 43 passing + a registry the cluster can pull the Mooncake image from (`E2E_MOONCAKE_IMAGE`, CPU-capable, carrying the python client); the backend must be a replicated HA leader - the k8s:// master address and the probe's member Role exist only above one replica | +| 45, 51 | The ModelDeployment admission surface: every refusal fires, each from the layer that owns it (schema / webhook / controller); every P/D refusal fires from the layer that owns it; and a replicas change replaces nobody | `pkg/worker/webhooks/worker/model_deployment.go`, `pkg/worker/controllers/worker/model_deployment{,_binding,_pod_group}.go`, `api/worker/v1alpha1/model_deployment.go` | yes | any (no GPU), and deliberately **no** KVCachePoolBinding - the case names one that does not exist. Optionally `E2E_MD_INSTANCE_TYPE` to pick the InstanceType; the two-instanceType row SKIPs unless a second InstanceType exists that a deployment can name | +| 47, 48 | Two ModelDeployments on one Binding are rendered one reuse domain and a third on another Binding a different one, and the rendered value is what the store partitions on; and the failure side - a Binding whose pool cannot be reached renders no connector AT ALL, measured against a control on a Ready Binding | `pkg/worker/controllers/worker/model_deployment_{connector,binding,cache_attached}.go`, `pkg/worker/kvcache/inject/**` | yes (confirm) | any (no GPU, no engine image) + a registry the cluster can pull the Mooncake image from + an InstanceType (`E2E_MD_INSTANCE_TYPE` to pick one); the replicas are never expected to start | +| 49, 61 | Every replica of a multi-role ModelDeployment becomes its own Kueue Workload and deleting the deployment completes; a Service deleted out of band comes back, and the watch is what brought it | `pkg/worker/controllers/worker/model_deployment{,_pod_group,_service}.go`, `api/worker/v1alpha1/model_deployment.go` | yes (confirm) | any (no GPU, no engine image) + an InstanceType and, in ``, the pool's entrance LocalQueue. The replicas are placeholders that never serve, so the two serving rows SKIP; the attribution row SKIPs unless every other object the controller watches is unchanged across the window | +| 50, 95, 106 | A short pool starves a P/D deployment WHOLE - the role that would have fit alone waits too; a change on a waiting replica's Workload alone reaches the deployment's QuotaReserved condition within a bounded window with no Pod moving; and with `instance-type-derived-from-node` off a P/D deployment on one administrator-authored InstanceType is admitted as a set | `pkg/worker/controllers/worker/model_deployment{,_pod_group,_status}.go`, `pkg/worker/controllers/worker/{node_queue,model_deployment_joint_admission}.go`, `pkg/worker/webhooks/worker/model_deployment.go` | yes (confirm) | as CASE 49, on a pool of any number of flavors whose queue is in no cohort, plus the `gpustack-settings` Secret in `E2E_SYSTEM_NS` (the case sets the key off and restores it). The shortage is made by OCCUPYING the quota, never by editing the ClusterQueue; the case SKIPs, printing the numbers, when the probe would exceed `E2E_C50_MAX_REPLICAS` (default 128) or the subject does not show exactly one role holding quota | | 52 | The NIC/RDMA inventory is published, is stable across passes, and the `rdma.*` labels agree with it in both directions. A detector restart is driven but its outcome is not claimed — an unchanged record is also what a replacement that never reported produces, so that check SKIPs | `pkg/devicemanager/detector/{network,link,detector}.go`, `pkg/devicemanager/detector/network_linux.go`, `pkg/nodefeature/rdma.go`, `pkg/device/topology_distance.go`, `api/worker/v1alpha1/devices.go`, `binding/helper*.go` | yes (confirm) | any node running a device manager; the RDMA-positive checks skip individually without RDMA hardware | -| 53 | The headline: a plain Deployment reads and writes a KV cache pool with one label | `pkg/worker/webhooks/worker/pod_kv_cache*.go`, `pkg/worker/kvcache/inject/**` | yes (confirm) | any (no GPU, no RDMA) + a registry the cluster can pull the Mooncake image from | -| 54 | No workload lock-in: an LWS Pod and a bare Pod get the identical injection | `pkg/worker/webhooks/worker/pod_kv_cache*.go`, `pkg/worker/kvcache/inject/**` | yes (confirm) | as CASE 53; the LWS half needs the leaderworkerset.x-k8s.io CRD, and self-skips without it | -| 55 | The stamp that replaces isolation: a declared domain is injected and the ACTION is recorded, per engine | `pkg/worker/kvcache/inject/engine.go`, `pkg/worker/webhooks/worker/pod_kv_cache_resolve.go`, `pkg/worker/webhooks/worker/pod_kv_cache_inject.go` | yes (confirm) | as CASE 53 | -| 56 | A Pod that did not opt in comes back byte-identical | `pkg/worker/webhooks/worker/pod_kv_cache.go`, `pkg/worker/webhooks/worker/zz_generated.webhooks.go` | yes (confirm) | as CASE 53 | -| 57 | The refusals on a live API server, each naming what its fix depends on | `pkg/worker/webhooks/worker/pod_kv_cache*.go`, `pkg/worker/kvcache/inject/**`, `pkg/worker/webhooks/worker/kv_cache_backend.go` | yes (confirm) | as CASE 53 | -| 59 | Each rendered artifact is one its own engine accepts: the vLLM file and the SGLang environment | `pkg/worker/kvcache/inject/client_config.go`, `pkg/worker/kvcache/inject/vllm.go`, `pkg/worker/kvcache/inject/sglang.go` | yes (confirm) | as CASE 53, plus `E2E_VLLM_IMAGE` / `E2E_SGLANG_IMAGE`; each half self-skips without its image | -| 60 | The connector name we render is one that engine's own factory can resolve, for both vLLM-family engines. Two halves, and only one is still open: registry membership is ANSWERED off-cluster — one `docker run` per image lists the whole registry, which a cluster run does not improve on — while whether either engine RUNS with what we inject is NOT EXERCISED and needs the accelerator ⇒ the case header says what does not count as filling it | `pkg/worker/kvcache/inject/vllm.go` (`vllmConnectorFor`), `pkg/worker/kvcache/inject/engine.go` | yes (confirm) | as CASE 53, plus `E2E_VLLM_IMAGE` / `E2E_VLLM_ASCEND_IMAGE`; each engine row self-skips without its image, and the vLLM-Ascend row skips whatever the image, because that runtime needs an ascend-transport pool this fixture does not build | -| 61 | A Service deleted out of band comes back, and the watch is what brought it | `pkg/worker/controllers/worker/model_deployment.go` (the `Owns(&core.Service{})` clause), `pkg/worker/controllers/worker/model_deployment_service.go` | yes (confirm) | any (no GPU, no engine image, no KV cache backend) + an InstanceType; the deployment names a KVCachePoolBinding that deliberately does not exist and its replica never serves. Optionally `E2E_MD_INSTANCE_TYPE` / `E2E_MD_IMAGE` / `E2E_SVC_RECREATE_BOUND`. The attribution row SKIPs unless every other object the controller watches is unchanged across the window and a replica was observed at all, which leaves the uid row passing unattributed | -| 62 | A replicated KV cache leader elects exactly one of three through a Lease, the backend's ElectionObserved condition reads True/Electing, the two rendered accounts are sufficient and no more, and an induced graceful failover moves the Lease within a 120s bound without touching a member | `pkg/worker/kvcache/mooncake/leader_workload.go`, `pkg/worker/kvcache/mooncake/ha_rbac.go`, `pkg/worker/kvcache/mooncake/leader_flags.go`, `pkg/worker/controllers/worker/kv_cache_backend_handover.go` | yes (confirm) | any (no GPU, no RDMA; members are DRAM over TCP, one node is enough) + a store image built with the Kubernetes Lease leadership backend (`E2E_MOONCAKE_IMAGE`); with a published upstream image every replica campaigns unanswered and the "exactly one ready" wait times out — that timeout is the wrong-image signature, not a flake | +| 53, 54, 55, 56, 57, 59, 60 | A plain Deployment reads and writes a KV cache pool with one label: an LWS Pod and a bare Pod get the identical injection, the declared domain is injected and the ACTION recorded per engine, a Pod that did not opt in comes back byte-identical, the refusals on a live API server each name what their fix depends on, and each rendered artifact is one its own engine accepts | `pkg/worker/webhooks/worker/pod_kv_cache*.go`, `pkg/worker/kvcache/inject/**`, `pkg/worker/webhooks/worker/kv_cache_backend.go` | yes (confirm) | any (no GPU, no RDMA) + a registry the cluster can pull the Mooncake image from, plus `E2E_VLLM_IMAGE` / `E2E_SGLANG_IMAGE` / `E2E_VLLM_ASCEND_IMAGE`; each engine half self-skips without its image, and the LWS half needs the leaderworkerset.x-k8s.io CRD and self-skips without it | +| 62, 64 | A replicated KV cache leader elects exactly one of three through a Lease, the backend's ElectionObserved condition reads True/Electing, the two rendered accounts are sufficient and no more, and both a graceful failover and a hard kill move the Lease within a 120s bound without touching a member | `pkg/worker/kvcache/mooncake/{leader_workload,ha_rbac,leader_flags}.go`, `pkg/worker/controllers/worker/kv_cache_backend_handover.go` | yes (confirm) | any (no GPU, no RDMA; members are DRAM over TCP, one node is enough) + a store image built with the Kubernetes Lease leadership backend (`E2E_MOONCAKE_IMAGE`); with a published upstream image every replica campaigns unanswered and the "exactly one ready" wait times out - that timeout is the wrong-image signature, not a flake | | 63 | How a member finds the leader under high availability: reading the Lease (`k8s://`) versus keeping the plain Service address — measured over several failover rounds (`E2E_C63_ROUNDS`, default 3) from deletion to the first put that completes within the put bound after the path became unreachable, every store call bounded, each interval split into old-Pod-gone / Lease-moved / endpoint-ready | `pkg/worker/kvcache/mooncake/member_workload.go` (which of the two `MOONCAKE_MASTER` forms a member is handed; the case writes both forms itself and never reads the rendered one), `pkg/worker/kvcache/mooncake/leader_workload.go` (the readiness gate that keeps standbys out of the Service) | yes (confirm) | as CASE 62 (same image need); two probe Pods, one bound to the member Role for `get leases`, one with no API access at all | -| 64 | The rude half of the same failover: a hard-killed leader (`--force --grace-period=0`, holder never releases the Lease) still yields the Lease within the same 120s bound, and the members come through untouched | `pkg/worker/kvcache/mooncake/leader_workload.go` (the readiness gate), the store's k8s Lease campaign/expiry path | yes (confirm) | as CASE 62 (same image need); complements the graceful-delete failover, which is the friendly path | | 65 | A member group's local disk tier takes real bytes on the tier's one write path — eviction-time, with writes pushed past aggregate memory so eviction is forced: a key the leader reports as holding a disk replica but no memory one reads back byte-identical, `master_allocated_file_size_bytes` (not the declared `status.capacity`) reads above zero, and files exist in the host directory | `pkg/worker/kvcache/mooncake/member_workload.go` (the localDisks env render), `pkg/worker/kvcache/mooncake/leader_flags.go` (the offload flags), `api/worker/v1alpha1` (`members[].localDisks`) | yes (confirm) | as CASE 62 (same image need) + `E2E_LOCALDISK_HOST_PATH` pointing at a prepared writable directory on EVERY Ready node (e.g. an i7ie's NVMe at `/mnt/kvcache-localdisk`), else AUTO-SKIP. NO LONGER a known-failure detector: the metric and host-directory checks were expected to FAIL while the store ran on its own 256 MB bucket default, which the old write set never closed; with the bucket limit rendered they pass, so a FAIL is now a regression to act on | | 66 | A stored pre-identity member listing is omitted during a Kubernetes 1.23 CRD/controller upgrade while the leader listing fails, then resumes as identified rows | `pkg/worker/controllers/worker/{kv_cache_backend,kv_cache_pool}.go`, `api/worker/v1alpha1/kv_cache_backend.go` | yes (confirm) | Kubernetes 1.23, helm and jq; old/new CRD manifests plus old/new operator images that every node can use (`E2E_CASE66_*`). The case AUTO-SKIPS if those historical upgrade assets are absent; it does not build or load them | | 67 | A drain between store members on two different nodes moves the bytes its report claims (read back at the target with the source deleted), and a disk-resident key is neither moved nor counted — the counter gap and the 404s are KNOWN-FAILURE DETECTORs that must go red the day the drain reaches the tier | `api/worker/v1alpha1/kv_cache_backend.go` (the scale-in drain premise), the Mooncake store image pin | yes (confirm) | >=2 Ready schedulable nodes (no GPU, no RDMA; members are DRAM over TCP) + a registry the nodes can pull the Mooncake image from. Creates NO operator objects: the store runs as three node-pinned Pods and the drain is driven against the master's own HTTP API | | 68 | A deployment whose roles sit on TWO InstanceTypes is admitted as a set or not at all, parks when the set never assembles, and scales one role without touching anything already running | `pkg/worker/controllers/worker/model_deployment{,_joint_admission,_pod_group,_rollout}.go`, `pkg/worker/kuberess/apps_kueue_admission_check.go` | yes (confirm) | any (no GPU; the infeasible group comes from an Inactive InstanceType the case creates on the working type's CPU pool, not from a shortage) + an InstanceType and the pool's entrance LocalQueue in ``. Optionally `E2E_MD_BINDING` | -| 70 | A routed P/D deployment owns and garbage-collects all six router objects, converges both `spec.router` transitions, remains Starting without an accelerator, and reports every cluster-observable `KVEventsPublishing` status/reason pair | `pkg/worker/controllers/worker/model_deployment_router.go`, `model_deployment_kv_events.go`, `model_deployment_status.go` | yes (confirm) | Single Ready node with no accelerator; CASE 1 has materialized one usable general InstanceType, and the cluster can pull the Mooncake fixture image so a real Ready Binding can make `Publishing` reachable. The router image is deliberately unpullable because readiness is isolated in CASE 71 | -| 71 | A Ready router becomes `status.endpoint`; inability to pull the upstream router image is a stated SKIP rather than loss of CASE 70's lifecycle coverage | `pkg/worker/controllers/worker/model_deployment_status.go`, the default llm-d-router image contract | yes (confirm) | As CASE 70 plus pull access to the default llm-d-router and Envoy images; AUTO-SKIP only for an image-pull reason | +| 70, 71 | A routed P/D deployment owns and garbage-collects all six router objects, converges both `spec.router` transitions, remains Starting without an accelerator, and reports every cluster-observable `KVEventsPublishing` status/reason pair; and a Ready router becomes `status.endpoint`, with an unpullable upstream router image a stated SKIP rather than loss of the lifecycle coverage | `pkg/worker/controllers/worker/model_deployment_router.go`, `model_deployment_kv_events.go`, `model_deployment_status.go` | yes (confirm) | Single Ready node with no accelerator; CASE 1 has materialized one usable general InstanceType, and the cluster can pull the Mooncake fixture image so a real Ready Binding can make `Publishing` reachable. The router image is deliberately unpullable because readiness is isolated in this case's own leg | | 73 | An engine under KV turnover writes the shared store, and the other replica's replay of the same prefixes is the reuse the chain exists for — the write half is a guard, the read half a KNOWN-FAILURE DETECTOR pair (case-67 polarity) that FAILS the day cross-replica reuse starts working, and must then be inverted into positive guards | `pkg/worker/controllers/worker/model_deployment_connector.go`, `pkg/worker/controllers/worker/model_deployment_binding.go`, `pkg/worker/kvcache/inject/**`, the engine image pin | yes (confirm) | A real accelerator pool with at least TWO free exclusive cards, model weights hostPath-staged on the accelerator nodes (`E2E_VB_WEIGHTS`, default `/mnt/kvcache-weights`), `E2E_VB_INSTANCE_TYPE` naming the accelerated InstanceType (exit 2 without it), and a registry the cluster can pull the Mooncake image from — a CUDA-only tag crashes members on CPU-only nodes, so the image must be CPU-capable. Where no two cards are free, `E2E_VB_EXISTING_BACKEND` + `E2E_VB_EXISTING_DOMAIN` + `E2E_VB_EXISTING_SERVICES` together run every verdict row against an existing engine pair and create nothing. The first case in the suite to run real vLLM engines; every verdict rides on counters (`master_key_count`, `vllm:external_prefix_cache_hits_total`, `mem_cache_hit_nums_`), never on TTFT | -| 74 | The store's snapshot flags are refused in leader.extraArgs on create and on an update to a running backend, each refusal on the entry with its reason (a restore can serve another key's bytes; the other keys are read only under a refused switch); the manifest without them, and an image-only update, are accepted as the positive baseline | `pkg/worker/kvcache/mooncake/keys.go`, `pkg/worker/webhooks/worker/kv_cache_backend.go` | yes (confirm) | any (no GPU, no RDMA, no storage class); the creates and the updates are server-side dry runs, and the one backend the case persists selects no node, so it renders a leader and no member Pod | -| 75 | The default single leader holds a Lease; `electionBackend: None` with three replicas is refused; scaling from one to three preserves the first leader Pod; an image bump then emits `KVCacheLeaderHandover` with its count equal to the Lease transitions, replaces all leader Pods, and settles Ready with one serving replica | `pkg/worker/controllers/worker/kv_cache_backend_handover.go`, `pkg/worker/kvcache/mooncake/leader_workload.go` | yes (confirm) | any (no GPU, no RDMA) + a registry the cluster can pull BOTH pinned Mooncake tags from (`E2E_MOONCAKE_IMAGE` start / `E2E_MOONCAKE_ROLLOUT_IMAGE` target; both must parse this operator's argv, carry the lease backend, and run on CPU-only nodes) | -| 76 | `RolloutComplete` stays truthful through a second mid-update (every sample is `Unknown/UpdateNotObserved`, `False/Progressing` or `True/Complete`, never a deadline-ish stall), the update converges with the election gate intact, and the member-re-registration dip clears within its window; the case never gates on `kubectl rollout status`, which times out on every multi-replica leader rollout by construction | `pkg/worker/controllers/worker/kv_cache_backend_rollout.go` | yes (confirm) | as CASE 75 (the same image pair and clauses) | -| 77 | The multi-tenant ledger gate: an unregistered tenant's put is refused `-1701` while the client itself stays healthy, and a Pool+Binding whose `domain.name` is the tenant id admits the identical put; the tenant rides the keyword `tenant_id=` (the next positional slot is a TransferEngine pointer and raises), and the teardown drains the domain because a held domain blocks pool deletion open-ended | `pkg/worker/controllers/worker/kv_cache_pool.go` (the domain registration pass), `pkg/worker/kvcache/mooncake/**` (the master argv and lease render) | yes (confirm) | any (no GPU, no RDMA) + a registry the cluster can pull the Mooncake image from (`E2E_MOONCAKE_IMAGE`, CPU-capable, carrying the python client); the backend must be a replicated HA leader — the k8s:// master address and the probe's member Role exist only above one replica | -| 78 | Scaling a role moves its queue's admitted quota by exactly one replica in each direction, and touches no replica it did not add or remove | `pkg/worker/controllers/worker/model_deployment{,_pod_group,_rollout}.go`, `api/worker/v1alpha1/model_deployment.go` | yes (confirm) | any (no GPU) + an InstanceType, the pool's entrance LocalQueue in ``, and room in the pool for three replicas of one role. Optionally `E2E_MD_INSTANCE_TYPE` / `E2E_MD_IMAGE` / `E2E_MD_SETTLE` | -| 79 | An instance of several Pods is admitted as ONE Kueue group with one PodSet of `size`, each member addressable by a derived name behind a headless Service of its own, and the role's Service fronts only the leader; adding an instance leaves the first one's members untouched | `pkg/worker/controllers/worker/model_deployment{,_render,_pod_group,_service}.go`, `api/worker/v1alpha1/model_deployment.go` (`size`) | yes (confirm) | any (no GPU) + an InstanceType, the pool's entrance LocalQueue in ``, and room in the pool for FOUR Pods of it -- two instances of two. The image must carry a shell and `nslookup` or the two DNS rows SKIP, which is why it defaults to busybox rather than pause. Optionally `E2E_MD_INSTANCE_TYPE` / `E2E_MD_IMAGE` / `E2E_MD_SETTLE` | +| 75, 74, 76 | The store's snapshot flags are refused in leader.extraArgs on create and on an update to a running backend; the default single leader holds a Lease, `electionBackend: None` with three replicas is refused, scaling from one to three preserves the first leader Pod, an image bump emits `KVCacheLeaderHandover` with its count equal to the Lease transitions, and `RolloutComplete` stays truthful through a second mid-update | `pkg/worker/controllers/worker/kv_cache_backend_{handover,rollout}.go`, `pkg/worker/kvcache/mooncake/{leader_workload,keys}.go`, `pkg/worker/webhooks/worker/kv_cache_backend.go` | yes (confirm) | any (no GPU, no RDMA) + a registry the cluster can pull BOTH pinned Mooncake tags from (`E2E_MOONCAKE_IMAGE` start / `E2E_MOONCAKE_ROLLOUT_IMAGE` target; both must parse this operator's argv, carry the lease backend, and run on CPU-only nodes); the snapshot refusals are server-side dry runs and the one backend the case persists selects no node | +| 78, 79, 93 | Scaling a role moves its queue's admitted quota by exactly one replica in each direction and touches no replica it did not add or remove; an instance of several Pods is admitted as ONE Kueue group with one PodSet of `size`; and a replica shed as surplus while it runs admitted releases Kueue's finalizer and takes its Workload with it | `pkg/worker/controllers/worker/model_deployment{,_pod_group,_rollout,_render,_service}.go`, `api/worker/v1alpha1/model_deployment.go` (`size`) | yes (confirm) | any (no GPU) + an InstanceType, the pool's entrance LocalQueue in ``, and room in the pool for four Pods of it - two instances of two, plus the scale legs. Optionally `E2E_MD_INSTANCE_TYPE` / `E2E_MD_IMAGE` / `E2E_MD_SETTLE` / `E2E_C78_HOLD` / `E2E_C78_BOUND` | | 80 | The node advertises exactly the RDMA endpoints its inventory says it has, sized per mode | `pkg/deviceplugin/rdma_{server,devices,endpoint}.go`, `pkg/nodefeature/rdma.go`, `pkg/devicemanager/detector/network*.go` | yes (confirm) | a node running a device manager with at least one RDMA endpoint. The correspondence and SR-IOV checks additionally need a host-probe Pod mounting `/sys` read-only, which a `restricted` PodSecurity namespace refuses — there those checks skip and the count checks still run. The SR-IOV checks also need virtual functions configured and the unhealthy-token check a `Failed` link already present; each skips individually and the case never induces either | | 81 | A granted RDMA endpoint opens inside an ordinary container, and an ungranted one does not | `pkg/deviceplugin/rdma_allocate.go`, `pkg/deviceplugin/rdma_endpoint.go` | yes (confirm) | as CASE 80 plus a WHOLE-FUNCTION endpoint (the shared key is served from those). The verbs half needs `E2E_RDMA_IMAGE` shipping `ibv_devinfo` — the image is probed, not trusted, and a plain base image makes that half skip naming the packages; the open() halves need only a shell, because `exec <>` is an open | | 82 | An accelerator and an RDMA endpoint in one container land on one NUMA node, or the container is refused | `pkg/deviceplugin/rdma_endpoint.go` (the NUMA hint), `pkg/deviceplugin/server.go` (the accelerator hint) | yes (confirm) | a node whose accelerators straddle its RDMA endpoints' NUMA nodes AND whose kubelet reports `single-numa-node` or `restricted` from its own configz, plus an accelerated InstanceType with an entrance LocalQueue. All three or the case answers nothing. The partition observation additionally needs a card already in a partitioning mode and is a RECORDING that can never pass — no machine yet satisfies its three properties at once | @@ -142,44 +124,30 @@ Each case is self-contained; its header (see **Case header contract**) states go | 89 | A ModelDeployment role's rendered Pod requests the selected fabric device | `api/worker/v1alpha1/model_deployment.go`, `pkg/worker/controllers/worker/model_deployment_{render,fabric}.go`, `pkg/worker/kvcache/mooncake/member_workload.go` (`FabricDeviceResource`), `pkg/worker/webhooks/worker/model_deployment.go` | no | an already reconciled ModelDeployment fixture for each requested count and protocol; inspect Pods even when Pending | | 90 | A ModelDeployment metrics snapshot reflects served traffic and Pod endpoints | `pkg/worker/extensionapis/worker/model_deployment.metrics.go`, `pkg/worker/controllers/worker/model_deployment_{render,router}.go`, `pkg/worker/kvcache/router/**` | yes | a serving ModelDeployment under representative traffic, and a pullable curl probe image; the probe Pod is removed by a trap | | 91 | A routed prefill/decode pair moves real KV bytes over its direct leg, and writes the store when it has one | `pkg/worker/kvcache/inject/{vllm,sglang,types}.go`, `pkg/worker/controllers/worker/model_deployment_render.go`, `cases/_serving-lib.sh` | no | a Ready vLLM or SGLang P/D ModelDeployment with a router on real accelerators, one Pod per half on two nodes, with or without a bound store; on NVIDIA a tcp leg, vLLM 0.29.0 (an older CUDA image embeds a client without the pin and exits 2), and no role setting `MC_FORCE_TCP` or an SGLang cache switch; a store on tcp and on the client's minor line, and for SGLang a prefill node with more than 10 GiB of host memory to spare beyond its host pool; plus `E2E_PD_PROBE_IMAGE` and `E2E_PD_MODEL`; the probe Pod is removed by a trap | -| 93 | A replica shed as surplus while it runs admitted releases Kueue's finalizer and takes its Workload with it, the replica beside it keeps its Pod and its Workload, and a twin in the seated group is removed without touching either — against a baseline showing an admitted Pod nobody releases stays held | `pkg/worker/controllers/worker/model_deployment{,_pod_group}.go` | yes (confirm) | any (no GPU, no engine image) + an InstanceType and, in ``, the pool's entrance LocalQueue; room for three single-replica placeholder Pods. Optionally `E2E_MD_INSTANCE_TYPE` / `E2E_MD_IMAGE`, `E2E_C93_HOLD` / `E2E_C93_BOUND` | | 94 | Scaling the router or a role under continuous traffic drops no request and puts the new Pod to work | `pkg/worker/controllers/worker/model_deployment{,_router,_pod_group}.go`, `pkg/worker/kvcache/router/**`, `cases/_serving-lib.sh` | yes (confirm) | a Ready routed ModelDeployment on real accelerators whose target (`router`, `prefill`, `decode` or `server`) runs one replica, room for one more, `E2E_PD_PROBE_IMAGE` and `E2E_PD_MODEL`; optionally `E2E_SCALE_READY_BOUND`, `E2E_SCALE_SETTLE`, `E2E_SCALE_LOG_DIR`; the trap restores the replica count and removes the probe Pod | -| 95 | A change on a waiting replica's Workload alone reaches the deployment's QuotaReserved condition within a bounded window, with no Pod of the deployment moving — against a control showing a Pod event makes the status quote it | `pkg/worker/controllers/worker/model_deployment.go` (the `Watches(&kueue.Workload{})` clause, `mapModelDeploymentWorkload`, `modelDeploymentWorkloadPredicate`), `pkg/worker/controllers/worker/model_deployment_status.go` | yes (confirm) | any (no GPU, no engine image) + an InstanceType and, in ``, the pool's entrance LocalQueue; the case sizes one role one replica past the quota of every flavor of the pool summed, so the pool must hold that many placeholder Pods' worth of objects, and SKIPs past `E2E_C95_MAX_REPLICAS` (default 128). Optionally `E2E_MD_INSTANCE_TYPE` / `E2E_MD_IMAGE`, `E2E_C95_WINDOW` | | 96 | A logical slice skips a fragmented node through the per-card fit labels, and livelocks there with the pin switched off | `pkg/worker/controllers/worker/{node_fit_label,fit_demand,node_devices_admission,node_devices_inflight}.go`, `pkg/worker/webhooks/worker/workload.go`, `pkg/nodefeature/fit.go`, `pkg/worker/settings/value.go` | yes (confirm) | two plain schedulable nodes | | 97 | A ModelArtifact resolves against the real Hub to the commit git reports, with the digest an independent script computes; a private and a gated repository are AccessDenied without a token and resolve with one; a rejected token is reported; a revoked token stops new Pods and leaves the running one; a referenced artifact waits in Terminating; the token is in no status, event or worker log | `pkg/modelartifact/**`, `pkg/worker/controllers/worker/model_artifact.go`, `pkg/worker/webhooks/worker/model_artifact.go`, `pkg/worker/controllers/worker/model_artifact_placement.go` | yes (confirm) | any (no GPU) + a worker that reaches the Hub, an InstanceType and, in ``, the pool's entrance LocalQueue; git, curl and python3 on this machine; `HF_TOKEN_READONLY` for the private rows, which SKIP without it. Optionally `E2E_C97_PRIVATE` / `E2E_C97_GATED` / `E2E_C97_REVOKE_BOUND` / `E2E_SYSTEM_NS` | -| 98 | ModelArtifact consumers: a claim deployment and an Instance get the PV's node affinity and run on its node; an unplaceable claim creates no Pod and says why; a replica that bound its claim is not rolled; an Engine Pod carries the pinned argv, a secretKeyRef token, a manifest-sized cache and a raised ephemeral-storage limit; admission refuses what would override the artifact | `pkg/worker/controllers/worker/{model_artifact_placement,model_deployment_artifact,instance}.go`, `pkg/worker/webhooks/worker/{model_deployment,instance}.go` | yes (confirm) | two schedulable workers (else SKIP) + a default StorageClass provisioning on the first Pod's node (kind's local-path), a worker that reaches the Hub, an InstanceType and the pool's entrance LocalQueue in ``. Creates one hostPath PV and two StorageClasses, all removed by the trap. Optionally `E2E_MD_INSTANCE_TYPE` / `E2E_MD_IMAGE` | +| 98, 105 | ModelArtifact consumers: a claim deployment and an Instance get the PV's node affinity and run on its node, an unplaceable claim creates no Pod and says why, an Engine Pod carries the pinned argv and a manifest-sized cache, and admission refuses what would override the artifact; and an Instance's persistent claims place it where the volume can attach | `pkg/worker/controllers/worker/{model_artifact_placement,model_deployment_artifact,instance,instance_persistent_volume}.go`, `pkg/worker/webhooks/worker/{model_deployment,instance}.go`, `pkg/worker/settings/value.go` | yes (confirm) | two schedulable workers (else SKIP: one node gives TAS no wrong node to pick) + a default StorageClass provisioning on the first Pod's node (kind's local-path, else that row SKIPs), a worker that reaches the Hub, an InstanceType and the pool's entrance LocalQueue in ``. Creates two hostPath PVs, two StorageClasses and a placeholder Pod, all removed by the trap; flips `instance-persistent-volume-placement` and restores it | | 99 | ModelArtifact serving: vLLM loads a claim artifact behind `llm-d-router` and serves, lists and labels only `spec.model.name` with the token producer never failing; SGLang downloads a gated artifact with its token at the resolved commit, tokenizer included, inside its size-limited cache | `pkg/worker/controllers/worker/{model_deployment_artifact,model_deployment_render,model_artifact_placement}.go` | yes (confirm) | two single-card replicas of one accelerator InstanceType (`E2E_C99_IT`) at once, a default StorageClass, nodes and worker reaching the Hub, `HF_TOKEN_READONLY` reading `E2E_C99_GATED` at `E2E_C99_GATED_COMMIT`; `E2E_PD_PROBE_IMAGE` sends the requests | | 100 | The weight identity in the KV store key: a second deployment of one artifact takes the writer's prompts from the store, a deployment of another commit on the same Binding takes none, and prompts nobody sent take nothing | `pkg/worker/kvcache/inject/{vllm,types}.go`, `pkg/worker/controllers/worker/{model_artifact_placement,model_deployment}.go` | yes (confirm) | two single-card replicas of `E2E_C100_IT` at once, a node for the store member (`E2E_C100_MEMBER_NODE`), nodes reaching the Hub; creates a cluster-scoped KVCacheBackend and KVCachePool, so hold the cluster lock | -| 101 | Node delivery: a cold mount by two Pods downloads each file once, a third Pod downloads nothing, a restricted namespace mounts, forged attributes are refused by their rules, a corrupting source is never published, a plugin killed mid-download publishes nothing early and resumes, a real Hub repository mounts, and the token is in no output | `pkg/modelmanager/**`, `pkg/worker/controllers/worker/{model_deployment_artifact,model_artifact_placement,node_model_store}.go`, `deploy/gpustack-operator/chart/templates/model-manager/**`, `cases/_model-hub{-lib.sh,.py}` | yes (confirm) | two workers, the chart with `modelManager.enabled`, the stock python image, and nodes reaching the public Hub (`E2E_C101_OFFLINE=1` SKIPs that row) | -| 102 | Only a node's plugin writes its NodeModelStore status and only the worker a ModelArtifact's, each refusal by its rule beside the admitted writer; a Setting reaches every node's spec and generation; Node delivery is refused without the CSIDriver; an HTTPS proxy carries every download, against a control without it | `pkg/worker/webhooks/worker/{node_model_store,model_artifact_status}.go`, `pkg/worker/controllers/worker/node_model_store.go`, `pkg/worker/settings/model_store.go`, `pkg/modelmanager/{report,download}/**`, `cases/_model-hub{-lib.sh,.py}` | yes (confirm; deletes and restores the CSIDriver) | two workers, the chart with `modelManager.enabled`, the stock python image, openssl on this machine. Optionally `E2E_MODEL_MANAGER_SA` / `E2E_SYSTEM_NS` | -| 103 | The node cache's lifecycle: a plugin roll leaves a mounted Pod reading and a mount asked for during it succeeds; collection removes the oldest unreferenced digest and never a referenced one; a filtered artifact downloads only its files and is held under Engine delivery; an Instance and a ModelDeployment mount a hub artifact | `pkg/modelmanager/{gc,store,driver,report}/**`, `pkg/worker/controllers/worker/{instance,model_artifact_placement,model_deployment_artifact}.go`, `deploy/gpustack-operator/chart/templates/model-manager/**`, `cases/_model-hub{-lib.sh,.py}` | yes (confirm; mounts a tmpfs on a kind node) | kind with two workers and docker reaching the node containers (else the collection rows SKIP), the chart with `modelManager.enabled`, an InstanceType, the stock python image; about fifteen minutes. Optionally `E2E_MD_INSTANCE_TYPE` / `E2E_MD_IMAGE` | +| 101, 102 | Node delivery: a cold mount by two Pods downloads each file once, a third Pod downloads nothing, a restricted namespace mounts, forged attributes are refused by their rules, a corrupting source is never published, a plugin killed mid-download publishes nothing early and resumes, a real Hub repository mounts, and the token is in no output; and only a node's plugin writes its NodeModelStore status and only the worker a ModelArtifact's | `pkg/modelmanager/**`, `pkg/worker/controllers/worker/{model_deployment_artifact,model_artifact_placement,node_model_store}.go`, `pkg/worker/webhooks/worker/{node_model_store,model_artifact_status}.go`, `pkg/worker/settings/model_store.go`, `deploy/gpustack-operator/chart/templates/model-manager/**`, `cases/_model-hub{-lib.sh,.py}` | yes (confirm; deletes and restores the CSIDriver) | two workers, the chart with `modelManager.enabled`, the stock python image, openssl on this machine, and nodes reaching the public Hub (`E2E_C101_OFFLINE=1` SKIPs that row) | +| 103, 109 | The node cache's lifecycle: a plugin roll leaves a mounted Pod reading and a mount asked for during it succeeds, collection removes the oldest unreferenced digest and never a referenced one, and a filtered artifact downloads only its files; and a plugin restart or the object's deletion each end with the node's entries rewritten from disk | `pkg/modelmanager/{gc,store,driver,report}/**`, `pkg/worker/controllers/worker/{instance,model_artifact_placement,model_deployment_artifact,node_model_store,node_model_store_kubelet}.go`, `deploy/gpustack-operator/chart/templates/model-manager/**`, `cases/_model-hub{-lib.sh,.py}` | yes (confirm; mounts a tmpfs on a kind node and deletes a worker's Node object) | kind with two workers and docker reaching the node containers (else the collection and Node rows SKIP), the chart with `modelManager.enabled`, an InstanceType, the stock python image; about fifteen minutes | | 104 | Serving from the node cache: vLLM serves a 7B model its node downloaded, verified and published once, under spec.model.name, and the replica that replaces it mounts the published tree with zero bytes downloaded; the download's wall time and duration are recorded | `pkg/modelmanager/**`, `pkg/worker/controllers/worker/{model_deployment_artifact,model_artifact_placement}.go` | yes (confirm) | a real accelerator: one InstanceType (`E2E_C104_IT`) with one free card, about 30 GB free in the node's cache filesystem, nodes reaching the Hub, `model-artifact-delivery-mode` Node, and the pool's entrance LocalQueue in ``. Optionally `E2E_C104_MODEL` / `E2E_C104_READY_BOUND` / `E2E_PD_PROBE_IMAGE` | -| 105 | An Instance's persistent claims place it where the volume can attach: a workspace and an additional-volume claim run on their PV's node on every start; with that node full the Workload waits without reserving quota and then runs there; a WaitForFirstConsumer claim binds on the first Pod's node; an unbound immediate claim creates no Pod and says why; the escape Setting restores the old render | `pkg/worker/controllers/worker/{instance,instance_persistent_volume,model_artifact_placement}.go`, `pkg/worker/settings/value.go` | yes (confirm) | two schedulable workers (else SKIP: one node gives TAS no wrong node to pick) + for the WaitForFirstConsumer row a default StorageClass provisioning on the first Pod's node (kind's local-path, else that row SKIPs), an InstanceType whose pool covers the workers and its entrance LocalQueue in ``. Creates two hostPath PVs, two StorageClasses and a placeholder Pod, all removed by the trap; flips `instance-persistent-volume-placement` and restores it. Optionally `E2E_MD_INSTANCE_TYPE` / `E2E_MD_IMAGE` | -| 106 | With `instance-type-derived-from-node` off, a queue born in that mode references the joint check, and a P/D deployment on one administrator-authored InstanceType is admitted as a set: the role that fits a short pool stays gated with its joint check naming the other | `pkg/worker/controllers/worker/{node_queue,model_deployment_joint_admission}.go`, `pkg/worker/webhooks/worker/model_deployment.go` | yes (confirm) | as CASE 50, plus the `gpustack-settings` Secret in `E2E_SYSTEM_NS` (the case sets the key off and restores it) and an InstanceType whose general group, OS and arch select a CPU pool; it runs `case-50.sh` on a type it authors and reports a SKIP there as a SKIP | | 107 | Placement preference: a node-delivered replica lands on the node holding its digest when it fits, measured against an exact chance baseline; it is admitted elsewhere at once when that node is full; a change in which nodes hold the digest rolls nothing; an Instance carries the preference too; no Pod carrying the preference asks for a preferred topology level | `pkg/worker/controllers/worker/{model_placement_preference,model_artifact_placement,model_deployment,instance}.go`, `deploy/gpustack-operator/chart/values.yaml` (Kueue's `TASRespectNodeAffinityPreferred`), `cases/_model-hub{-lib.sh,.py}` | yes (confirm) | two or more workers of one CPU model running the plugin (a mixed pool is cut to its largest same-model group, since TAS ranks nodes only inside the flavor Kueue picked; four, as in `references/kind-two-zones.yaml`, gives the sharpest baseline), the chart with `modelManager.enabled`, an InstanceType (run case-1 first), the stock python image; about 20 minutes | -| 108 | Download progress: a node writes downloadedBytes on 30 s / 5% thresholds and nothing at rest, the artifact's status.nodes moves in steps of 5, the v1 progress subresource reads the download live, only to its namespace's subjects with the subresource rule and naming no node, no NodeModelStore names a tenant, and the v1 views write nothing as the worker | `pkg/modelmanager/{report,materialize,download}/**`, `pkg/modelmanager/downloads.go`, `pkg/worker/controllers/worker/model_artifact{,_nodes}.go`, `pkg/worker/extensionapis/worker/{model_artifact,model_artifact.progress,node_model_store}.go`, `cases/_model-hub{-lib.sh,.py}` | yes (confirm) | a worker, the chart with `modelManager.enabled`, the stock python image; about ten minutes (a three-minute download, then a five-minute quiet window) | -| 109 | The NodeModelStore's lifecycle: a plugin restart and a deletion of the object each end with the node's entries rewritten from disk, and deleting a Node collects its object while the node registered again gets a new one | `pkg/modelmanager/{report,store}/**`, `pkg/worker/controllers/worker/node_model_store{,_kubelet}.go` | yes (confirm; deletes a worker's Node object and restarts its kubelet) | kind with two workers and docker reaching the node containers (else the Node row SKIPs), the chart with `modelManager.enabled`, the stock python image | -| 110 | ModelPrefetch: the store layer names itself in the node's spec; a warm-up pod pinned to the target node mounts label-free and the digest goes Ready; a projection past the grant is refused at admission; pinning is refused without `allowPinned` and lands under it; deleting the prefetch removes its pod and its pin while the tree stays for the grace | `pkg/worker/controllers/worker/{model_prefetch,model_store,model_store_binding}.go`, `pkg/worker/webhooks/worker/{model_store_binding,model_prefetch}.go`, `pkg/worker/settings/value.go`, `cases/_model-hub{-lib.sh,.py}` | yes (confirm) | one worker labeled `e2e.gpustack.ai/warm-pool=true` by the case (case-110 labels and unlabels it), the chart with `modelManager.enabled`, the stock python image; docker reaching the node container for the tree row | +| 108, 110, 114 | Download progress: a node writes downloadedBytes on 30 s / 5% thresholds and nothing at rest, the artifact's status.nodes moves in steps of 5, the v1 progress subresource reads the download live and only to its namespace's subjects with the subresource rule, and the v1 views write nothing as the worker; ModelPrefetch names the store layer in the node's spec and refuses a projection past the grant; and an expected digest resolves, refuses a foreign anchor and mounts from the peer | `pkg/modelmanager/{report,materialize,download}/**`, `pkg/modelmanager/downloads.go`, `pkg/modelmanager/peer/client.go`, `pkg/worker/controllers/worker/model_artifact{,_nodes}.go`, `pkg/worker/controllers/worker/{model_prefetch,model_store,model_store_binding}.go`, `pkg/worker/extensionapis/worker/{model_artifact,model_artifact.progress,node_model_store}.go`, `pkg/worker/webhooks/worker/{model_artifact,model_store_binding,model_prefetch}.go`, `api/worker/v1alpha1/model_artifact.go`, `cases/_model-hub{-lib.sh,.py}` | yes (confirm) | a worker, the chart with `modelManager.enabled`, the stock python image; two workers for the peer leg (`E2E_OFFLINE=1` skips that leg); about ten minutes (a three-minute download, then a five-minute quiet window) | | 111 | Node-to-node sync: a cold node materializes a digest from a peer's published tree with source=Peer and its hub byte counter at zero; the seed's plugin pod dying mid-pull still ends Ready (checkpoint resume); a tenant pod cannot reach the peer port | `pkg/modelmanager/peer/**`, `pkg/modelmanager/materialize/materialize.go`, `pkg/modelmanager/config.go`, `deploy/gpustack-operator/chart/templates/model-manager/**`, `docs/modules/model-delivery/peer-sync.md`, `specs/2026-09-27-model-peer-sync.md` | yes (confirm) | two schedulable workers (a seed and a puller), the chart installed with `modelManager.port` non-zero and the peer NetworkPolicy enabled, `model-store-peer-sync` at its `true` default | | 112 | Image source: a tag-only reference is refused naming the digest contract; an image artifact resolves claim-shaped (no manifest digest, no revision, no `status.nodes`); an Instance pinned to the node mounts the pinned image read-only through an image volume and reads the fixture byte for byte; a `ModelPrefetch` naming the artifact is refused | `pkg/worker/webhooks/worker/{model_artifact,model_prefetch}.go`, `pkg/worker/controllers/worker/{model_artifact_placement,model_deployment_artifact,instance,model_placement_preference}.go`, `pkg/kubediscovery/feature.go`, `cases/case-112.sh` | yes (confirm) | one worker whose kubelet/containerd serve image volumes (kubelet 1.35+, containerd 2.1+), docker on the runner, the stock `registry:2`, `python:3.12-slim` and `busybox:1.36` images pullable | | 113 | ModelScope source: a `modelScope` artifact resolves to the commit `git ls-remote` also names and materializes on a cold node hashing to the hub's own sha256; a second node pulls from the peer with its hub byte counter flat; a `modelscope` SDK at the floor downloads at the resolved commit; a well-formed `modelScope` source is admitted and a three-part repository refused | `pkg/modelartifact/modelscope.go`, `pkg/worker/controllers/worker/model_artifact.go`, `pkg/modelmanager/{driver/authorize.go,materialize/materialize.go,report/report.go}`, `pkg/worker/webhooks/worker/model_artifact.go`, `cases/case-113.sh` | yes (confirm) | two schedulable workers, the chart with `modelManager.enabled`, the stock python image pullable, and an internet path to www.modelscope.cn and pypi.org from the nodes and the runner (`E2E_C113_OFFLINE=1` skips) | -| 114 | Expected digest: a match resolves with `digestSource Hub`, a foreign anchor is refused `DigestMismatch` naming both digests, a twice-unreachable hub writes the anchor as the identity (`digestSource Expected`, no revision) with the hub's request log flat, `Engine` delivery holds both anchored artifacts with `AnchorNeedsNodeDelivery` and no Pod, and an `Expected` identity mounts on a second node from the peer, hub log still flat | `api/worker/v1alpha1/model_artifact.go`, `pkg/worker/webhooks/worker/model_artifact.go`, `pkg/worker/controllers/worker/{model_artifact,model_artifact_placement,model_deployment}.go`, `pkg/modelmanager/materialize/materialize.go`, `pkg/modelmanager/peer/client.go`, `cases/case-114.sh` | yes (confirm) | one schedulable worker for the anchor legs, two for the peer leg (`E2E_C114_OFFLINE=1` skips that leg), the chart with `modelManager.enabled`, the stock python image pullable, one InstanceType for the engine-block rows; no internet — the hub is the case's own test hub | | 115 | Ordinary role updates replace whole groups: initially queued resources without a replacement slot, size growth and shrink, whole-card/slice resources, takeover command, optional cross-pool InstanceType, and P/D sibling identity | `pkg/worker/controllers/worker/model_deployment{,_initial_queue,_replacement_slot,_pod_group,_render,_service}.go`, `pkg/worker/webhooks/worker/model_deployment.go`, `cases/case-115.sh` | yes (confirm) | an idle NVIDIA pool with onceMaxRequest=1; two free whole cards for the size leg; logical slice capacity for slice edits; a different accelerator group for the cross-pool leg; a CPU type for P/D size and command checks; no engine inference | -| 116 | Every encoder-role admission refusal fires, each naming what its rule refuses | `pkg/worker/webhooks/worker/model_deployment_epd.go`, `pkg/worker/webhooks/worker/model_deployment.go` (the kind rules) | yes (confirm) | any (no GPU, no engine image; run case-1 first) + one Ready schedulable worker node and the `gpustack-settings` Secret in `E2E_SYSTEM_NS`; the refusals are server-side dry runs, and the fixture injects a CPU vendor label through a NodeFeature and flips CPU-manufacturer awareness on, both restored by the trap. AUTO-SKIPS its matrix rows when the vendor label does not take (NFD owns `cpu-model.vendor_id` and may decline an injected override), printing why; the shape rows then still run | -| 117 | The encoder's own Service exists, and the front door never points at it | `pkg/worker/controllers/worker/model_deployment_{service,status}.go`, `pkg/worker/extensionapis/worker/model_deployment.go`, `pkg/worker/webhooks/worker/model_deployment_epd.go` | yes (confirm) | as CASE 116's fixture; the replicas are placeholders that never serve and the router image may be unpullable — the endpoint row records NO-READ then, and the selector rows carry the front-door rule | +| 116, 122, 123, 124 | Every encoder-role admission refusal fires, each naming what its rule refuses: the vllm-router and sglang-gateway pairings, the admitted engine rows, the operator-owned `--epd-config` refusal, the prefill/decoder shape rules, and the platform the matrix does not admit | `pkg/worker/webhooks/worker/model_deployment_epd.go`, `pkg/worker/webhooks/worker/model_deployment.go` (the kind rules), `pkg/worker/controllers/worker/model_deployment_connector_epd.go` (the router rows), `pkg/nodefeature/knowns.go` (the declared manufacturers), `api/worker/v1alpha1/model_deployment.go` (the claim source) | yes (confirm) | any (no GPU, no engine image; run case-1 first) + one Ready schedulable worker node and the `gpustack-settings` Secret in `E2E_SYSTEM_NS`; the refusals are server-side dry runs, and the fixture injects a CPU vendor label through a NodeFeature and flips CPU-manufacturer awareness on, both restored by the trap. AUTO-SKIPS its matrix rows when the vendor label does not take (NFD owns `cpu-model.vendor_id` and may decline an injected override), printing why; the shape rows then still run | +| 117, 119, 125 | The encoder's own Service exists and the front door never points at it; the shared claim mounts the same files in the encoder's and the prefiller's Pods; and the llm-d-router picker carries the encoder stage, read against a control deployment with no encoder | `pkg/worker/controllers/worker/model_deployment_{service,status,render}.go`, `pkg/worker/extensionapis/worker/model_deployment.go`, `pkg/worker/webhooks/worker/model_deployment_epd.go`, `pkg/worker/kvcache/router/router.go` (the encode filter, profile, decider and exclusion), `api/worker/v1alpha1/model_deployment.go` (the claim source) | yes (confirm) | as CASE 116's fixture plus `E2E_C117_WITNESS_IMAGE` (default busybox:1.36) pullable for the runtime half, the pool's entrance LocalQueue in ``, and `pyyaml` in the runner to read the router ConfigMap; the replicas are placeholders that never serve and the router image may be unpullable - the endpoint row records NO-READ then. The case creates its own hostPath PV and PVC (removed by the trap) | | 118 | A short pool starves an encoder/prefill/decode deployment WHOLE: all three groups wait together | `pkg/worker/controllers/worker/model_deployment{,_joint_admission,_pod_group}.go`, `pkg/worker/kuberess/apps_kueue_admission_check.go`, `pkg/worker/webhooks/worker/model_deployment_epd.go` | yes (confirm) | as CASE 116's fixture plus Kueue pod integration and the pool's entrance LocalQueue in ``; the queue is CPU-only, and the case skips, printing its numbers, when the pool cannot be shorted to a width that fits two of the three groups or would need more than `E2E_C118_MAX_REPLICAS` (default 128) placeholder replicas | -| 119 | The shared claim mounts the same files in the encoder's and the prefiller's Pods | `pkg/worker/controllers/worker/model_deployment_render.go` (the additionalVolumes render), `pkg/worker/webhooks/worker/model_deployment_epd.go` (the shared-claim rules), `api/worker/v1alpha1/model_deployment.go` (the claim source) | yes (confirm) | as CASE 116's fixture plus `E2E_C119_WITNESS_IMAGE` (default busybox:1.36) pullable for the runtime half — that row skips naming the image when it cannot pull; the case creates its own hostPath PV and PVC (removed by the trap), so no StorageClass is needed | | 120 | The admitted network rows run: llm-d-router + vLLM on NVIDIA CUDA or AMD ROCm with the NIXL encoder-cache leg, and a multimodal request is answered through the router | `pkg/worker/kvcache/inject/vllm_ec.go`, `pkg/worker/controllers/worker/model_deployment_connector_epd.go`, `pkg/worker/webhooks/worker/model_deployment_epd.go` (the matrix rows) | yes (confirm) | a pool of 2+ NVIDIA CUDA or AMD ROCm accelerators, an InstanceType (`E2E_C120_INSTANCE_TYPE`) reporting nvidia or amd with `E2E_C120_CARDS` (default 3) free exclusive cards, weights at a hostPath, a VISION-LANGUAGE model (a text-only model never engages the encoder stage, so every row would pass with the leg dormant), the matching engine image (default the CUDA runner on nvidia and the ROCm runner on amd, `E2E_C120_IMAGE`) and the llm-d-router image pullable, `E2E_C120_PROBE_IMAGE` pullable, and the pool's entrance LocalQueue in ``; auto-skips on any other platform, on too few free cards, and on an unpullable engine or router image (naming it — the router image is watched during the endpoint wait, because the router's own Deployment is what publishes the endpoint). The endpoint is polled for `E2E_C120_READY_BOUND` seconds: it is a PASS when it appears, a FAIL when the bound is spent with all three role Pods AND the router Pod Ready (the controller writes it once the router's own Deployment reports a ready replica), and a SKIP naming the missing Pods when the bound is spent without them. Overrides `model-deployment-router-image` and `model-deployment-routing-sidecar-image` in the `gpustack-settings` Secret, with the trap armed before the first patch; the restore happens before the deployment is deleted and is waited for on the router Deployment's image (`E2E_C120_SETTLE_BOUND`, default 180) | | 121 | The admitted Ascend row runs: llm-d-router + vLLM with the shared-directory encoder-cache leg over one ReadWriteMany claim, and a multimodal request is answered through the router | `pkg/worker/kvcache/inject/vllm_ec.go`, `pkg/worker/controllers/worker/model_deployment_connector_epd.go`, `pkg/worker/webhooks/worker/model_deployment_epd.go` (the matrix row and the claim rules) | yes (confirm) | real Ascend, an InstanceType (`E2E_C121_INSTANCE_TYPE`) reporting ascend with `E2E_C121_CARDS` (default 3) free exclusive cards, weights at a hostPath, a VISION-LANGUAGE model, the vllm-ascend and llm-d-router images and `E2E_C121_PROBE_IMAGE` pullable, the pool's entrance LocalQueue in ``, and an existing ReadWriteMany claim in `` (`E2E_C121_CLAIM`, required — this row's two ends are two Pods, and a ReadWriteOnce claim binds one node's disk); auto-skips on any other platform, on too few free cards, and on an unpullable image. A claim that is not Bound or does not offer ReadWriteMany is a SKIP ROW, not a failure: it is a storage answer about the cluster and nothing about the operator was exercised. The endpoint follows CASE 120's contract. The claim is the CALLER's and is never deleted; the router and sidecar image settings are overridden and restored as in CASE 120 | -| 122 | The admitted SGLang rows are admitted and the pairing the matrix does not carry is refused: llm-d-router + SGLang and sglang-gateway + SGLang, mounting no claim, are ACCEPTED; sglang-gateway + vLLM is REFUSED naming the router AND the engine it is not paired with | `pkg/worker/controllers/worker/model_deployment_connector_epd.go` (the six SGLang rows), `pkg/worker/webhooks/worker/model_deployment_epd.go` (the matrix gap and the floor project) | no | any cluster with a materialized scheduling chain; if an active InstanceType already reports `E2E_C122_VENDOR` (default `Ascend`; also supports `nvidia` or `amd`) it is used directly without mutating node labels or settings, otherwise the case injects the CPU vendor label and flips CPU-manufacturer awareness on as in CASE 116. Every assertion is a server-side dry run, so nothing is stored. Auto-skips the platform rows when no InstanceType for the target manufacturer can be resolved | -| 123 | vllm-router admits vLLM encoder deployments on NVIDIA and AMD, refuses SGLang pairings, refuses user-supplied `--epd-config` as operator-owned, and admits plain prefill/decode without an encoder | `pkg/worker/controllers/worker/model_deployment_connector_epd.go` (the two vllm-router rows), `pkg/worker/webhooks/worker/{model_deployment,model_deployment_epd}.go` | yes | any cluster with a materialized scheduling chain; if an active InstanceType already reports `nvidia` or `amd` it is used directly without mutating node labels or settings, otherwise the case injects `E2E_C123_VENDOR` (default `NVIDIA`) and flips CPU-manufacturer awareness on as in CASE 116. Every assertion is a server-side dry run | -| 124 | The matrix admits ascend, nvidia and amd only: the refusal names the platform beside the router and engine it accepted and the admitted rows, a second refusal names the engine instead, and two roles on two platforms are refused by the mixing rule | `pkg/worker/webhooks/worker/model_deployment_epd.go` (the matrix gap and the mixed-platform rule), `pkg/nodefeature/knowns.go` (the declared manufacturers), `pkg/worker/controllers/worker/model_deployment_connector_epd.go` (the admitted rows) | no | any cluster with a materialized scheduling chain and at least one usable InstanceType whose reported manufacturer is one this operator declares and the matrix does not admit — it iterates every declared accelerator manufacturer minus ascend, nvidia and amd (cambricon, hygon, iluvatar, metax, mthreads, thead) and takes the first usable type, so no supported accelerator is left out by a guess; the accepted baseline shape follows whichever admitted platform it finds, carrying the shared-claim mounts on ascend and none on nvidia or amd (the network rows forbid the shared mount); auto-skips the unadmitted rows naming the manufacturers it looked for and the types it saw on a cluster of admitted platforms only. MOCKS nothing — no injected label, no settings change, no cleanup | -| 125 | The llm-d-router picker carries the encoder stage: an encode filter, an unscored encode profile, the multimodal decider, and an event producer whose `podLabelSelector` excludes both eventless kinds — read against a control deployment with no encoder | `pkg/worker/kvcache/router/router.go` (the encode filter, profile, decider and exclusion), `pkg/worker/controllers/worker/model_deployment_connector_epd.go` | yes (confirm) | as CASE 116's fixture plus the pool's entrance LocalQueue in `` (without it nothing is admitted and no router document is rendered), and `pyyaml` in the runner to read the router ConfigMap. The case creates its own hostPath PV and PVC (removed by the trap). Waits for each deployment's own role count of Pods — all three for the encoder, two for the control. An unpullable ROUTER image does not skip the rows — they read the ConfigMap, which the operator renders before any image is pulled | -| 126 | A ModelDeployment without `engine.image` is refused by the API server's schema before any webhook runs, in either measured wording: "Required value" while the key is absent, "Invalid value" once the defaulting patch materializes it empty; the accepted baseline proves the refusal is about the missing field, and a role image does not substitute for it | `api/worker/v1alpha1/model_deployment.go` (the required engine image), `cases/case-126.sh` | no | any cluster with the ModelDeployment CRD installed; every assertion is a server-side dry run; auto-skips the accepted-baseline row when no usable InstanceType reports an admitted platform | +| 126, 130 | A ModelDeployment without `engine.image` is refused by the API server's schema before any webhook runs, in either measured wording; and the SGLang External DP shape is REFUSED naming `RouterRequired` when no `spec.router` is declared, while the three controls are each ACCEPTED without a Router | `api/worker/v1alpha1/model_deployment.go` (the required engine image), `pkg/worker/webhooks/worker/model_deployment.go` (`validateModelDeploymentExternalDPRouter`), `pkg/worker/controllers/worker/model_deployment_parallelism.go` (the engine-aware External vote) | no | any cluster with the ModelDeployment CRD installed; every assertion is a server-side dry run, so no accelerator, no engine image and no GPU are needed and nothing is stored | | 127 | A managed role's Pod mounts its own subdirectory of the host compilation cache at one constant in-container path, is pointed at it by the engine's cache-root variable, and is given a startup gate scaled by the budget the role declares, while a replica still inside its startup window is held out of `Revoked`; schema validation bounds `startupTimeoutSeconds` to `[10, 86400]`, and admission allows a user `additionalVolumes` mount at the compilation cache path (yielding the default `HostPath` volume) | `pkg/worker/controllers/worker/model_deployment_compile_cache.go`, `pkg/worker/controllers/worker/model_deployment_render.go` (the cache mount, user volume yield, and startup-gate scaling), `pkg/worker/controllers/worker/model_deployment_health.go`, `api/worker/v1alpha1/model_deployment.go` (`startupTimeoutSeconds`), `cases/case-127.sh` | yes (confirm) | any cluster with the ModelDeployment CRD installed for the admission rows (server-side dry runs); a usable InstanceType and the pool's entrance LocalQueue in `` for the rendered-Pod rows, else they auto-skip; the warm-restart rows additionally `E2E_C127_ENGINE_IMAGE` and `E2E_C127_MODEL` on a real accelerator pool, plus a pullable `E2E_C127_PROBE_IMAGE` (default busybox) and a namespace whose PodSecurity admits a hostPath probe Pod, else they skip | | 128 | A ModelDeploymentScaler fills the fields it omits, refuses the shapes that could never commit anything, and holds a bound deployment's replica count against every writer but itself: the defaults are read back from a stored object, the refusals are owned one by one (the schema for the bounds it states, the webhook for the signal it names and the roles it plans for), a shadow scaler binds the target and changes nothing, a scaler may not be pointed at another target, a second scaler on the same target is refused the binding, a user may write anything of a bound deployment but not its count and not the two annotations that record who owns it, a suspended scaler gives the binding up and the count becomes writable again, and the scaler that comes back writes a higher term | `api/worker/v1alpha1/model_deployment_scaler.go` (the spec), `pkg/worker/webhooks/worker/model_deployment_scaler.go` (the defaulting and the refusals), `pkg/worker/webhooks/worker/model_deployment.go` (the single-writer fence), `pkg/worker/controllers/worker/model_deployment_scaler.go` (the binding and its term), `cases/case-128.sh` | yes (confirm) | any cluster with the ModelDeploymentScaler CRD installed; the defaulting rows and the refusals that name no target run everywhere, and the rows that read a target need a usable InstanceType reporting an admitted platform (ascend, nvidia or amd), because the webhook reads the type a role names, else they auto-skip naming what is missing; the elastic-EP refusal needs a target whose role the ModelDeployment webhook admits, and is a SKIP ROW when that target is refused on the cluster. The target rows use a pause image (`E2E_MD_IMAGE`, default registry.k8s.io/pause:3.10) that is never pulled | | 129 | A ModelDeploymentScaler acting on a live target reads the engine's running-request counter, raises the replica count under load, lowers it when the scaler is narrowed, and the count that goes down is retired through the target's retirement protocol; the new replica's own request counter grows while the heavy phase runs, and every phase answers requests and fails none | `pkg/worker/controllers/worker/model_deployment_scaler_sampler.go` (the counter), `pkg/worker/controllers/worker/model_deployment_scaler_planner.go` (the arithmetic), `pkg/worker/scaling/algorithm/proportional.go` (the recommendation), `pkg/worker/controllers/worker/model_deployment_scaler.go` (the commit), `pkg/worker/controllers/worker/model_deployment.go` (the retirement), `cases/case-129.sh` | yes (confirm) | a serving ModelDeployment: either one the caller names (`case-129.sh `) or one the case builds, which needs a real engine image (`E2E_C129_ENGINE_IMAGE`) and real weights at a host path (`E2E_C129_WEIGHTS`); a pullable probe image (`E2E_C129_PROBE_IMAGE`, default curlimages/curl:8.10.1); and the phase list (`E2E_C129_PHASES`, default "base:2:30 up:24:90 two:24:90 down:2:120 one:2:60"). The whole leg auto-skips naming what is missing, because a scaler that reads a number nobody produced is not the thing under test. The load runs from a probe Pod inside the cluster, because the signal is read per pod, and the case never uses `kubectl port-forward` | -| 130 | SGLang External DP at admission: the external shape — attention width at least two, a node count of at least two and the Rust server on — is REFUSED naming `RouterRequired` when no `spec.router` is declared, and ACCEPTED with `sglang-gateway` or with `llm-d-router`; the three controls (attention width one, one node, the Rust server off) are each ACCEPTED without a Router, which proves the refusal is the external-shape rule and not a blanket rejection of SGLang multi-Pod roles | `pkg/worker/webhooks/worker/model_deployment.go` (`validateModelDeploymentExternalDPRouter`), `pkg/worker/controllers/worker/model_deployment_parallelism.go` (the engine-aware External vote), `cases/case-130.sh` | no | any cluster with the ModelDeployment CRD installed; every assertion is a server-side dry run, so no accelerator, no engine image and no GPU are needed and nothing is stored | Each note below is something the **lead** must act on before or around a run. What a case *does* — its goal, environment, inputs, assertions and cleanup — lives in its own header, which the **Case header contract** below requires to be readable on its own; the index never restates it. @@ -226,14 +194,14 @@ Each note below is something the **lead** must act on before or around a run. Wh fails on a timeout that reads like a placement defect. Worse where a ceiling comes from the pool — CASE 11 reads its over-commit limit from `Status.Detail.Memory` — because a mismatched pool with larger cards makes a real over-commit pass vacuously. -- **`spec.os`/`spec.arch` has no case of its own** — CASE 1 (cpu pool) and CASE 6 (accelerated) assert it inline. +- **`spec.os`/`spec.arch` has no case of its own** — CASE 1 (cpu pool) and CASE 4 (accelerated) assert it inline. - **CASE 4 is safe on a real-accelerator cluster**: its mock uses a fake product key (`nvidia-e2emock`) that never collides with a real GPU pool, and it writes its ledger only on a node that has none and reports no `nvidia.com/gpu`, auto-skipping when there is no such node. -- **CASES 23–32, 34 and 39 need a node address you must ask the user for.** 23–32 and 34 read `MIG_NODE_SSH=`; CASE 39 reads `PPU_NODE_SSH=`, and only when it is not already running on the node. Each **exits 2 (input required)**, going no further, when its address is unset — ask for it and pass it inline at run time, never hardcoded. All **auto-skip (exit 0)** when the hardware itself is missing. +- **CASES 23–29, 32, 34 and 39 need a node address you must ask the user for.** 23–29, 32 and 34 read `MIG_NODE_SSH=`; CASE 39 reads `PPU_NODE_SSH=`, and only when it is not already running on the node. Each **exits 2 (input required)**, going no further, when its address is unset — ask for it and pass it inline at run time, never hardcoded. All **auto-skip (exit 0)** when the hardware itself is missing. - **Vendor cases carry a vendor image**, because every in-container reading is that vendor's own tool: CASE 35 a CANN-family image (a bare one exits 127 on an Ascend slice), CASE 38 a ROCm-family image, CASE 39 any image shipping `ppu-smi`. Each takes an `E2E_*_IMAGE=` override, and all are large — pre-pull on the node, or the first claim spends its whole wait pulling. - **CASE 23 owns the whole MIG mode transition** — enabling MIG moves a card out of the logical family and into the partition one entirely — and is self-recovering: its trap restores the card's original mode on pass AND fail. -- **CASES 24–32 and 34 are the hardware-partition family**, sharing `cases/_partition-lib.sh` (node correlation, the SSH gate, profile/key discovery, pod plumbing, the `record` idiom). Its leading underscore marks it **not a case**; reuse it when adding another partition case rather than copying the discovery block. +- **CASES 24–25, 27–29, 32 and 34 are the hardware-partition family**, sharing `cases/_partition-lib.sh` (node correlation, the SSH gate, profile/key discovery, pod plumbing, the `record` idiom). Its leading underscore marks it **not a case**; reuse it when adding another partition case rather than copying the discovery block. - Each partitions a card only if none is partitioned yet and restores exactly the card it toggled, so one up-front `nvidia-smi -i 0 -mig 1` plus a Device Manager rollout restart lets most of them run back to back — but **CASE 27 requires the opposite** and skips if any card is already partitioned. That ordering — **27 first, then 24, then the rest** — is what `run-partition-block.sh` encodes; drive the family with it. - - Optional environment, all with defaults: `MIG_NODE_NAME`, `MIG_NODE_SSH_OPTS`, `MIG_GPU_INDEX` (0), `MIG_SSH_TIMEOUT` (90), `IMAGE`, `MIG_MIXED_INDEXES` / `MIG_MIXED_ROUNDS` (CASE 24), `MIG_WRITE_IDLE_WINDOW` (CASE 25), `MIG_MAX_FILL` (CASE 27), `F11_IMAGE` / `F11_EXPECT` (CASE 28), `MIG_RECLAIM_BOUND` (CASE 31), `MIG_OOB_WINDOW` (CASE 32). + - Optional environment, all with defaults: `MIG_NODE_NAME`, `MIG_NODE_SSH_OPTS`, `MIG_GPU_INDEX` (0), `MIG_SSH_TIMEOUT` (90), `IMAGE`, `MIG_MIXED_INDEXES` / `MIG_MIXED_ROUNDS` (CASE 24), `MIG_WRITE_IDLE_WINDOW` (CASE 25), `MIG_MAX_FILL` (CASE 27), `F11_IMAGE` / `F11_EXPECT` (CASE 28), `MIG_RECLAIM_BOUND` (CASE 29), `MIG_OOB_WINDOW` (CASE 32). - **CASE 24 is the headline regression guard** for the failure the two-family split exists to remove: a single token pool used to let the kubelet hand a partition request a token from a card that cannot be partitioned, and the Pod died with a terminal `UnexpectedAdmissionError`. - **CASE 14 and CASES 11/22/41 want opposite POOL shapes.** CASE 14's over-budget assertion needs a pool of a **single** logically sliceable card — with a free sibling the third slice simply lands there @@ -244,18 +212,18 @@ Each note below is something the **lead** must act on before or around a run. Wh skipping — check which pool it picked before reading the skip as a hardware verdict. What genuinely cannot be arranged is hiding a card: the `*_VISIBLE_DEVICES` variables are what the allocators inject into a workload container, not a filter on what the device manager detects. -- **CASE 40's accelerator sub-check cannot execute on its own Instance.** The case creates a **CPU** +- **CASE 37's accelerator sub-check cannot execute on its own Instance.** The case creates a **CPU** Instance on the general pool, which holds no accelerator, so the accelerator-family label set — including `mode` — always skips, on any hardware. Reading it needs an accelerated Instance, and no case does it on this surface: **CASE 14** asserts the `mode` spelling on the Instance metrics subresource, but the Prometheus accelerator-label path is **uncovered** — CASE 41 and CASE 37 read neither `mode` nor those labels. -- **CASE 40's "exactly one target" only bites on a multi-vendor node.** A node running one device +- **CASE 37's "exactly one target" only bites on a multi-vendor node.** A node running one device manager satisfies it trivially; the rule it guards — that a node's Instances are published by one device manager, never by each — is only exercised where two manufacturers are present. Read a PASS there as "not double-counted on this node", not as "the election was tested". -- **CASES 25, 31, 32 and 34 record observations, not thresholds** — accepted consequences with stated containment, measured and printed as a copyable block for the design record; each still carries one hard assertion where a real regression would hide. **CASE 28 prints the same block but is a guard**, and only an explicit `F11_EXPECT=observe` demotes its verdict to a recording. -- **CASES 49–51 carry a placeholder image, and it is plumbing rather than a mock.** A ModelDeployment +- **CASES 25, 29, 32 and 34 record observations, not thresholds** — accepted consequences with stated containment, measured and printed as a copyable block for the design record; each still carries one hard assertion where a real regression would hide. **CASE 28 prints the same block but is a guard**, and only an explicit `F11_EXPECT=observe` demotes its verdict to a recording. +- **CASES 45, 49 and 50 carry a placeholder image, and it is plumbing rather than a mock.** A ModelDeployment role names no image of its own; the operator synthesizes one from what the InstanceType has observed, and a CPU-only InstanceType has observed no accelerator — so on the cluster these three are meant to run on, every role would be refused by the CONTROLLER for a reason none of them is @@ -263,7 +231,7 @@ Each note below is something the **lead** must act on before or around a run. Wh `image`, `E2E_MD_IMAGE` overrides it, and no assertion in any of the three reads anything a replica runs. Cases 47 and 48 take the same escape with a real Mooncake image, because theirs read the rendered Pod spec. -- **Run CASES 49–51 in a namespace that carries the pool's entrance LocalQueue**, which is what routes +- **Run CASES 45, 49 and 50 in a namespace that carries the pool's entrance LocalQueue**, which is what routes a group into its ClusterQueue — `kubectl -n get localqueue` before deciding. It is not created everywhere: on a fresh docker-desktop the operator materialized one in `default`, `kube-public` and `kube-node-lease` and **none in `gpustack-system`**, which is the namespace the Flow's `$NS` holds. @@ -419,7 +387,7 @@ kubectl -n "$NS" rollout status deploy/gpustack-operator-worker --timeout=300s bash .claude/skills/gpustack-operator-e2e/cases/case-1.sh "$NS" 2>&1 | tee "$RPT"/raw/10-case1.txt # mandatory, read-only # then, per the picked cases (each confirmed), e.g.: bash .claude/skills/gpustack-operator-e2e/cases/case-2.sh "$NS" 2>&1 | tee "$RPT"/raw/11-case2.txt -# … case-3 … case-9 (case-8 auto-skips without real accelerators) +# … case-4 … case-9 (case-8 auto-skips without real accelerators) ``` Read the PASS/FAIL table; do not re-derive from raw output. @@ -463,7 +431,7 @@ Re-check the credential first — a run this long outlives a short-lived cloud t - `../_e2e-lib/references/orchestration.md` — the shared multi-specialist flow: roles, phases, rendezvous rules, report layout, the context-checkpoint discipline (with a worked focus block), and the fix-and-retest loop. - `../_e2e-lib/references/cluster-provisioning.md` — where the cluster comes from: bring-your-own first, the three modalities and their cost asymmetry, the credential-is-a-teardown-dependency rule, node preparation, and the destroy obligation. -- `references/drain-recycle.md` — why CASE 2–6 need a real cluster (the fake-client blind spots), the managed-toggle code path, and the accelerated mock recipes (fake accelerator NodeFeature + a node-named `Devices` ledger, patched on the **v1alpha1** CRD, + the charging Pod the node-devices check reads). +- `references/drain-recycle.md` — why CASES 2, 4 and 5 need a real cluster (the fake-client blind spots), the managed-toggle code path, and the accelerated mock recipes (fake accelerator NodeFeature + a node-named `Devices` ledger, patched on the **v1alpha1** CRD, + the charging Pod the node-devices check reads). - `references/rdma-host-shapes.md` — which host shape answers which RDMA reading, why each gate is written as a positive discriminating reading rather than as "the cluster has RDMA", the one reading no machine in reach answers, and the environment variables and image contents CASES 80–85 need. It also carries the **hardware tables to borrow or buy against**: every property stated as a command and the answer that qualifies, never as a part number, so the list survives a hardware refresh and matches what the cases' own gates read. - `references/manual-ssh-verification.md` — the manual pass CASE 21 cannot drive in CI: a real VS Code Remote-SSH session (workspace opens, integrated terminal in `main`) and an `sshfs` mount round-trip. - `../_e2e-lib/references/troubleshooting.md` — shared image/rollout/teardown failure modes. diff --git a/.agents/skills/gpustack-operator-e2e/cases/_epd-lib.sh b/.agents/skills/gpustack-operator-e2e/cases/_epd-lib.sh new file mode 100644 index 000000000..6e2ea4f4a --- /dev/null +++ b/.agents/skills/gpustack-operator-e2e/cases/_epd-lib.sh @@ -0,0 +1,558 @@ +#!/usr/bin/env bash +# +# _epd-lib.sh — the CPU-vendor fixture a platform case stands on, and the readers a live +# three-role serving group is inspected through. +# +# NOT A CASE. The leading underscore keeps it out of the `case-N.sh` namespace: it carries no case +# header, no trap and no results table. Every one of those stays with the case that sources it. +# What it removes is the duplication between the cases that prove the same two things: the fixture +# that manufactures a CPU platform on a cluster that has none, and the inspection of a live +# encoder/prefill/decode group. +# +# A case uses it as: +# +# set -uo pipefail +# NS="${1:?usage: ...}" +# # shellcheck source=/dev/null +# . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_epd-lib.sh" +# +# # the fixture: a CPU platform when the cluster carries none +# NODE="$(epd_pick_worker_node)" +# epd_setup_cpu_vendor_fixture "$VENDOR" "$NS" "$NF" "$SETTLE_ROUNDS" "$SYSTEM_NS" "$NODE" +# IT="$EPD_FIXTURE_TYPE" +# ...checks, each `record`ing one row... +# epd_teardown_cpu_vendor_fixture "$NS" "$NF" "$SETTLE_ROUNDS" "$SYSTEM_NS" cleanup_deployments +# +# # a live serving group +# epd_poll_role "$NS" "$MD" "$READY_BOUND" encode; rc=$? +# DOC="$(epd_ec_document "$NS" "$EN_POD")" +# +# ------------------------------------------------------------------------------------------------ +# PART ONE: THE CPU-VENDOR FIXTURE +# +# A case that asserts something about a platform needs a platform. On a cluster that carries one, +# the case reads it. On a CPU-only cluster it has to manufacture one: the operator derives an +# InstanceType from the CPU vendor a node reports, so a case can inject that vendor through a +# NodeFeature object and turn on the setting that makes the worker read it. +# +# The fixture is three mutations and their undo: a NodeFeature object carrying the vendor label on +# one node, the CPU-manufacturer awareness setting flipped on, and the InstanceType that follows +# from them. The undo is not a delete of the settings key but a patch of the exact bytes that were +# there, and it is not finished until the type this run derived is gone: the worker reads the +# setting through a thirty-second cache, and the type outlives the patch by that cache plus one +# reconcile. A case that returned while its type was still there would hand the next case in a +# serial run a platform it did not choose. +# ------------------------------------------------------------------------------------------------ + +# The settings key the operator reads the CPU-manufacturer awareness from. One key serves every +# case that builds the fixture, so it is written once here. +EPD_AWARENESS_KEY=instance-type-aware-cpu-manufacturer + +# The fixture's own state. EPD_FIXTURE_INJECTED says whether this run changed anything at all. +# EPD_FIXTURE_ORIG_SETTING holds the bytes the settings Secret carried before, still encoded, so +# the restore can put back exactly what was there. EPD_FIXTURE_TYPES holds the InstanceTypes this +# run derived, and the settle waits for those to go. +EPD_FIXTURE_INJECTED=false +EPD_FIXTURE_ORIG_SETTING="" +EPD_FIXTURE_TYPES=() +# shellcheck disable=SC2034 # Output consumed by the sourcing case. +EPD_FIXTURE_TYPE="" + +# epd_pick_worker_node prints the name of one Ready, schedulable worker node (not a control-plane +# one): the vendor label lands there and that node's CPU becomes the pool whose manufacturer the +# case controls. +epd_pick_worker_node() { + kubectl get nodes -o json 2>/dev/null | python3 -c ' +import json,sys +for n in json.load(sys.stdin).get("items",[]): + cs=n.get("status",{}).get("conditions",[]) + if not any(c.get("type")=="Ready" and c.get("status")=="True" for c in cs): + continue + if n.get("spec",{}).get("unschedulable"): + continue + # A CPU-vendor probe only reads back on a node that feeds the CPU-only catalog. An + # acceleratable node feeds the GPU InstanceType instead, whose manufacturer is the + # accelerator vendor, so an injected cpu-model.vendor_id never surfaces here and the + # platform-type poll below times out. Skip those nodes, exactly as case-16 reads the + # CPU pool (acceleratable=false) rather than the accelerator one. + if n.get("metadata",{}).get("labels",{}).get("feature.gpustack.ai/acceleratable") == "true": + continue + ts=n.get("spec",{}).get("taints",[]) + if any(str(t.get("key","")).startswith("node-role.kubernetes.io/") for t in ts): + continue + print(n["metadata"]["name"]) + break +' +} + +# epd_platform_type prints the InstanceType reporting that manufacturer, skipping +# types on their way out and Inactive ones for the reasons case-45 records: naming either is +# refused at admission for a reason no row here is about. +# +# The manufacturer is read from status, not from spec: the platform of a role is whatever its +# InstanceType reports, which is the same fact the webhook reads. A type reporting nothing is not +# a platform of its own — that is the not-yet-synced state, and the webhook refuses it. +epd_platform_type() { # + kubectl get instancetypes.worker.gpustack.ai \ + -o jsonpath='{range .items[*]}{.metadata.name}|{.metadata.deletionTimestamp}|{.spec.inactive}|{.status.detail.manufacturer}{"\n"}{end}' \ + 2>/dev/null \ + | while IFS='|' read -r name deleting inactive manufacturer; do + [ -n "$name" ] || continue + [ -z "$deleting" ] || continue + [ "$inactive" = true ] && continue + [ -n "$manufacturer" ] || continue + if [ "$manufacturer" = "$1" ]; then + echo "$name" + break + fi + done +} + +# epd_type_exists succeeds while the InstanceType is still there. +epd_type_exists() { kubectl get instancetypes.worker.gpustack.ai "$1" >/dev/null 2>&1; } + +# epd_any_type_exists ... prints the first of the named InstanceTypes that is still +# there, and fails when none of them is. +epd_any_type_exists() { + local t + for t in "$@"; do + [ -n "$t" ] || continue + if epd_type_exists "$t"; then + printf '%s' "$t" + return 0 + fi + done + return 1 +} + +# epd_set_awareness turns CPU-manufacturer awareness on or off in the +# settings Secret. The value rides the Secret base64-encoded (case-50's mechanic). +epd_set_awareness() { # + kubectl -n "$1" patch secret gpustack-settings --type=merge \ + -p "{\"data\":{\"${EPD_AWARENESS_KEY}\":\"$(printf '%s' "$2" | base64)\"}}" >/dev/null 2>&1 +} + +# epd_restore_setting puts the setting back exactly as it was: a +# patch of the prior bytes rather than a delete of the key, because a key the cluster shipped +# absent is indistinguishable from one a user set. +epd_restore_setting() { # + local v=null + [ -n "$2" ] && v="\"$2\"" + kubectl -n "$1" patch secret gpustack-settings --type=merge \ + -p "{\"data\":{\"${EPD_AWARENESS_KEY}\":${v}}}" >/dev/null 2>&1 +} + +# epd_settle_back ... +# +# Restoring the value is not the end of the restore: the worker reads it through a thirty-second +# cache, and the type derived from it outlives the patch by that cache plus one reconcile. Wait for +# the derived type to go, so the next case in a serial run does not list an aware type this run +# left behind (case-16's wait_generic, on this same fixture). A cluster that started with the +# setting already on keeps its derived type, so there is nothing to wait for and the wait is +# skipped. +epd_settle_back() { # ... + local orig="$1" rounds="$2" + shift 2 + [ "$(printf '%s' "$orig" | base64 -d 2>/dev/null)" = "true" ] && return 0 + for _ in $(seq 1 "$rounds"); do + if epd_any_type_exists "$@" >/dev/null; then + sleep 5 + continue + fi + return 0 + done + return 0 +} + +# epd_setup_cpu_vendor_fixture [platform] +# +# Builds the fixture. The InstanceType that reported the platform is left in EPD_FIXTURE_TYPE, +# empty when none appeared within the rounds. +# +# The call must not be made in a command substitution: the fixture's state — the settings value it +# found, the types it derived and the fact that it changed anything at all — is left in this +# shell, and a subshell would take it with it, leaving the teardown with nothing to undo. +# +# The label rides a NodeFeature object (case-4's mechanism): the object stays and keeps +# re-asserting the label, where a bare `kubectl label` is a cpuid-owned value NFD reverts on its +# next pass. The same object carries a second vendor when a case needs one, so one delete restores +# both. +# +# The poll IS the wait for the settings cache (30s), the node flavor re-derivation and the type +# authoring. A timeout is a skip, not a failure: NFD owns cpu-model.vendor_id and may decline an +# injected override, which is an environment answer the case reports rather than argues with. +# +# The platform waited for defaults to the lowercased vendor; a case that injects one vendor and +# waits for another names the platform it waits for. +epd_setup_cpu_vendor_fixture() { # [platform] + local vendor="$1" ns="$2" nf="$3" rounds="$4" system_ns="$5" node="$6" + local platform="${7:-}" type="" + [ -n "$platform" ] || platform="$(printf '%s' "$vendor" | tr '[:upper:]' '[:lower:]')" + + # The settings value is read before the first patch, and only the first time: a second vendor on + # the same object must not capture the value this run already changed. + if [ "$EPD_FIXTURE_INJECTED" != true ]; then + EPD_FIXTURE_ORIG_SETTING="$(kubectl -n "$system_ns" get secret gpustack-settings \ + -o jsonpath="{.data.${EPD_AWARENESS_KEY}}" 2>/dev/null)" + fi + EPD_FIXTURE_INJECTED=true + + kubectl apply -f - >/dev/null 2>&1 < [cleanup-hook] +# +# Removes the fixture: the NodeFeature object is deleted, the settings value the fixture found is +# put back exactly as it was, and the types this run derived are waited for. +# +# The cleanup hook runs FIRST, so a case can delete the ModelDeployments it made while the platform +# they were admitted on still exists: the type goes with the label, and a group whose platform has +# gone is one the operator can no longer serve. +# +# Nothing is changed when no fixture was set up. +epd_teardown_cpu_vendor_fixture() { # [cleanup-hook] + local ns="$1" nf="$2" rounds="$3" system_ns="$4" hook="${5:-}" + if [ -n "$hook" ]; then + "$hook" + fi + [ "$EPD_FIXTURE_INJECTED" = true ] || return 0 + kubectl -n "$ns" delete nodefeatures.nfd.k8s-sigs.io "$nf" --ignore-not-found >/dev/null 2>&1 + epd_restore_setting "$system_ns" "$EPD_FIXTURE_ORIG_SETTING" + if [ "${#EPD_FIXTURE_TYPES[@]}" -gt 0 ]; then + epd_settle_back "$EPD_FIXTURE_ORIG_SETTING" "$rounds" "${EPD_FIXTURE_TYPES[@]}" + else + epd_settle_back "$EPD_FIXTURE_ORIG_SETTING" "$rounds" + fi + return 0 +} + +# ------------------------------------------------------------------------------------------------ +# PART TWO: THE LIVE EPD SERVING HELPERS +# +# A three-role encoder/prefill/decode group is read from the outside: the Pods it rendered, the +# router that fronts it, the connector document the engine was given and the claim its roles +# mount. The readings are the same in every case that runs one, and the ways they fail are quiet: +# a Pod that never became Ready, a router image that would not pull, a connector handed a directory +# where the platform wanted a network leg. +# +# The image override belongs to the same reading. The router and the sidecar are images the +# operator chooses from the settings Secret, so a case that wants a specific one pins it there and +# puts the prior bytes back afterwards; the restore is finished only when the router Deployment has +# been re-rendered without it. +# ------------------------------------------------------------------------------------------------ + +# The bytes the settings Secret carried for the two image settings, kept encoded so the restore can +# put back exactly what was there, and the flag that says the pin ran at all. +EPD_ROUTER_ORIG="" +EPD_SIDECAR_ORIG="" +EPD_IMAGES_PINNED=false + +# epd_cluster_registry prints the registry the operator prefixes onto every image it +# chooses, empty when the cluster sets none. +epd_cluster_registry() { # + kubectl -n "$1" get secret gpustack-settings \ + -o jsonpath='{.data.container-registry}' 2>/dev/null | base64 -d 2>/dev/null +} + +# epd_registry_relative prints the image with its registry host dropped. +# +# The operator prefixes `container-registry` onto every image it chooses, and the two settings +# below take a reference of the form `namespace/name:tag`. A value that already names a host would +# render as `//...` on a cluster that sets a registry, so drop the host there. +epd_registry_relative() { # + case "${1%%/*}" in + *.*|*:*|localhost) printf '%s' "${1#*/}" ;; + *) printf '%s' "$1" ;; + esac +} + +# epd_set_image_setting writes a value into the settings Secret. +epd_set_image_setting() { # + kubectl -n "$1" patch secret gpustack-settings --type=merge \ + -p "{\"data\":{\"$2\":\"$(printf '%s' "$3" | base64)\"}}" >/dev/null 2>&1 +} + +# epd_restore_image_setting puts a value back exactly as it +# was: a patch of the prior bytes rather than a delete of the key, because a key the cluster +# shipped at its default is indistinguishable from one a user set, and deleting the former would +# leave this run's baseline changed for every case after it. +epd_restore_image_setting() { # + local v=null + [ -n "$3" ] && v="\"$3\"" + kubectl -n "$1" patch secret gpustack-settings --type=merge \ + -p "{\"data\":{\"$2\":${v}}}" >/dev/null 2>&1 +} + +# epd_pin_router_sidecar_images +# +# Pins the router and the routing sidecar into the settings Secret. +# +# The bytes the Secret carried before are read first and kept encoded in EPD_ROUTER_ORIG and +# EPD_SIDECAR_ORIG, so the restore can put back exactly what was there. +epd_pin_router_sidecar_images() { # + local system_ns="$1" router_setting="$2" sidecar_setting="$3" router_image="$4" sidecar_image="$5" + EPD_IMAGES_PINNED=true + EPD_ROUTER_ORIG="$(kubectl -n "$system_ns" get secret gpustack-settings \ + -o jsonpath="{.data.${router_setting}}" 2>/dev/null)" + EPD_SIDECAR_ORIG="$(kubectl -n "$system_ns" get secret gpustack-settings \ + -o jsonpath="{.data.${sidecar_setting}}" 2>/dev/null)" + epd_set_image_setting "$system_ns" "$router_setting" "$router_image" + epd_set_image_setting "$system_ns" "$sidecar_setting" "$sidecar_image" +} + +# epd_restore_router_sidecar_images +# +# Puts the two image settings back exactly as they were, from the bytes the pin read. Nothing is +# written when no image was pinned. +epd_restore_router_sidecar_images() { # + [ "$EPD_IMAGES_PINNED" = true ] || return 0 + epd_restore_image_setting "$1" "$2" "$EPD_ROUTER_ORIG" + epd_restore_image_setting "$1" "$3" "$EPD_SIDECAR_ORIG" +} + +# epd_router_image prints the image the router Deployment runs. +epd_router_image() { # + kubectl -n "$1" get deploy "$2-router" \ + -o jsonpath='{.spec.template.spec.containers[*].image}' 2>/dev/null +} + +# epd_settle_router_image +# +# Restoring the bytes is not the end of the restore: the worker reads settings through a cache and +# re-renders the router Deployment by the value, so a case that returned while the router still ran +# this case's image would hand the next one a router it did not choose. Wait for the restore to be +# READ BACK, which is what the Deployment's own image shows. +# +# Returns 1 when the bound was spent with the router still on this case's image: the restore was +# written but the worker has not read it back. Cleanup cannot record a row from here, so the case +# says it on stderr. +epd_settle_router_image() { # + local ns="$1" deployment="$2" bound="$3" image="$4" + local deadline=$((SECONDS + bound)) rendered + while [ "$SECONDS" -lt "$deadline" ]; do + rendered="$(epd_router_image "$ns" "$deployment")" + # No router was ever rendered — an apply that was refused, or a run that never reached it — so + # nothing carries this case's image and there is nothing to wait for. + [ -n "$rendered" ] || return 0 + # The restored value is not always comparable: a key the cluster shipped absent is restored by + # deleting it, and what the Deployment then carries is the operator's own default, which this + # case never knew. "No longer this case's image" is the condition both spellings satisfy. + case "$rendered" in + *"$image"*) ;; + *) return 0 ;; + esac + sleep 5 + done + return 1 +} + +# epd_pull_reason prints what any container of the role's +# Pods is waiting on, joined; empty when none is waiting on an image. +epd_pull_reason() { # + kubectl -n "$1" get pods -l "app.kubernetes.io/instance=$2,app.kubernetes.io/component=$3" \ + -o jsonpath='{range .items[*]}{.status.containerStatuses[*].state.waiting.reason}{" "}{end}' 2>/dev/null +} + +# epd_pod_pull_reason prints what the Pod's first container is waiting on. A Pod +# outside the deployment's own labels — a witness Pod of a case's own making — is read by name. +epd_pod_pull_reason() { # + kubectl -n "$1" get pod "$2" \ + -o jsonpath='{.status.containerStatuses[0].state.waiting.reason}' 2>/dev/null +} + +# epd_wait_pods +# +# Prints the role's Pod name once it is Ready, and returns 0 on arrival, 2 when an image would not +# pull, 1 once the bound is spent. +# +# A pull failure is read on every round, because it is the one answer that is cheaper to give than +# to wait for: a container nobody could start is an environment answer, not a verdict about the +# operator, and the case turns it into a skip naming the image. +epd_wait_pods() { # + local ns="$1" deployment="$2" bound="$3" comp="$4" + local deadline=$((SECONDS + bound)) pods ready p + while [ "$SECONDS" -lt "$deadline" ]; do + pods="$(kubectl -n "$ns" get pods \ + -l "app.kubernetes.io/instance=${deployment},app.kubernetes.io/component=${comp}" \ + -o json 2>/dev/null)" + ready="$(printf '%s' "$pods" | jq -r \ + '[.items[] | select([.status.conditions // [] | .[] | select(.type=="Ready" and .status=="True")] | length > 0)] | length')" + if [ "${ready:-0}" -ge 1 ]; then + printf '%s' "$pods" | jq -r '.items[0].metadata.name' + return 0 + fi + for p in ImagePullBackOff ErrImagePull; do + case "$(epd_pull_reason "$ns" "$deployment" "$comp")" in + *"$p"*) return 2 ;; + esac + done + sleep 15 + done + return 1 +} + +# epd_poll_role +# +# Polls one role and returns the wait code: 0 on arrival, 2 when an image would not pull, 1 on a +# timeout. The Pod name is left in EN_POD, PF_POD or DC_POD, the variable the role owns. +# +# Each role is polled separately and the exit code is read IMMEDIATELY, because a pull failure on +# the first role means the other two polls would sit out the whole bound before anything reported +# it. +epd_poll_role() { # + local ns="$1" deployment="$2" bound="$3" comp="$4" pod rc varname="" + pod="$(epd_wait_pods "$ns" "$deployment" "$bound" "$comp")"; rc=$? + case "$comp" in + encode) varname=EN_POD ;; + prefill) varname=PF_POD ;; + decode) varname=DC_POD ;; + esac + if [ -n "$varname" ]; then + printf -v "$varname" '%s' "$pod" + fi + return "$rc" +} + +# epd_router_pods prints the router's own Pods as JSON. +# +# The router's Pods are selected by the router Deployment's OWN selector rather than by a label +# spelled here. The router carries the same instance label the roles carry, plus a router key of +# its own, so an instance-only selector would return the role Pods too and learn nothing about the +# router; the Deployment's selector is the one that cannot drift from what was rendered. +epd_router_pods() { # + local sel + # shellcheck disable=SC2016 # the dollar names are kubectl jsonpath map variables, and must not expand + sel="$(kubectl -n "$1" get deploy "$2-router" \ + -o go-template='{{range $k,$v := .spec.selector.matchLabels}}{{$k}}={{$v}},{{end}}' 2>/dev/null | sed 's/,$//')" + [ -n "$sel" ] || return 0 + kubectl -n "$1" get pods -l "$sel" -o json 2>/dev/null +} + +# epd_router_pull_reason prints what any router container is WAITING on, +# joined; empty when none is waiting on an image. Read for the same two reasons the role Pods' own +# reasons are read. +epd_router_pull_reason() { # + epd_router_pods "$1" "$2" | jq -r \ + '[.items[]? | .status.containerStatuses[]? | .state.waiting?.reason // empty] | join(" ")' 2>/dev/null +} + +# epd_router_pod prints the Ready router Pod's name, or empty when the +# router is not up. Read once when the verdict is formed, because by then the bound is spent and +# the router's state is settled. +epd_router_pod() { # + epd_router_pods "$1" "$2" | jq -r \ + '[.items[]? | select([.status.conditions // [] | .[] | select(.type=="Ready" and .status=="True")] | length > 0) | .metadata.name] | if length > 0 then .[0] else "" end' 2>/dev/null +} + +# epd_wait_endpoint +# +# Prints the endpoint the deployment published and returns 0 on arrival, 2 when the router image +# would not pull (the same code the role wait returns for a role image), 1 once the bound is spent. +# +# The endpoint is written once the ROUTER's OWN Deployment reports a ready replica — it is the +# router's status the operator copies, not a fact about the role Pods — so a single read here +# races that writer by design, and the wait is bounded because the router is a fourth workload no +# role poll covers. +epd_wait_endpoint() { # + local ns="$1" deployment="$2" bound="$3" + local deadline=$((SECONDS + bound)) ep p + while [ "$SECONDS" -lt "$deadline" ]; do + ep="$(kubectl -n "$ns" get modeldeployments.worker.gpustack.ai "$deployment" \ + -o jsonpath='{.status.endpoint}' 2>/dev/null)" + if [ -n "$ep" ]; then + printf '%s' "$ep" + return 0 + fi + for p in ImagePullBackOff ErrImagePull; do + case "$(epd_router_pull_reason "$ns" "$deployment")" in + *"$p"*) return 2 ;; + esac + done + sleep 5 + done + return 1 +} + +# epd_ec_document prints the document the engine was handed. +# +# The document is the argument AFTER --ec-transfer-config, which is how the operator renders it: +# the flag has no environment equivalent, so it is always two argv entries. Reading the pair off +# the live argv rather than pattern-matching the JSON keeps the row a question about the flag. +epd_ec_document() { # + kubectl -n "$1" get pod "$2" -o json 2>/dev/null | jq -r \ + --arg f "--ec-transfer-config" \ + '[(.spec.containers[] | select(.name=="main") | (.command // []) + (.args // []))][0] // [] + | (index($f) // -1) as $i | if $i < 0 then "" else .[$i + 1] end' 2>/dev/null +} + +# epd_ec_env prints the value the engine was given for one variable. +epd_ec_env() { # + kubectl -n "$1" get pod "$2" -o json 2>/dev/null | jq -r \ + --arg n "$3" '[(.spec.containers[] | select(.name=="main") | .env // [])[] | select(.name==$n)][0].value // ""' 2>/dev/null +} + +# epd_ec_env_field_path prints the downward-API half of the same read: +# the field selector the kubelet resolves, not the value it resolved. Rendered by +# valueFrom.fieldRef.fieldPath, so the variable has no .value to read. +epd_ec_env_field_path() { # + kubectl -n "$1" get pod "$2" -o json 2>/dev/null | jq -r \ + --arg n "$3" '[(.spec.containers[] | select(.name=="main") | .env // [])[] | select(.name==$n)][0].valueFrom.fieldRef.fieldPath // ""' 2>/dev/null +} + +# epd_role_pod_claim prints the claim one role's FIRST Pod +# mounts, as "claimName|mountPath|readOnly". +# +# EVERY CONTAINER IS SEARCHED, not just the first. A direct decode renders the routing sidecar as +# the FIRST regular container, and reading that one would let the decoder's row pass whatever the +# engine mounts: the sidecar carries no mounts of its own, so a claim the decode role grew would +# still read as "no claim" there. Looking through all of them keeps every row a question about the +# whole Pod. +epd_role_pod_claim() { # + kubectl -n "$1" get pods \ + -l "app.kubernetes.io/instance=$2,app.kubernetes.io/component=$3" \ + -o json 2>/dev/null | python3 -c ' +import json,sys +items=json.load(sys.stdin).get("items",[]) +if not items: + print("") + raise SystemExit +pod=items[0] +claims={} +for v in pod["spec"].get("volumes",[]): + p=v.get("persistentVolumeClaim") + if p and p.get("claimName"): + claims[v["name"]]=p["claimName"] +for ctr in pod["spec"].get("containers",[]): + for m in ctr.get("volumeMounts",[]): + claim=claims.get(m.get("name","")) + if claim: + print(claim+"|"+m.get("mountPath","")+"|"+str(m.get("readOnly",False)).lower()) + raise SystemExit +print("")' +} diff --git a/.agents/skills/gpustack-operator-e2e/cases/_kvcache-ha-lib.sh b/.agents/skills/gpustack-operator-e2e/cases/_kvcache-ha-lib.sh new file mode 100644 index 000000000..c6a15b79b --- /dev/null +++ b/.agents/skills/gpustack-operator-e2e/cases/_kvcache-ha-lib.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +# +# _kvcache-ha-lib.sh — the shared queries of the HA KV cache cases: the leader's Lease, the member +# DaemonSet, the failover handoff and the backend-plus-Lease teardown. +# +# NOT A CASE. The leading underscore keeps it out of the `case-N.sh` namespace: it carries no case +# header, no trap and no results table; those stay with the case that sources it. +# +# # shellcheck source=/dev/null +# . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_kvcache-ha-lib.sh" +# +# Every helper takes what it needs as arguments and returns what the caller needs. Nothing here +# reaches into a case's variables, and nothing here prints the results table. + +# kvha_wait_for [secs] polls one jsonpath until it equals what is +# wanted, and prints the LAST value seen, so a timeout says what the object was doing. +kvha_wait_for() { + local ns="$1" kind="$2" name="$3" path="$4" want="$5" secs="${6:-180}" + local got="" i + for ((i = 0; i < secs; i += 3)); do + got="$(kubectl -n "$ns" get "$kind" "$name" -o "jsonpath=$path" 2>/dev/null)" + [ "$got" = "$want" ] && { echo "$got"; return 0; } + sleep 3 + done + echo "$got" + return 1 +} + +# kvha_teardown deletes the backend and then the Lease it leaves behind: the +# Lease carries no owner reference, so it goes after the backend. +kvha_teardown() { + local ns="$1" backend="$2" lease="$3" + kubectl delete kvcachebackends.worker.gpustack.ai "$backend" --ignore-not-found --wait=false >/dev/null 2>&1 || true + kubectl wait --for=delete "kvcachebackends.worker.gpustack.ai/${backend}" --timeout=120s >/dev/null 2>&1 || true + kubectl -n "$ns" delete leases.coordination.k8s.io "$lease" --ignore-not-found --wait=false >/dev/null 2>&1 || true +} + +# kvha_ready_leader_pod prints the one leader Pod that is Ready, or nothing. +kvha_ready_leader_pod() { + local ns="$1" leader_sel="$2" + kubectl -n "$ns" get pod -l "$leader_sel" \ + -o jsonpath='{range .items[?(@.status.containerStatuses[0].ready==true)]}{.metadata.name}{"\n"}{end}' \ + 2>/dev/null | head -1 +} + +# kvha_lease_holder prints the Lease's holderIdentity, empty when it is absent. +kvha_lease_holder() { + local ns="$1" lease="$2" + kubectl -n "$ns" get leases.coordination.k8s.io "$lease" \ + -o jsonpath='{.spec.holderIdentity}' 2>/dev/null +} + +# kvha_lease_renew_time prints the Lease's renewTime, empty when it is absent. +kvha_lease_renew_time() { + local ns="$1" lease="$2" + kubectl -n "$ns" get leases.coordination.k8s.io "$lease" \ + -o jsonpath='{.spec.renewTime}' 2>/dev/null +} + +# kvha_lease_transitions prints the Lease's handover count, zero when it is absent. +kvha_lease_transitions() { + local ns="$1" lease="$2" t + t="$(kubectl -n "$ns" get leases.coordination.k8s.io "$lease" -o jsonpath='{.spec.leaseTransitions}' 2>/dev/null)" + echo "${t:-0}" +} + +# kvha_holder_pod_uid prints the UID of the Pod the holderIdentity +# names: the holder string is the serving process's rpc address, so it names a Pod by its IP. +kvha_holder_pod_uid() { + local ns="$1" leader_sel="$2" holder="$3" + local holder_ip pod_uid pod_ip deleting phase + case "$holder" in + \[*\]:*) + holder_ip="${holder#\[}" + holder_ip="${holder_ip%%\]*}" + ;; + *:*) holder_ip="${holder%:*}" ;; + *) holder_ip="$holder" ;; + esac + while IFS='|' read -r pod_uid pod_ip deleting phase; do + if [ "$phase" != "Running" ] || [ -n "$deleting" ] || [ -z "$pod_ip" ] \ + || [ "$pod_ip" != "$holder_ip" ]; then + continue + fi + echo "$pod_uid" + return 0 + done < <(kubectl -n "$ns" get pod -l "$leader_sel" \ + -o jsonpath='{range .items[*]}{.metadata.uid}{"|"}{.status.podIP}{"|"}{.metadata.deletionTimestamp}{"|"}{.status.phase}{"\n"}{end}' \ + 2>/dev/null) +} + +# kvha_pod_uid prints one Pod's UID, empty when the Pod cannot be read. +kvha_pod_uid() { + local ns="$1" pod="$2" + kubectl -n "$ns" get pod "$pod" -o jsonpath='{.metadata.uid}' 2>/dev/null +} + +# kvha_member_snapshot prints the member Pods as name=UID:restarts, so a +# replacement Pod is caught by its UID. +kvha_member_snapshot() { + local ns="$1" member_sel="$2" + kubectl -n "$ns" get pod -l "$member_sel" \ + -o jsonpath='{range .items[*]}{.metadata.name}={.metadata.uid}:{.status.containerStatuses[0].restartCount}{"\n"}{end}' \ + 2>/dev/null | sort +} + +# kvha_members_mounted_sample prints one reading of the backend's own report, as +# phase|MembersMounted status/reason. +kvha_members_mounted_sample() { + local ns="$1" backend="$2" + kubectl -n "$ns" get kvcachebackends.worker.gpustack.ai "$backend" \ + -o jsonpath='{.status.phase}{"|"}{.status.conditions[?(@.type=="MembersMounted")].status}{"/"}{.status.conditions[?(@.type=="MembersMounted")].reason}{"\n"}' \ + 2>/dev/null +} + +# kvha_observe_handoff_snapshot +# watches the Lease until the handoff is observable and prints what it +# saw, as one pipe-separated record: +# +# moved-at|moved-holder|moved-holder-pod-uid|move-evidence|handoff-observed|new-holder| +# new-holder-pod-uid|new-ready|new-ready-uid|wait-secs +# +# The watch and the detection are one function because the detection is only meaningful between two +# reads of the same Lease: a changed holderIdentity or a changed renewTime timestamps the move +# directly, while a Ready holder is an upper bound. The deadline is the delete epoch plus 240s. +kvha_observe_handoff_snapshot() { + local ns="$1" backend="$2" lease="$3" leader_sel="$4" traj_file="$5" + local delete_epoch="$6" old_ready_uid="$7" old_holder="$8" + local new_holder="" new_holder_pod_uid="" new_ready="" new_ready_uid="" new_renew_time="" + local reused_holder_pod_uid="" reused_holder_renew_time="" + local moved_at="" moved_holder="" moved_holder_pod_uid="" move_evidence="" handoff_observed=0 + local observed_at deadline wait_secs + + deadline=$((delete_epoch + 240)) + while [ "$(date +%s)" -lt "$deadline" ]; do + kvha_members_mounted_sample "$ns" "$backend" >>"$traj_file" + new_holder="$(kvha_lease_holder "$ns" "$lease")" + new_holder_pod_uid="$(kvha_holder_pod_uid "$ns" "$leader_sel" "$new_holder")" + new_renew_time="" + if [ -n "$new_holder_pod_uid" ] && [ "$new_holder_pod_uid" != "$old_ready_uid" ] \ + && [ "$new_holder" = "$old_holder" ]; then + new_renew_time="$(kvha_lease_renew_time "$ns" "$lease")" + fi + new_ready="$(kvha_ready_leader_pod "$ns" "$leader_sel")" + new_ready_uid="" + if [ -n "$new_ready" ]; then + new_ready_uid="$(kvha_pod_uid "$ns" "$new_ready")" + fi + + observed_at="$(date +%s)" + if [ -z "$moved_at" ] && [ -n "$new_holder_pod_uid" ] \ + && [ "$new_holder_pod_uid" != "$old_ready_uid" ]; then + if [ "$new_holder" != "$old_holder" ]; then + moved_at="$observed_at" + moved_holder="$new_holder" + moved_holder_pod_uid="$new_holder_pod_uid" + move_evidence="holderIdentity changed" + elif [ -n "$new_renew_time" ]; then + if [ "$reused_holder_pod_uid" != "$new_holder_pod_uid" ] \ + || [ -z "$reused_holder_renew_time" ]; then + reused_holder_pod_uid="$new_holder_pod_uid" + reused_holder_renew_time="$new_renew_time" + elif [ "$new_renew_time" != "$reused_holder_renew_time" ]; then + moved_at="$observed_at" + moved_holder="$new_holder" + moved_holder_pod_uid="$new_holder_pod_uid" + move_evidence="replacement renewed the reused holderIdentity" + fi + fi + fi + if [ -n "$new_holder_pod_uid" ] && [ "$new_holder_pod_uid" != "$old_ready_uid" ] \ + && [ "$new_holder_pod_uid" = "$new_ready_uid" ]; then + handoff_observed=1 + if [ -z "$moved_at" ]; then + moved_at="$observed_at" + moved_holder="$new_holder" + moved_holder_pod_uid="$new_holder_pod_uid" + move_evidence="Ready replacement confirmed the reused holderIdentity" + fi + fi + [ "$handoff_observed" = "1" ] && break + sleep 2 + done + wait_secs=$(( $(date +%s) - delete_epoch )) + printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \ + "$moved_at" "$moved_holder" "$moved_holder_pod_uid" "$move_evidence" "$handoff_observed" \ + "$new_holder" "$new_holder_pod_uid" "$new_ready" "$new_ready_uid" "$wait_secs" +} + +# kvha_failover_bound_verdict judges one failover +# measurement and prints the verdict as STATUS|object: PASS, FAIL or SKIP, and the text that goes +# into the row. The row itself stays with the case, which calls `record`. The Ready replacement is +# an upper bound rather than a direct reading of the move, so it SKIPs. +kvha_failover_bound_verdict() { + local moved_at="$1" delete_epoch="$2" bound_secs="$3" evidence="$4" + local moved_secs=$((moved_at - delete_epoch)) + case "$evidence" in + "holderIdentity changed"|"replacement renewed the reused holderIdentity") + if [ "$moved_secs" -le "$bound_secs" ]; then + printf 'PASS|%ss <= %ss; %s\n' "$moved_secs" "$bound_secs" "$evidence" + else + printf 'FAIL|%ss > %ss -- %s; the election still works but no longer meets the failover budget\n' \ + "$moved_secs" "$bound_secs" "$evidence" + fi + ;; + "Ready replacement confirmed the reused holderIdentity") + printf 'SKIP|%ss after the failover trigger; the Ready replacement upper bound includes Pod scheduling, image pulling, and readiness probing\n' \ + "$moved_secs" + ;; + *) + printf 'FAIL|unrecognized move evidence: %s\n' "${evidence:-}" + ;; + esac +} + +# kvha_resolve_rollout_images prints the two images of a rollout +# pair, separated by a space: the start image, then the rollout target. +kvha_resolve_rollout_images() { + printf '%s %s\n' "${E2E_MOONCAKE_IMAGE:-$1}" "${E2E_MOONCAKE_ROLLOUT_IMAGE:-$2}" +} diff --git a/.agents/skills/gpustack-operator-e2e/cases/_kvcache-inject-lib.sh b/.agents/skills/gpustack-operator-e2e/cases/_kvcache-inject-lib.sh index da051d10d..758d0a0a4 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/_kvcache-inject-lib.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/_kvcache-inject-lib.sh @@ -27,8 +27,8 @@ # # Environment the caller may set: # E2E_MOONCAKE_IMAGE the store image (default docker.io/kvcacheai/mooncake:0.3.13) -# E2E_VLLM_IMAGE a vLLM image, for case 59 (unset => that half SKIPs, loudly) -# E2E_SGLANG_IMAGE an SGLang image, for case 59 (unset => that half SKIPs, loudly) +# E2E_VLLM_IMAGE a vLLM image, for case 53 (unset => that half SKIPs, loudly) +# E2E_SGLANG_IMAGE an SGLang image, for case 53 (unset => that half SKIPs, loudly) # E2E_CLIENT_IMAGE an image carrying the mooncake python client, for the read/write probes # # EXERCISED 2026-09-04 on a three-node k3s cluster: all seven completed with no failures, and case @@ -37,8 +37,8 @@ # # Four of the seven failed on their first real run, and not one was failing for the reason it printed: # case 53 hit a missing prerequisite (no Binding registered the tenant its writes land on) and then -# read usage off the wrong Binding; case 56 could not have passed at all, because it compared fields -# the API server defaults onto every Pod; case 59 timed out on a 19.1GB image pull. Reading them had +# read usage off the wrong Binding; case 53 could not have passed at all, because it compared fields +# the API server defaults onto every Pod; case 53 timed out on a 19.1GB image pull. Reading them had # shown none of that, and three of the four were defects in the cases rather than in the webhook. E2E_SHIM_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" @@ -98,7 +98,7 @@ DOMAIN="dom-i-${SFX}" ENDPOINT="" # The launch stub kvi_setup projects into every injected fixture, and where each Pod mounts it. The # three names are shared because four manifests in this family declare the same volume: this file's -# kvi_pod_manifest, case 53's Deployment, case 54's LWS leader and case 56's submit(). +# kvi_pod_manifest, case 53's Deployment, LWS leader and submit(). # # LAUNCH_DIR is NOT on any image's PATH, and must not be: the stub is reached by the absolute path in # `command`, while a probe running `python3` inside the container has to get the image's real @@ -120,7 +120,11 @@ record() { . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" -# kvi_wait_for polls one jsonpath until it equals want. +# kvi_wait_for is the family's own jsonpath poller, self-contained on purpose: a consumer may +# define a wait_for of its own (case-70 replaces its argument list with seconds-then-predicate), +# and forwarding through a replaceable global would run this family's kind/name/path/want calls +# against that incompatible signature. The base corpus had this implementation inline; the +# extraction keeps it whole rather than aliasing it. kvi_wait_for() { local kind="$1" name="$2" path="$3" want="$4" secs="${5:-120}" ns_args=() [ -n "${6:-}" ] && ns_args=(-n "$6") @@ -128,7 +132,7 @@ kvi_wait_for() { # ${ns_args[@]+...} rather than a bare "${ns_args[@]}": under set -u, bash 3.2 - macOS's stock # shell, which this family is usually launched from - treats expanding an EMPTY array as an unbound # variable and aborts the case mid-run. Every cluster-scoped wait passes no namespace, so the empty - # case is the common one, not the edge one. Measured both ways on bash 3.2.57. + # case is the common one, not the edge one. for ((i = 0; i < secs; i += 3)); do got="$(kubectl ${ns_args[@]+"${ns_args[@]}"} get "$kind" "$name" -o "jsonpath=$path" 2>/dev/null)" [ "$got" = "$want" ] && { echo "$got"; return 0; } @@ -398,6 +402,52 @@ kvi_env() { -o "jsonpath={.spec.containers[0].env[?(@.name=='${name}')].value}" 2>/dev/null } +# kvi_store_roundtrip_py POD CONTAINER KEY PAYLOAD runs the mooncake client library inside an injected +# Pod and prints what it learned: the tenant the projected file named, the store's setup status, the +# put's status, and the length of the bytes that came back with whether they match what was written. +# +# The probe reads the file the webhook PROJECTED - not an address this suite knows - and calls setup() +# with the seven positional arguments plus the file's tenant_id. The tenant rides the keyword: the next +# positional slot is a TransferEngine pointer, and passing the tenant there raises instead of +# registering it. A file without a tenant_id falls back to the store's own "default", which the usage +# rows catch. +# +# The python source stays at column 0: it is a program carried in a quoted argument, so indenting it +# here would change the program rather than the shell. +# +# The exit status is the exec's own. A Pod that cannot run the client fails here, and the caller reads +# the printed lines to say which half of the round trip was reached. +kvi_store_roundtrip_py() { + local pod="$1" container="$2" key="$3" payload="$4" out rc + out="$(kubectl -n "$TEST_NS" exec "$pod" -c "$container" -- python3 -c ' +import json, os, sys +from mooncake.store import MooncakeDistributedStore + +cfg = json.load(open(os.environ["MOONCAKE_CONFIG_PATH"])) +tenant = cfg.get("tenant_id") or "default" +print("TENANT %s" % tenant) +store = MooncakeDistributedStore() +rc = store.setup( + os.environ.get("POD_IP", "127.0.0.1"), + cfg["metadata_server"], + cfg["global_segment_size"], + cfg["local_buffer_size"], + cfg["protocol"], + cfg["device_name"], + cfg["master_server_address"], + tenant_id=tenant, +) +print("SETUP rc=%d" % rc) +if rc != 0: + sys.exit(0) +print("PUT rc=%d" % store.put(sys.argv[1], sys.argv[2].encode())) +got = store.get(sys.argv[1]) +print("GET len=%d match=%s" % (len(got or b""), (got == sys.argv[2].encode()))) +' "$key" "$payload" 2>&1)" && rc=0 || rc=$? + printf '%s\n' "$out" + return "$rc" +} + # kvi_refused applies a manifest that must be REJECTED, and reports whether the message names what it # should. A create that succeeds is the failure: this family exists to prove the webhook refuses # rather than injecting something that looks configured and is not. @@ -539,28 +589,24 @@ kvi_teardown() { kubectl -n "$TEST_NS" delete kvcachepoolbindings.worker.gpustack.ai "$BINDING" "$BINDING_DEFAULT" \ --ignore-not-found --wait=false >/dev/null 2>&1 || true - local left="" _ - for _ in $(seq 1 20); do - left="$(kubectl -n "$TEST_NS" get kvcachepoolbindings.worker.gpustack.ai -o name 2>/dev/null)" - [ -z "$left" ] && break - sleep 3 - done - if [ -n "${left:-}" ]; then - echo "[kvcache-inject] a binding is still held after 60s; what the operator says about it:" - kubectl -n "$TEST_NS" get kvcachepoolbindings.worker.gpustack.ai \ - -o 'jsonpath={range .items[*]}{.metadata.name}{"\t"}{range .status.conditions[*]}{.type}={.status}({.reason}) {.message}{"\n"}{end}{end}' 2>/dev/null \ - | sed 's/^/ /' - # Forcing the finalizer off bypasses the operator's own release path, which removes it only once - # the master's ledger entry for that tenant is gone (kv_cache_pool.go, releaseKVCachePoolBinding). - # So this can leave a tenant registered in a master with no Binding owning it. It is still the - # right trade here - a Binding left held forever poisons every later run of this family, and these - # names are cluster-scoped - but it is not clean, and a forced action that prints nothing reads as - # a clean one. - echo "[kvcache-inject] forcing the finalizer off; a tenant may remain registered in the master." - echo "[kvcache-inject] the pool and backend are deleted below, which normally takes it with them;" - echo "[kvcache-inject] if either is also stuck, check the master's tenant list by hand." - kubectl -n "$TEST_NS" get kvcachepoolbindings.worker.gpustack.ai -o name 2>/dev/null | while read -r b; do - kubectl -n "$TEST_NS" patch "$b" --type=merge -p '{"metadata":{"finalizers":null}}' >/dev/null 2>&1 || true + # One binding at a time: each is waited for, and one still held after 60s has its finalizer forced + # off, with a report of what the operator said about it. The pool and the backend are deleted + # below, which normally takes the tenant with them; if either is also stuck, the master's tenant + # list has to be read by hand. + kvi_release_binding_finalizer "$TEST_NS" "$BINDING" + kvi_release_binding_finalizer "$TEST_NS" "$BINDING_DEFAULT" + # A case's own probe can leave a binding this teardown never named - the refusal probe creates + # bind-term with a case-added finalizer, and an interrupted run leaves it held with no controller + # to release it. Sweep every binding still standing in the fixture namespace, so a leftover of + # any name releases the same way the named ones do and cannot wedge the namespace in + # Terminating against the next run. + local leftover="" + leftover="$(kubectl -n "$TEST_NS" get kvcachepoolbindings.worker.gpustack.ai -o name 2>/dev/null || true)" + if [ -n "$leftover" ]; then + echo "[kvcache-inject] releasing binding(s) the fixture did not name:" + printf '%s\n' "$leftover" | sed 's/^/ /' + printf '%s\n' "$leftover" | while read -r b; do + [ -n "$b" ] && kvi_release_binding_finalizer "$TEST_NS" "$(printf '%s' "$b" | awk -F/ '{print $NF}')" done fi diff --git a/.agents/skills/gpustack-operator-e2e/cases/_md-lib.sh b/.agents/skills/gpustack-operator-e2e/cases/_md-lib.sh new file mode 100644 index 000000000..e89d77427 --- /dev/null +++ b/.agents/skills/gpustack-operator-e2e/cases/_md-lib.sh @@ -0,0 +1,389 @@ +#!/usr/bin/env bash +# +# _md-lib.sh — the ModelDeployment readers, renderers and dry-run harness the cases share. +# +# NOT A CASE. The leading underscore keeps it out of the `case-N.sh` namespace. The rows and the +# verdict stay with the case that sources it: this file prints the readings a row is built from, and +# it never prints the results table, records a row or decides the exit code. +# +# A case uses it as: +# +# record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } +# # shellcheck source=/dev/null +# . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_md-lib.sh" +# +# Every helper here takes what it reads as an argument. Nothing reaches into a case's variables, +# because the same helper serves cases whose names for the same thing differ. +# +# `_epd-lib.sh` holds the CPU-vendor fixture and its platform lookup, and it stays a separate +# library. `md_usable_type_for` below is the same lookup as that library's `epd_platform_type`; the +# two are deliberately not merged, so a case that sources one does not inherit the other's fixture. + +# The cluster context every reader in this library talks to. It is empty by default, which is the +# context plain kubectl talks to as well; a case that runs against another context sets it. +MD_KUBE_CONTEXT="${MD_KUBE_CONTEXT:-}" + +# md_kubectl runs kubectl against MD_KUBE_CONTEXT when one is named, and against the default +# context otherwise. +md_kubectl() { + if [ -n "$MD_KUBE_CONTEXT" ]; then + kubectl --context "$MD_KUBE_CONTEXT" "$@" + else + kubectl "$@" + fi +} + +# ---------------------------------------------------------------- the InstanceType a role names + +# md_usable_instance_type prints the first InstanceType a deployment can name. A type that is +# deleting or inactive is skipped: naming one is refused at admission, or the deployment is admitted +# and then never scheduled. +# +# Every name passed as an argument is excluded. A case that authors a type of its own excludes it +# here, because a leftover of it would otherwise be picked and measured as if it were a working one. +md_usable_instance_type() { + local name deleting inactive skip + md_kubectl get instancetypes.worker.gpustack.ai \ + -o jsonpath='{range .items[*]}{.metadata.name}|{.metadata.deletionTimestamp}|{.spec.inactive}{"\n"}{end}' \ + 2>/dev/null \ + | while IFS='|' read -r name deleting inactive; do + [ -n "$name" ] || continue + [ -z "$deleting" ] || continue + [ "$inactive" = true ] && continue + for skip in "$@"; do + [ "$name" = "$skip" ] && continue 2 + done + echo "$name" + break + done +} + +# md_usable_type_for prints the first InstanceType reporting the named manufacturer. The webhook +# reads the platform the type a role names reports, so a row that needs a real type reads it by the +# manufacturer it expects rather than by list position. +md_usable_type_for() { + local name deleting inactive manufacturer + md_kubectl get instancetypes.worker.gpustack.ai \ + -o jsonpath='{range .items[*]}{.metadata.name}|{.metadata.deletionTimestamp}|{.spec.inactive}|{.status.detail.manufacturer}{"\n"}{end}' \ + 2>/dev/null \ + | while IFS='|' read -r name deleting inactive manufacturer; do + [ -n "$name" ] || continue + [ -z "$deleting" ] || continue + [ "$inactive" = true ] && continue + [ -n "$manufacturer" ] || continue + if [ "$manufacturer" = "$1" ]; then + echo "$name" + break + fi + done +} + +# md_usable_instance_types prints every InstanceType a deployment can name. +# +# AN ACCELERATED TYPE WITH NO WHOLE CARD IS EXCLUDED TOO. A role that names no accelerator count is +# defaulted to one card, and the webhook refuses a count above the type's whole-card capacity, so +# naming a type whose pool has no card left is refused for a reason none of the rows is about. +md_usable_instance_types() { + local name deleting inactive acceleratable capacity + md_kubectl get instancetypes.worker.gpustack.ai \ + -o jsonpath='{range .items[*]}{.metadata.name}|{.metadata.deletionTimestamp}|{.spec.inactive}|{.spec.acceleratable}|{.status.accelerator.capacity}{"\n"}{end}' \ + 2>/dev/null \ + | while IFS='|' read -r name deleting inactive acceleratable capacity; do + [ -n "$name" ] || continue + [ -z "$deleting" ] || continue + [ "$inactive" = true ] && continue + if [ "$acceleratable" = true ]; then + case "$capacity" in "" | 0) continue ;; esac + fi + echo "$name" + done +} + +# md_cq_of_it prints the ClusterQueue an InstanceType schedules into, resolved the way the operator +# resolves it: the type's published entrance LocalQueue, then that LocalQueue's ClusterQueue. +# +# It prints nothing when the type names no entrance, which is the same reading a lookup of an empty +# LocalQueue name gives. +md_cq_of_it() { + local lq + lq="$(md_kubectl get instancetypes.worker.gpustack.ai "$2" \ + -o jsonpath='{.status.entrance}' 2>/dev/null)" + [ -n "$lq" ] || return 0 + md_kubectl -n "$1" get localqueue.kueue.x-k8s.io "$lq" \ + -o jsonpath='{.spec.clusterQueue}' 2>/dev/null +} + +# ---------------------------------------------------------------- the manifest and its roles + +# md_role_block prints one role of a ModelDeployment. The image and command are placeholder +# plumbing: no reading that matters is about what the container runs. +md_role_block() { + printf ' - name: %s\n' "$1" + [ -n "$2" ] && printf ' kind: %s\n' "$2" + printf ' instanceType: %s\n replicas: %s\n' "$3" "$4" + printf ' image: %s\n command: ["/pause"]\n' "$5" + [ -n "${6:-}" ] && printf ' size: %s\n' "$6" + if [ -n "${7:-}" ]; then + printf ' topology:\n requiredLevel: %s\n' "$7" + fi +} + +# md_manifest prints a ModelDeployment manifest whose roles block is supplied whole by the caller. +# +# The poolRef name expands with a bare dash and NOT with a colon-dash when the CALLER resolves it: +# the colon form substitutes the default for an EMPTY value as well as an unset one, which would +# rewrite an empty name a row is trying to submit. The kvCache block is the extra lines under +# kvCache, and it is empty for a manifest that declares nothing but the pool reference. +md_manifest() { + cat <&1)" +} + +# ---------------------------------------------------------------- what the cluster holds + +# md_workloads prints every Workload of one ModelDeployment, one name per line. +# +# THE LOOKUP RUNS WORKLOAD-TO-DEPLOYMENT, never deployment-to-Workload: the Workload is enumerated, +# each of its owner references is read, and the Workload is attributed when one of them is a Pod this +# deployment owns. Looking the other way -- searching for a Workload that carries a direct +# ModelDeployment owner -- finds nothing on any correct cluster, because Kueue's pod integration +# writes no such reference. The UIDs are matched padded so one UID cannot hit a prefix of a longer +# one. +md_workloads() { + local ns="$1" md="$2" uids row wl owners u + uids="$(md_kubectl -n "$ns" get pods -l "app.kubernetes.io/instance=${md}" \ + -o jsonpath='{range .items[*]}{.metadata.uid}{"\n"}{end}' 2>/dev/null)" + [ -n "$uids" ] || return 0 + while IFS= read -r row; do + [ -n "$row" ] || continue + wl="${row%%=*}" + owners="${row#*=}" + for u in $uids; do + case " $owners " in *" $u "*) echo "$wl"; break ;; esac + done + done </dev/null) +EOF +} + +# md_workload_verdicts prints every Workload owning one of a deployment's Pods, one +# "==" line each: status is its QuotaReserved condition, "" while Kueue has not +# judged it yet, and role is the component of the Pod it owns. +# +# TWO API CALLS PER INVOCATION, NOT ONE PER WORKLOAD, and here that is correctness rather than +# tidiness. This is the hot path: a verdict poll calls it every five seconds for up to five minutes, +# over a probe that can hold a hundred replicas. At one get per Workload the poll budget goes on +# round-trips and the case reports a pool Kueue never judged. Names, verdicts and owner uids come +# back together and the match happens locally. +md_workload_verdicts() { + local ns="$1" md="$2" pods row wl rest status owners p + pods="$(md_kubectl -n "$ns" get pods -l "app.kubernetes.io/instance=${md}" \ + -o jsonpath='{range .items[*]}{.metadata.uid}={.metadata.labels.app\.kubernetes\.io/component}{"\n"}{end}' \ + 2>/dev/null)" + [ -n "$pods" ] || return 0 + while IFS= read -r row; do + [ -n "$row" ] || continue + wl="${row%%=*}" + rest="${row#*=}" + status="${rest%%=*}" + owners="${rest#*=}" + for p in $pods; do + # Padded on both sides so a uid cannot match a longer one it is a prefix of. + case " $owners " in *" ${p%%=*} "*) echo "${wl}=${status}=${p#*=}"; break ;; esac + done + done </dev/null) +EOF +} + +# md_pod_uids prints the Pod name=UID of every Pod of one deployment, sorted. +md_pod_uids() { + md_kubectl -n "$1" get pods -l "app.kubernetes.io/instance=$2" \ + -o jsonpath='{range .items[*]}{.metadata.name}={.metadata.uid}{"\n"}{end}' 2>/dev/null \ + | grep -v '^$' | sort | tr '\n' ' ' +} + +# md_role_uids prints the Pod name=UID of one role, sorted. The UID is what tells a Pod that STAYED +# from one replaced by an identical render: everything else about the two agrees. +md_role_uids() { + md_kubectl -n "$1" get pods -l "app.kubernetes.io/instance=$2,app.kubernetes.io/component=$3" \ + -o jsonpath='{range .items[*]}{.metadata.name}={.metadata.uid}{"\n"}{end}' 2>/dev/null \ + | grep -v '^$' | sort | tr '\n' ' ' +} + +# md_group_names prints the distinct pod-group names across a deployment's members: one per +# INSTANCE, so their count is how many instances Kueue is being asked to admit. +md_group_names() { + md_kubectl -n "$1" get pods -l "app.kubernetes.io/instance=$2" \ + -o jsonpath='{range .items[*]}{.metadata.labels.kueue\.x-k8s\.io/pod-group-name}{"\n"}{end}' 2>/dev/null \ + | grep -v '^$' | sort -u +} + +# md_surviving_uids prints how many of a recorded UID set are still present. With a role named, the +# set is compared against that role's Pods; without one, against every Pod of the deployment. +md_surviving_uids() { + local ns="$1" md="$2" before="$3" role="${4:-}" now n=0 e + if [ -n "$role" ]; then + now="$(md_role_uids "$ns" "$md" "$role")" + else + now="$(md_pod_uids "$ns" "$md")" + fi + for e in $before; do + case " $now " in *" $e "*) n=$((n + 1)) ;; esac + done + echo "$n" +} + +# md_force_release deletes every Workload owning a ModelDeployment's Pods, which is what releases +# Kueue's finalizer on a serving group. +# +# EVERY OWNED WORKLOAD, NOT THE FIRST ONE FOUND, and ONE LIST CALL PER INVOCATION. This runs from +# teardown, a deployment of several replicas holds several Workloads, and releasing one of them +# leaves the rest holding finalizers on Pods a later case then inherits. +md_force_release() { + local ns="$1" md="$2" row wl uids u + uids="$(md_kubectl -n "$ns" get pods -l "app.kubernetes.io/instance=${md}" \ + -o jsonpath='{range .items[*]}{.metadata.uid}{"\n"}{end}' 2>/dev/null)" + [ -n "$uids" ] || return 0 + while IFS= read -r row; do + [ -n "$row" ] || continue + wl="${row%%=*}" + for u in $uids; do + case " ${row#*=} " in + *" $u "*) + md_kubectl -n "$ns" delete workloads.kueue.x-k8s.io "$wl" \ + --ignore-not-found --wait=false >/dev/null 2>&1 + break + ;; + esac + done + done </dev/null) +EOF + + return 0 +} + +# ---------------------------------------------------------------- the server-side dry run + +# The four helpers below read the manifest on STDIN, print the OBJECT text of the row they describe +# and return 0 when the outcome is the one the row asserts. The ROW ITSELF STAYS WITH THE CALLING +# CASE: the case turns the exit status into a PASS or a FAIL and records it, so the table and the +# verdict never leave the case that owns them. +# +# The manifest travels on stdin because a caller builds it, and a caller handing it over as an +# argument would have to strip the trailing newlines the builder emitted. + +# md_dryrun_refuses asserts that the manifest is REFUSED and that the refusal quotes the given text +# and every further fragment passed after it. A refusal carrying somebody else's message is a +# failure: it means the manifest tripped a rule the row is not about. +# +# AN EMPTY OUTPUT MUST NOT PASS. The refusal is read out of the apply's output, and an output that +# is empty at all means nothing was refused: the row would report a refusal it never saw. +md_dryrun_refuses() { + local want="$1" + shift + local out w frags="$*" + out="$(md_kubectl apply --dry-run=server -f - 2>&1 | tr '\n' ' ')" + if [ -z "$out" ]; then + echo "the apply produced no output at all, so nothing was refused" + + return 1 + fi + for w in "$want" "$@"; do + case "$out" in + *"$w"*) ;; + *) echo "refused with the wrong message, or accepted (missing '${w}'): ${out:0:220}"; return 1 ;; + esac + done + echo "refused naming: ${want}${frags:+ + $frags}" +} + +# md_dryrun_accepts asserts that the manifest is ACCEPTED. It carries the same empty-output trap as +# md_dryrun_refuses for the same reason: "created" must be observed, because a row that passes on no +# output at all reports an acceptance nothing produced. +md_dryrun_accepts() { + local out + out="$(md_kubectl apply --dry-run=server -f - 2>&1 | tr '\n' ' ')" + case "$out" in + *"created (server dry run)"* | *"configured (server dry run)"*) + echo "accepted by both the schema and the webhook" + + return 0 + ;; + *) echo "refused: ${out:0:220}"; return 1 ;; + esac +} + +# md_dryrun_rejects asserts that the manifest is REFUSED, whatever the refusal says. The refusal's +# own text is the row's object, cut to the given number of characters. +md_dryrun_rejects() { + local max="${1:-160}" out + if out="$(md_kubectl apply --dry-run=server -f - 2>&1)"; then + echo "admitted: $(printf '%s' "$out" | tr '\n' ' ' | cut -c1-160)" + + return 1 + fi + echo "$(printf '%s' "$out" | tr '\n' ' ' | cut -c1-"$max")" +} + +# md_dryrun_admits asserts that the manifest is ACCEPTED, whatever the acceptance says. The +# acceptance's own text is the row's object, cut to the given number of characters. +md_dryrun_admits() { + local max="${1:-160}" out + if out="$(md_kubectl apply --dry-run=server -f - 2>&1)"; then + echo "$(printf '%s' "$out" | tr '\n' ' ' | cut -c1-"$max")" + + return 0 + fi + echo "refused: $(printf '%s' "$out" | tr '\n' ' ' | cut -c1-160)" +} diff --git a/.agents/skills/gpustack-operator-e2e/cases/_model-hub-lib.sh b/.agents/skills/gpustack-operator-e2e/cases/_model-hub-lib.sh index f6c6a14de..df280022f 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/_model-hub-lib.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/_model-hub-lib.sh @@ -125,26 +125,8 @@ mh_manifest_sha() { mh_call "$1" "$2" GET /_manifest | python3 -c "import json,sys; print(json.load(sys.stdin)['$3']['files']['$4']['sha256'])" } -# setting_get KEY / setting_set KEY VALUE: a Setting in the store, bypassing admission the way a -# seed from the environment does. setting_set prints the previous value. -setting_get() { - kubectl -n "$SYSTEM_NS" get secret gpustack-settings -o jsonpath="{.data.$1}" 2>/dev/null | base64 -d 2>/dev/null -} -setting_set() { - local b64 - b64="$(printf '%s' "$2" | base64 | tr -d '\n')" - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge -p "{\"data\":{\"$1\":\"${b64}\"}}" >/dev/null -} - -# setting_unset KEY: removes KEY from the store, so the Setting reads its default again. -setting_unset() { - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge -p "{\"data\":{\"$1\":null}}" >/dev/null -} - -# settings_settle: waits out the worker's thirty-second Settings read cache, so its next read of a -# Setting written through the store sees the new value. The plugin needs no wait: it reads its -# NodeModelStore's spec, which the worker rewrites on the Secret's change. -settings_settle() { sleep 35; } +# setting_get KEY, setting_set KEY VALUE, setting_unset KEY and settings_settle live in _rows-lib.sh, +# which every case sourcing this file also sources, and sources before it. # artifact NS NAME REPOSITORY [SECRET] [LABEL] [SPEC_YAML]: a Hugging Face ModelArtifact at main, # with SPEC_YAML (such as allowPatterns) appended to its spec. @@ -249,6 +231,35 @@ plugin_metric() { | awk -v m="$2" '$1 ~ "^"m"({|$)" {s += $2} END {printf "%d", s}' } +# mh_peer_sync_check NS POD NODE ARTIFACT UID DIGEST [BOUND]: mounts ARTIFACT on NODE through a +# fresh consumer Pod and reports the plugin's download counters around the mount, as +# "peer_before hub_before peer_after hub_after". The plugin Pod on NODE is deleted first: its +# cache is an emptyDir that dies with it, so the node starts empty and the mount can only be +# served by a peer. The counters are the plugin's own, so a peer pull raises the peer one and +# leaves the hub one alone. The status is 0 when the consumer Pod ran; the counters are reported +# either way. +mh_peer_sync_check() { + local ns="$1" pod="$2" node="$3" artifact="$4" uid="$5" digest="$6" bound="${7:-300}" + local peer_metric='gpustack_model_manager_download_bytes_total{source="peer"}' + local hub_metric='gpustack_model_manager_download_bytes_total{source="hub"}' + local peer_before hub_before peer_after hub_after plugin status=0 + kubectl -n "$SYSTEM_NS" delete pod -l app.kubernetes.io/component=model-manager \ + --field-selector "spec.nodeName=$node" --wait=true --timeout=120s >/dev/null 2>&1 + for _ in $(seq 1 30); do + plugin="$(plugin_pod "$node")" + [ -n "$plugin" ] && [ "$(kubectl -n "$SYSTEM_NS" get pod "$plugin" -o jsonpath='{.status.phase}' 2>/dev/null)" = Running ] && break + sleep 3 + done + peer_before="$(plugin_metric "$node" "$peer_metric")" + hub_before="$(plugin_metric "$node" "$hub_metric")" + consumer "$ns" "$pod" "$node" "$artifact" "$uid" "$digest" + pod_ready "$ns" "$pod" "$bound" || status=$? + peer_after="$(plugin_metric "$node" "$peer_metric")" + hub_after="$(plugin_metric "$node" "$hub_metric")" + printf '%s %s %s %s\n' "$peer_before" "$hub_before" "$peer_after" "$hub_after" + return "$status" +} + # store_state NODE DIGEST: the node's NodeModelStore entry for DIGEST, "state|reason". store_state() { kubectl get nodemodelstores.worker.gpustack.ai "$1" \ diff --git a/.agents/skills/gpustack-operator-e2e/cases/_partition-lib.sh b/.agents/skills/gpustack-operator-e2e/cases/_partition-lib.sh index b7c00f7c6..b49c450de 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/_partition-lib.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/_partition-lib.sh @@ -30,6 +30,11 @@ # MIG_SSH_TIMEOUT= per-ssh timeout (default 90) # IMAGE= test-pod image (default ubuntu:24.04) +# The pool lookup lives in its own library; part_discover needs it, and so does every case that +# submits a claim through a queue. +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_pool-lib.sh" + MIGSSH_TIMEOUT="${MIG_SSH_TIMEOUT:-90}" GPU_INDEX="${MIG_GPU_INDEX:-0}" IMAGE="${IMAGE:-ubuntu:24.04}" @@ -260,30 +265,7 @@ for g in json.load(sys.stdin).get('spec',{}).get('groups',[]): print(g.get('id','')); break " 2>/dev/null) - read -r IT LQ MANUF <<<"$(kubectl get instancetypes.worker.gpustack.ai -o json 2>/dev/null | NODE_GID="$GROUPID" NODE_JSON="$(kubectl get node "$GPU_NODE" -o json 2>/dev/null)" python3 -c " -import json,sys,os -gid=os.environ.get('NODE_GID','') -nl=(json.loads(os.environ.get('NODE_JSON') or '{}').get('metadata',{}) or {}).get('labels',{}) or {} -# A pool BACKS this node when every discriminator it carries is a label the node carries too. Those are -# the schedule labels the webhook stamps from PoolScheduleLabels, so this is the pool's whole identity — -# os/arch, the acceleratable boolean, the accelerator group and, under CPU-aware grouping, the general -# group. schedule.gpustack.ai/* is the pool's own bookkeeping and never a node label. -def backs(it): - d={k:v for k,v in (it['metadata'].get('labels') or {}).items() if not k.startswith('schedule.gpustack.ai/')} - return bool(d) and all(nl.get(k)==v for k,v in d.items()) -def in_group(s): - return bool(gid) and (s.get('acceleratorGroup') or '').endswith('-'+gid) -items=json.load(sys.stdin).get('items',[]) -# The pool identity this node belongs to, not its rendered name — see the index's note on pool lookup. -for it in items: - s=it.get('spec',{}); st=it.get('status',{}) - if s.get('acceleratable') and in_group(s) and backs(it) and st.get('entrance'): - print(it['metadata']['name'], st['entrance'], (st.get('detail') or {}).get('manufacturer','nvidia')); sys.exit(0) -for it in items: - s=it.get('spec',{}); st=it.get('status',{}) - if s.get('acceleratable') and st.get('entrance'): - print(it['metadata']['name'], st['entrance'], (st.get('detail') or {}).get('manufacturer','nvidia')); sys.exit(0) -")" + read -r IT LQ MANUF <<<"$(discover_node_pool "$GPU_NODE" "$GROUPID" --require-entrance --fallback-any name entrance manufacturer)" if [ -z "${IT:-}" ] || [ -z "${LQ:-}" ]; then echo "[case-${CASE_ID}] no accelerated InstanceType with an entrance LocalQueue — chain not materialized" exit 1 @@ -756,12 +738,31 @@ wait_pod_cards() { echo "$c" } +# exec_out ... — run a command in the Pod and echo its output, retried. +# +# A Pod reports Running slightly before its container is attachable, so an exec issued the instant +# wait_running returns can come back empty — or fail outright with `container not found` — on a Pod +# whose allocation is perfectly correct. That matters most in pod_mig_devices below: `grep -c` prints +# 0 even when the exec failed, so an attach race reads as "no MIG device inside the Pod" and fails a +# correct allocation. +exec_out() { + local pod="$1"; shift + local out + for _ in $(seq 1 8); do + out=$(kubectl -n default exec "$pod" -- "$@" 2>/dev/null) + [ -n "$out" ] && { printf '%s\n' "$out"; return 0; } + sleep 3 + done + return 1 +} + # pod_mig_devices — how many MIG devices nvidia-smi -L reports inside the Pod (want exactly 1 for # a single-instance partition request). pod_mig_devices() { - local out - out="$(kubectl -n default exec "$1" -- nvidia-smi -L 2>/dev/null | grep -c 'MIG')" - echo "${out:-0}" + local out n + out="$(exec_out "$1" nvidia-smi -L)" + n="$(printf '%s\n' "$out" | grep -c 'MIG' || true)" + echo "${n:-0}" } # in_set — membership test against a " a b c " style set. diff --git a/.agents/skills/gpustack-operator-e2e/cases/_pool-lib.sh b/.agents/skills/gpustack-operator-e2e/cases/_pool-lib.sh new file mode 100644 index 000000000..d99dd91d9 --- /dev/null +++ b/.agents/skills/gpustack-operator-e2e/cases/_pool-lib.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# +# _pool-lib.sh — the shared pool lookup: which pool backs this node. +# +# NOT A CASE. The leading underscore keeps it out of the `case-N.sh` namespace: it carries no case +# header, no trap and no results table. It answers one question, and every case that submits a claim +# through a queue must answer it the same way — through the wrong pool's queue the ResourceFlavor +# selects other nodes and a node-pinned Pod is simply never admitted, which reads like a placement +# defect rather than like a lookup mistake. +# +# A case uses it as: +# +# # shellcheck source=/dev/null +# . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_pool-lib.sh" +# read -r IT LQ <<<"$(discover_node_pool "$NODE" "$GROUP_ID" --sliceable name entrance)" +# +# A pool BACKS a node when every discriminator the pool carries is a label the node carries too. +# Those are the schedule labels the webhook stamps from PoolScheduleLabels, so this is the pool's +# whole identity — os/arch, the acceleratable boolean, the accelerator group and, under CPU-aware +# grouping, the general group. schedule.gpustack.ai/* is the pool's own bookkeeping and never a node +# label. Matching on a substring of the pool's NAME instead picks a neighbouring pool on any cluster +# whose product tokens nest. + +# discover_node_pool [--sliceable] [--require-entrance] [--fallback-any] ... +# +# Prints the pool that backs on one line: the requested fields, separated by a space, in the +# order asked for. The accelerator group the node's cards belong to is ; the pool's +# spec.acceleratorGroup must end with it. +# +# --sliceable the pool must serve logical slices (status.detail.slicedDetail.logical.count) +# --require-entrance the pool must name an entrance LocalQueue +# --fallback-any when nothing backs the node, accept any accelerated pool with an entrance +# +# Fields: name, entrance, manufacturer, memory, unit-cpu, unit-ram. +# +# Exit status: 0 a pool was found, 1 no pool backs the node, 2 the InstanceTypes or the Node could +# not be read, 3 the InstanceTypes could not be parsed. The last two mean "cannot tell" and must +# never be read as "no pool": a caller that folds them into an absent pool turns a broken query into +# a clean skip, which a summary table reads as coverage. +discover_node_pool() { + [ "$#" -ge 2 ] || return 1 + local node="$1" gid="$2" + shift 2 + local want_sliceable=0 want_entrance=0 fallback_any=0 + local fields=() + while [ "$#" -gt 0 ]; do + case "$1" in + --sliceable) want_sliceable=1 ;; + --require-entrance) want_entrance=1 ;; + --fallback-any) fallback_any=1 ;; + *) fields+=("$1") ;; + esac + shift + done + + local its node_json + if ! its="$(kubectl get instancetypes.worker.gpustack.ai -o json 2>&1)"; then + return 2 + fi + if ! node_json="$(kubectl get node "$node" -o json 2>&1)"; then + return 2 + fi + + # A Node JSON that does not parse is an instrument failure, not a no-match: the matcher below + # distinguishes them for the InstanceType half (parse error exits 3), and the Node half must not + # silently turn a broken read into an empty match - an empty match reads as an environmental + # SKIP in the caller, and a broken instrument must not SKIP. + if ! printf '%s' "$node_json" | python3 -c 'import json,sys; json.load(sys.stdin)' 2>/dev/null; then + return 3 + fi + + printf '%s' "$its" \ + | NODE_GID="$gid" NODE_JSON="$node_json" \ + WANT_SLICEABLE="$want_sliceable" WANT_ENTRANCE="$want_entrance" \ + FALLBACK_ANY="$fallback_any" FIELDS="${fields[*]}" \ + python3 -c ' +import json, os, sys + +gid = os.environ.get("NODE_GID", "") +node_json = os.environ.get("NODE_JSON") or "{}" +nl = (json.loads(node_json).get("metadata", {}) or {}).get("labels", {}) or {} +fields = (os.environ.get("FIELDS") or "").split() +want_sliceable = os.environ.get("WANT_SLICEABLE") == "1" +want_entrance = os.environ.get("WANT_ENTRANCE") == "1" +fallback_any = os.environ.get("FALLBACK_ANY") == "1" + + +def backs(it): + d = {k: v for k, v in (it["metadata"].get("labels") or {}).items() + if not k.startswith("schedule.gpustack.ai/")} + return bool(d) and all(nl.get(k) == v for k, v in d.items()) + + +def in_group(s): + return bool(gid) and (s.get("acceleratorGroup") or "").endswith("-" + gid) + + +def sliceable(st): + sd = (st.get("detail") or {}).get("slicedDetail") or {} + return ((sd.get("logical") or {}).get("count", 0) or 0) > 0 + + +def emit(it): + st = it.get("status", {}) or {} + d = st.get("detail") or {} + u = (it.get("spec", {}) or {}).get("unitResources") or {} + vals = { + "name": it["metadata"]["name"], + "entrance": st.get("entrance", ""), + "manufacturer": d.get("manufacturer", ""), + "memory": d.get("memory", ""), + "unit-cpu": u.get("cpu", "") or "-", + "unit-ram": u.get("ram", "") or "-", + } + print(" ".join(vals.get(f, "") for f in fields)) + + +try: + items = json.load(sys.stdin).get("items", []) +except Exception: + sys.exit(3) + +for it in items: + s = it.get("spec", {}) or {} + st = it.get("status", {}) or {} + if not s.get("acceleratable"): + continue + if want_entrance and not st.get("entrance"): + continue + if want_sliceable and not sliceable(st): + continue + if not (in_group(s) and backs(it)): + continue + emit(it) + sys.exit(0) + +if fallback_any: + for it in items: + s = it.get("spec", {}) or {} + st = it.get("status", {}) or {} + if s.get("acceleratable") and st.get("entrance"): + emit(it) + sys.exit(0) + +sys.exit(1) +' +} diff --git a/.agents/skills/gpustack-operator-e2e/cases/_rows-lib.sh b/.agents/skills/gpustack-operator-e2e/cases/_rows-lib.sh index 880897003..fbe4624de 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/_rows-lib.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/_rows-lib.sh @@ -1,9 +1,11 @@ #!/usr/bin/env bash # -# _rows-lib.sh — the one printer for the STATUS | CHECK | OBJECT table a case ends with. +# _rows-lib.sh — the helpers a case builds on: the one printer for the STATUS | CHECK | OBJECT table +# a case ends with, the jsonpath poller, the Settings store and the release of a KV +# cache Binding the operator is holding. # # NOT A CASE. The leading underscore keeps it out of the `case-N.sh` namespace. The rows and the -# verdict stay with the case that sources it; this file only prints the rows. +# verdict stay with the case that sources it; this file prints the rows and provides helpers. # # A row is "STATUS|CHECK|OBJECT", as every case's `record` writes it, and only its first two "|" are # separators. A CHECK name is several words, so splitting on whitespace moves most of it into OBJECT. @@ -20,6 +22,38 @@ # print_rows # [ "$FAILS" -eq 0 ] || { echo "[case-N] ${FAILS} check(s) FAILED"; exit 1; } +# The kubectl shim every case routes its kubectl calls through. Against a remote API endpoint a +# read can fail on transport alone, and a check that takes such a failure for an answer reports a +# verdict about the network rather than about the operator. +E2E_SHIM_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" +if [ -n "$E2E_SHIM_DIR" ]; then + PATH="$E2E_SHIM_DIR:$PATH" +fi + +# The counters and the row list a case starts from. A case that counts something else keeps its own +# record(), defined after it sources this file. +FAILS=0 +SKIPS=0 +ROWS=() + +# record STATUS CHECK OBJECT appends one row and counts it. A FAIL raises the failure count and a +# SKIP raises the skip count. +record() { + ROWS+=("$1|$2|$3") + [ "$1" = FAIL ] && FAILS=$((FAILS + 1)) + [ "$1" = SKIP ] && SKIPS=$((SKIPS + 1)) + return 0 +} + +# results prints the rows and returns the verdict: 0 when nothing failed, 1 otherwise. A case that +# wants another verdict overrides this, after it sources this file. +results() { + print_rows + [ "$FAILS" -eq 0 ] || { echo "[case-${CASE_ID:-unknown}] ${FAILS} check(s) FAILED"; return 1; } + echo "[case-${CASE_ID:-unknown}] all checks passed" + return 0 +} + # print_rows prints a blank line, the header and every row of ROWS. The expansion is guarded because # bash 3.2 under `set -u` treats an empty array as unbound, and a case that fails before its first # row would abort here instead of printing the failure that stopped it. @@ -32,3 +66,149 @@ print_rows() { printf '%s | %s | %s\n' "${r%%|*}" "${rest%%|*}" "${rest#*|}" done } + +# wait_for [secs] [ns] polls one jsonpath until it reads what is wanted, +# and prints the LAST value seen, so a timeout says what the object was doing rather than only that +# it timed out. +# +# The namespace comes last and is optional: a cluster-scoped object is waited for without one, and +# that is the common case rather than the edge one. The poll runs every three seconds until the +# budget is spent. +wait_for() { + local kind="$1" name="$2" path="$3" want="$4" secs="${5:-120}" ns_args=() + [ -n "${6:-}" ] && ns_args=(-n "$6") + local got="" i + # ${ns_args[@]+...} rather than a bare "${ns_args[@]}": under set -u, bash 3.2 - macOS's stock + # shell, which this family is usually launched from - treats expanding an EMPTY array as an unbound + # variable and aborts the case mid-run. Measured both ways on bash 3.2.57. + for ((i = 0; i < secs; i += 3)); do + got="$(kubectl ${ns_args[@]+"${ns_args[@]}"} get "$kind" "$name" -o "jsonpath=$path" 2>/dev/null)" + [ "$got" = "$want" ] && { echo "$got"; return 0; } + sleep 3 + done + echo "$got" + return 1 +} + +# kvi_release_binding_finalizer waits for a KVCachePoolBinding the operator is +# holding, and when it is still held after 60s it reports what the operator says about it and forces +# the finalizer off. +# +# A domain that still holds objects makes the master refuse to drop its quota, and the operator's +# finalizer is right to wait. Forcing it off bypasses the operator's own release path, which removes +# it only once the master's ledger entry for that tenant is gone, so a tenant can remain registered +# in a master with no Binding owning it. It is still the right trade in a cleanup - a Binding left +# held forever poisons every later run of the family - but it is not clean, and a forced action that +# prints nothing reads as a clean one. +# +# The tag on the two lines it prints is the case that is cleaning up, so a reader of a suite log +# knows which run left the binding held. +kvi_release_binding_finalizer() { + local ns="$1" binding="$2" left="" tag + tag="${0##*/}" + tag="${tag%.sh}" + for _ in $(seq 1 20); do + left="$(kubectl -n "$ns" get kvcachepoolbindings.worker.gpustack.ai "$binding" -o name 2>/dev/null)" + [ -z "$left" ] && break + sleep 3 + done + [ -z "$left" ] && return 0 + echo "[${tag}] the binding ${binding} is still held after 60s; what the operator says about it:" + kubectl -n "$ns" get kvcachepoolbindings.worker.gpustack.ai "$binding" \ + -o 'jsonpath={range .status.conditions[*]}{.type}={.status}({.reason}) {.message}{"\n"}{end}' 2>/dev/null \ + | sed 's/^/ /' + echo "[${tag}] forcing the finalizer off; a tenant may remain registered in the master." + kubectl -n "$ns" patch kvcachepoolbindings.worker.gpustack.ai "$binding" \ + --type=merge -p '{"metadata":{"finalizers":null}}' >/dev/null 2>&1 || true + return 0 +} + +# The Settings store: the gpustack-settings Secret in the system namespace, where a Setting's value +# lives. The helpers below read and write it the way a seed from the environment does, bypassing +# admission. + +# setting_get KEY prints the value KEY carries, empty when the store does not carry it. +setting_get() { + kubectl -n "$SYSTEM_NS" get secret gpustack-settings -o jsonpath="{.data.$1}" 2>/dev/null | base64 -d 2>/dev/null +} + +# setting_set KEY VALUE writes VALUE into the store. An empty VALUE is written as an empty value and +# not as an absent key, because a key that is present but empty is what the caller asked for. +setting_set() { + local b64 + b64="$(printf '%s' "$2" | base64 | tr -d '\n')" + kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge -p "{\"data\":{\"$1\":\"${b64}\"}}" >/dev/null +} + +# setting_unset KEY removes KEY from the store, so the Setting reads its default again. +setting_unset() { + kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge -p "{\"data\":{\"$1\":null}}" >/dev/null +} + +# settings_settle waits out the worker's thirty-second Settings read cache, so its next read of a +# Setting written through the store sees the new value. The plugin needs no wait: it reads its +# NodeModelStore's spec, which the worker rewrites on the Secret's change. +settings_settle() { sleep 35; } + +# The keys settings_save remembered, the values they carried, and whether each was present at all. +SETTINGS_SAVED_KEYS=() +SETTINGS_SAVED_VALUES=() +SETTINGS_SAVED_PRESENT=() + +# setting_present KEY says whether the store carries KEY at all. A key the store carries with an +# empty value and a key the store does not carry both read as empty through setting_get; telling +# them apart is what lets settings_restore put back an empty value as an empty value and an absent +# key as an absent key. +setting_present() { + kubectl -n "$SYSTEM_NS" get secret gpustack-settings -o json 2>/dev/null \ + | jq -e --arg k "$1" '.data[$k] != null' >/dev/null 2>&1 +} + +# settings_save KEY... remembers the value each KEY carries in the store and whether it carried +# one, so settings_restore can put it back exactly: absent restores as absent, empty as empty, +# and a value as itself. +settings_save() { + local key value present i + for key in "$@"; do + value="$(setting_get "$key")" + if setting_present "$key"; then present=1; else present=0; fi + i=0 + while [ "$i" -lt "${#SETTINGS_SAVED_KEYS[@]}" ] && [ "${SETTINGS_SAVED_KEYS[$i]}" != "$key" ]; do + i=$((i + 1)) + done + if [ "$i" -lt "${#SETTINGS_SAVED_KEYS[@]}" ]; then + SETTINGS_SAVED_VALUES[$i]="$value" + SETTINGS_SAVED_PRESENT[$i]="$present" + else + SETTINGS_SAVED_KEYS+=("$key") + SETTINGS_SAVED_VALUES+=("$value") + SETTINGS_SAVED_PRESENT+=("$present") + fi + done +} + +# settings_restore puts back every key settings_save remembered, and waits out the worker's read +# cache - but only when a key actually changed. A key that never existed restores as absent; a key +# that existed with an empty value restores as that empty value, because a present-but-empty key +# and a deleted key read differently downstream. +settings_restore() { + local key current present i changed=0 + i=0 + while [ "$i" -lt "${#SETTINGS_SAVED_KEYS[@]}" ]; do + key="${SETTINGS_SAVED_KEYS[$i]}" + present="${SETTINGS_SAVED_PRESENT[$i]}" + if setting_present "$key"; then current=1; else current=0; fi + if [ "$current" != "$present" ] \ + || { [ "$present" -eq 1 ] && [ "$(setting_get "$key")" != "${SETTINGS_SAVED_VALUES[$i]}" ]; }; then + changed=1 + if [ "$present" -eq 0 ]; then + setting_unset "$key" + else + setting_set "$key" "${SETTINGS_SAVED_VALUES[$i]}" + fi + fi + i=$((i + 1)) + done + [ "$changed" -eq 1 ] && settings_settle + return 0 +} diff --git a/.agents/skills/gpustack-operator-e2e/cases/_ssh-lib.sh b/.agents/skills/gpustack-operator-e2e/cases/_ssh-lib.sh new file mode 100644 index 000000000..9e515d473 --- /dev/null +++ b/.agents/skills/gpustack-operator-e2e/cases/_ssh-lib.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# +# _ssh-lib.sh — how the runner reaches an SSH-enabled Instance: the keypair and its Secret, a local +# port in front of the Instance's sshd, and the sessions themselves. +# +# NOT A CASE. The leading underscore keeps it out of the `case-N.sh` namespace: it carries no case +# header, no trap and no results table. Every one of those stays with the case that sources it, so a +# case still reads end to end on its own terms. +# +# A case uses it as: +# +# # shellcheck source=/dev/null +# . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_ssh-lib.sh" +# ssh_setup_key_secret default "$SECRET" "$KEYDIR/id" +# ssh_open_port_forward default "pod/$POD" 22 "$LOCAL_PORT" +# out="$(ssh_exec "$KEYDIR/id" "$LOCAL_PORT" 'cat /etc/hostname')" +# ssh_cleanup +# +# The temporary key and the local port are the caller's: every case brings its own, so nothing here +# is state two cases share. +# ------------------------------------------------------------------------------------------------ + +# The pids of the forwards this harness opened, and the options every session in it uses. The port +# goes in as -o Port= rather than as -p, so that ssh and sftp take one list. +PF_PID="" +FWD_PID="" +SSH_OPTS=() + +_ssh_opts() { + SSH_OPTS=(-o "Port=$2" -i "$1" -o IdentitiesOnly=yes + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=15) +} + +# ssh_setup_key_secret — write an ed25519 keypair at and +# publish its public half as the authorized_keys of the Secret in namespace . +ssh_setup_key_secret() { + local ns="$1" secret="$2" key="$3" + ssh-keygen -t ed25519 -f "$key" -N "" -q + kubectl -n "$ns" delete secret "$secret" --ignore-not-found >/dev/null 2>&1 + kubectl -n "$ns" create secret generic "$secret" --from-file=authorized_keys="${key}.pub" >/dev/null +} + +# ssh_open_port_forward — forward a local port to the +# Instance's sshd through kubectl, and wait until the local port accepts a TCP connection. +ssh_open_port_forward() { + local ns="$1" target="$2" remote_port="$3" local_port="$4" + kubectl -n "$ns" port-forward "$target" "${local_port}:${remote_port}" >/dev/null 2>&1 & + PF_PID=$! + for _ in $(seq 1 20); do + (exec 3<>"/dev/tcp/127.0.0.1/${local_port}") 2>/dev/null && { exec 3>&- 3<&-; break; } + sleep 1 + done +} + +# ssh_exec [command...] — run a command on the Instance over SSH. With no +# command, the session reads the caller's stdin, which is how a login shell is driven. +ssh_exec() { + local key="$1" port="$2" + shift 2 + _ssh_opts "$key" "$port" + ssh -T "${SSH_OPTS[@]}" root@127.0.0.1 "$@" +} + +# ssh_sftp — an sftp session on the Instance; the batch commands come from +# the caller's stdin. +ssh_sftp() { + local key="$1" port="$2" + _ssh_opts "$key" "$port" + sftp "${SSH_OPTS[@]}" root@127.0.0.1 +} + +# ssh_open_loopback_forward +# — forward a local TCP port through the SSH session to a port inside the Instance. +ssh_open_loopback_forward() { + local key="$1" port="$2" local_port="$3" remote_host="$4" remote_port="$5" + _ssh_opts "$key" "$port" + ssh -N -T "${SSH_OPTS[@]}" -o ExitOnForwardFailure=yes \ + -L "127.0.0.1:${local_port}:${remote_host}:${remote_port}" root@127.0.0.1 >/dev/null 2>&1 & + FWD_PID=$! +} + +# ssh_cleanup — stop every forward this harness opened. A forward that already exited needs no kill, +# and its pid is cleared either way, so a second call finds nothing to do. +ssh_cleanup() { + [ -n "$FWD_PID" ] && kill "$FWD_PID" 2>/dev/null + FWD_PID="" + [ -n "$PF_PID" ] && kill "$PF_PID" 2>/dev/null + PF_PID="" + return 0 +} diff --git a/.agents/skills/gpustack-operator-e2e/cases/_topology-tas-lib.sh b/.agents/skills/gpustack-operator-e2e/cases/_topology-tas-lib.sh index d02bd45ba..1dc1aedd4 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/_topology-tas-lib.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/_topology-tas-lib.sh @@ -42,6 +42,12 @@ E2E_SHIM_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" [ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# The ModelDeployment readers, renderers and dry-run harness live in _md-lib.sh, which the plain +# ModelDeployment cases read as well. The tas_md_* names below are thin aliases of it, so a TAS case +# and a plain ModelDeployment case share one implementation. +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_md-lib.sh" + # The label key that carries the topology this library schedules on. Zone is a standard Kubernetes # key every cloud writes; hostname is the level Kueue requires last when it is present at all. The # fixtures below spell both levels out, in that order. @@ -410,40 +416,19 @@ tas_source_nodefeatures() { # waiting, and that name sorts ahead of ordinary derived types); inactive types are excluded for # the mirror reason -- a deployment on one is admitted and then never scheduled. tas_md_usable_instance_type() { - kubectl get instancetypes.worker.gpustack.ai \ - -o jsonpath='{range .items[*]}{.metadata.name}|{.metadata.deletionTimestamp}|{.spec.inactive}{"\n"}{end}' \ - 2>/dev/null \ - | while IFS='|' read -r name deleting inactive; do - [ -n "$name" ] || continue - [ -z "$deleting" ] || continue - [ "$inactive" = true ] && continue - echo "$name" - break - done + md_usable_instance_type "$@" } # The ClusterQueue an InstanceType schedules into, resolved the way the operator resolves it: the # type's published entrance LocalQueue, then that LocalQueue's ClusterQueue. tas_md_cq_of_it() { - local lq - lq="$(kubectl get instancetypes.worker.gpustack.ai "$2" \ - -o jsonpath='{.status.entrance}' 2>/dev/null)" - [ -n "$lq" ] || return 0 - kubectl -n "$1" get localqueue.kueue.x-k8s.io "$lq" \ - -o jsonpath='{.spec.clusterQueue}' 2>/dev/null + md_cq_of_it "$@" } # One role block for a ModelDeployment. The image and command are placeholder plumbing: no reading # that matters is about what the container runs. tas_md_role_block() { - printf ' - name: %s\n' "$1" - [ -n "$2" ] && printf ' kind: %s\n' "$2" - printf ' instanceType: %s\n replicas: %s\n' "$3" "$4" - printf ' image: %s\n command: ["/pause"]\n' "$5" - [ -n "${6:-}" ] && printf ' size: %s\n' "$6" - if [ -n "${7:-}" ]; then - printf ' topology:\n requiredLevel: %s\n' "$7" - fi + md_role_block "$@" } # Apply one ModelDeployment. The InstanceType, the image and the command live in the role blocks @@ -451,55 +436,21 @@ tas_md_role_block() { # The apply's output is kept in TAS_APPLY_OUT: a schema or webhook refusal surfaces as text a row # can quote instead of as a timeout that blames the wrong thing. tas_md_apply() { - local name="$1" ns="$2" binding="$3" roles="$4" + md_apply "$@" # shellcheck disable=SC2034 # Output consumed by the sourcing case. - TAS_APPLY_OUT="$(cat <&1 -apiVersion: worker.gpustack.ai/v1alpha1 -kind: ModelDeployment -metadata: - name: ${name} - namespace: ${ns} -spec: - engine: - name: vLLM - image: registry.k8s.io/pause:3.10 - model: - name: Qwen/Qwen2.5-0.5B-Instruct - kvCache: - poolRef: - name: ${binding} - roles: -${roles} -YAML -)" + TAS_APPLY_OUT="$MD_APPLY_OUT" } # Every Workload of one ModelDeployment, one name per line. # # THE LOOKUP RUNS WORKLOAD-TO-DEPLOYMENT, never deployment-to-Workload: the Workload is enumerated, -# each of its owner references is read as a Pod NAME and UID, and the Workload is attributed when -# either matches a Pod this deployment owns. Looking the other way -- searching for a Workload that -# carries a direct ModelDeployment owner -- finds nothing on any correct cluster, because Kueue's -# pod integration writes no such reference; a case built on it would pass as vacuously as it -# failed. The UIDs are matched padded so one UID cannot hit a prefix of a longer one. +# each of its owner references is read, and the Workload is attributed when one of them is a Pod this +# deployment owns. Looking the other way -- searching for a Workload that carries a direct +# ModelDeployment owner -- finds nothing on any correct cluster, because Kueue's pod integration +# writes no such reference; a case built on it would pass as vacuously as it failed. The UIDs are +# matched padded so one UID cannot hit a prefix of a longer one. tas_md_workloads() { - local ns="$1" md="$2" pods uids names row wl refs t - pods="$(kubectl -n "$ns" get pods -l "app.kubernetes.io/instance=${md}" \ - -o jsonpath='{range .items[*]}{.metadata.name}={.metadata.uid}{"\n"}{end}' 2>/dev/null)" - [ -n "$pods" ] || return 0 - uids="$(printf '%s\n' "$pods" | cut -d= -f2 | tr '\n' ' ')" - names="$(printf '%s\n' "$pods" | cut -d= -f1 | tr '\n' ' ')" - - kubectl -n "$ns" get workloads.kueue.x-k8s.io \ - -o jsonpath='{range .items[*]}{.metadata.name}|{range .metadata.ownerReferences[*]}{.name}/{.uid}{";"}{end}{"\n"}{end}' \ - 2>/dev/null \ - | while IFS='|' read -r wl refs; do - [ -n "$wl" ] || continue - for t in ${refs//;/ }; do - case " ${uids} " in *" ${t##*/} "*) echo "$wl"; continue 2 ;; esac - case " ${names} " in *" ${t%/*} "*) echo "$wl"; continue 2 ;; esac - done - done + md_workloads "$@" } # Wait until a ModelDeployment holds exactly $3 Pods. Kueue composes nothing for a group short of @@ -529,27 +480,5 @@ tas_md_wait_workloads() { # this runs from teardown, a deployment of several replicas holds several Workloads, and releasing # one of them leaves the rest holding finalizers on Pods a later case then inherits. tas_md_force_release() { - local ns="$1" md="$2" uids row wl owners u - uids="$(kubectl -n "$ns" get pods -l "app.kubernetes.io/instance=${md}" \ - -o jsonpath='{range .items[*]}{.metadata.uid}{"\n"}{end}' 2>/dev/null)" - [ -n "$uids" ] || return 0 - while IFS= read -r row; do - [ -n "$row" ] || continue - wl="${row%%=*}" - owners="${row#*=}" - for u in $uids; do - case " $owners " in - *" $u "*) - kubectl -n "$ns" delete workloads.kueue.x-k8s.io "$wl" \ - --ignore-not-found --wait=false >/dev/null 2>&1 - break - ;; - esac - done - done </dev/null) -EOF - - return 0 + md_force_release "$@" } diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-1.sh b/.agents/skills/gpustack-operator-e2e/cases/case-1.sh index 1dff3259d..0abe96100 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-1.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-1.sh @@ -37,8 +37,8 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: case-1.sh }" LIB="$(cd "$(dirname "$0")/../../_e2e-lib/scripts" && pwd)" @@ -46,9 +46,6 @@ LIB="$(cd "$(dirname "$0")/../../_e2e-lib/scripts" && pwd)" # Operator core first (rollout / revision==HEAD / apiservices / CRDs / bundled applications). bash "$LIB/assert-core.sh" "$NS" || exit 1 -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } assert_nonempty() { # check command-output if [ -n "$2" ]; then record PASS "$1" "$(echo "$2" | tr '\n' ' ' | cut -c1-60)"; else record FAIL "$1" "none found"; fi } @@ -161,10 +158,7 @@ cohorts=$(kubectl get cohorts.kueue.x-k8s.io -A --no-headers 2>/dev/null | grep echo echo "== CASE 1 — CPU-only scheduling chain materializes ==" -{ - echo "STATUS|CHECK|OBJECT" - printf '%s\n' "${ROWS[@]}" -} | column -t -s '|' +print_rows if [ "$FAILS" -ne 0 ]; then echo diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-100.sh b/.agents/skills/gpustack-operator-e2e/cases/case-100.sh index a768173aa..b87159e18 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/case-100.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-100.sh @@ -30,8 +30,6 @@ # Cleanup: Trap deletes the deployments, the artifacts, the Binding, the pool and the backend. set -uo pipefail -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" # shellcheck source=/dev/null . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" @@ -52,10 +50,6 @@ P=c100 LABEL="e2e.gpustack.ai/case=100" SERVED=e2e/c100 -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } - cleanup() { echo echo "[case-100] cleanup" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-101.sh b/.agents/skills/gpustack-operator-e2e/cases/case-101.sh index ad9f080a2..5c03f1839 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/case-101.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-101.sh @@ -1,28 +1,37 @@ #!/usr/bin/env bash # -# CASE 101 — The node plugin delivers a Hugging Face artifact: one verified download per node, zero -# bytes for the next Pod, never a corrupted or partial tree, and a mount only for the -# Pod's own namespace (MUTATING, self-recovering; AUTO-SKIPS the real-Hub row with -# E2E_C101_OFFLINE=1) +# CASE 101 — Node delivery and the control plane that owns it: one verified download per node, each +# status guard admitting only its own writer, a Setting that reaches every node, the +# CSIDriver Node delivery needs, and an HTTPS proxy carrying every download +# (MUTATING, self-recovering; AUTO-SKIPS the real-Hub row with E2E_C101_OFFLINE=1) # # case-101.sh # # Goal: Prove node delivery end to end with bare Pods, which is what a tenant can write by -# hand: a cold mount lasting longer than one kubelet mount call downloads each file once -# for two Pods and serves the manifest's bytes; a third Pod on the node downloads -# nothing; a corrupting source is never published; a plugin killed mid-download never -# exposes a partial tree; a namespace enforcing restricted mounts; forged volume -# attributes are refused by the rule they break; the real Hub serves through its -# redirects; and no token reaches any output. +# hand, and the control-plane contract that governs it on a real API server: a cold +# mount lasting longer than one kubelet mount call downloads each file once for two +# Pods and serves the manifest's bytes; a third Pod on the node downloads nothing; a +# corrupting source is never published; a plugin killed mid-download never exposes a +# partial tree; a namespace enforcing restricted mounts; forged volume attributes are +# refused by the rule they break; each status guard admits its writer and refuses the +# others by the rule it names; a Setting change rewrites every node's spec and every +# plugin applies it; without the CSIDriver Node delivery is refused and every +# NodeModelStore is removed; a cold download through an HTTPS proxy reaches the hub +# only from the proxy; the real Hub serves through its redirects; and no token reaches +# any output. # Environment: A cluster installed from this chart with modelManager.enabled (the default), two or -# more schedulable workers, and the stock python image pullable. NO GPU, no engine, no -# InstanceType: the consumers are bare Pods. The real-Hub row needs the public Hugging -# Face Hub from the nodes and is SKIPPED with E2E_C101_OFFLINE=1. +# more schedulable workers, the stock python image pullable and openssl on this machine. +# NO GPU, no engine, no InstanceType: the consumers are bare Pods. The real-Hub row +# needs the public Hugging Face Hub from the nodes and is SKIPPED with +# E2E_C101_OFFLINE=1. # Inputs: MOCKED: a test hub (_model-hub.py) in standing in for the Hugging Face Hub, -# whose repositories are generated from their names, so every digest is known; a random -# token for its private repository, created here and never printed. Real: the plugin, -# kubelet, admission, and hf-internal-testing/tiny-random-bert on the real Hub, -# filtered to its JSON, text and safetensors files (about 0.5 MB of its 27 MB). +# whose repositories are generated from their names, so every digest is known; a second +# hub over HTTPS with a self-signed certificate made here, handed to the plugin and the +# controller through the model-artifact-ca-bundle Setting; a forward proxy; a random +# token for the hub's private repository, created here and never printed. Real: the +# plugin, kubelet, admission, the webhooks, the kubelet-issued tokens, the plugin's +# downloads, and hf-internal-testing/tiny-random-bert on the real Hub, filtered to its +# JSON, text and safetensors files (about 0.5 MB of its 27 MB). # Expected: - cold: both Pods Running, each file fetched once, the cold mount outlasting 120 s, # every file's SHA-256 equal to the hub's; # - warm: a third Pod Running, no new request at the hub, the node's download counter @@ -36,15 +45,24 @@ # - forged: another namespace's artifact, its own artifact with another digest, and a # forged pod.namespace attribute are each refused with their rule, beside a Pod of the # same namespace that mounts; +# - guards: the plugin's token writes only its own node's status, a token bound to no +# Pod and the cluster administrator are refused, and the worker alone writes a +# ModelArtifact's status; +# - a Setting change reaches every node's spec and every plugin applies it; +# - without the CSIDriver Node is refused and every NodeModelStore is removed; with it +# back Node is accepted and every NodeModelStore returns; +# - through the HTTPS proxy every file the hub serves is fetched from the proxy's +# address, and without it from the plugin's; equal addresses make that row undecidable; # - real Hub: a Pod runs on the real repository and its model.safetensors hashes as this # machine's own download of it at the resolved commit; # - the token is in no log, event, status or metric, after a planted copy is found. -# Cleanup: Trap deletes the Pods, artifacts, Secrets, the second namespace and the hub, and puts -# back the Settings and namespace PSA labels it changed. +# Cleanup: Trap deletes the Pods, artifacts, Secrets, the second namespace, the hubs, the proxy +# and the CA ConfigMap, puts back the CSIDriver and the Settings it changed, and +# restores the namespace PSA labels it changed. set -uo pipefail -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" CASES_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null . "${CASES_DIR}/_rows-lib.sh" @@ -58,14 +76,15 @@ if [ "$NS" = "$SYSTEM_NS" ]; then fi NSB="${NS}-b" P=c101 -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } +TLS_HUB="${P}-tlshub" PSA_SNAPSHOT=0 SCRATCH="$(mktemp -d)" TOKEN="hf_$(python3 -c 'import secrets; print(secrets.token_hex(16))')" -ORIG_ENDPOINT="$(setting_get model-artifact-huggingface-endpoint)" -ORIG_DELIVERY="$(setting_get model-artifact-delivery-mode)" +SA="${E2E_MODEL_MANAGER_SA:-gpustack-operator-model-manager}" +# Remember what the store carried before the case touched it, so the cleanup puts every key back. +settings_save model-artifact-huggingface-endpoint model-artifact-https-proxy \ + model-artifact-no-proxy model-artifact-ca-bundle model-store-download-concurrency \ + model-artifact-delivery-mode cleanup() { local labels @@ -76,12 +95,15 @@ cleanup() { kubectl -n "$ns" delete modelartifacts.worker.gpustack.ai -l e2e.gpustack.ai/case=101 --ignore-not-found >/dev/null 2>&1 done kubectl -n "$NS" delete secret "${P}-token" --ignore-not-found >/dev/null 2>&1 - kubectl -n "$NS" delete deploy,svc,configmap -l e2e.gpustack.ai/model-hub=true --ignore-not-found >/dev/null 2>&1 - kubectl -n "$NS" delete deploy,svc "${P}-hub" --ignore-not-found >/dev/null 2>&1 - kubectl -n "$NS" delete configmap "${P}-hub-script" --ignore-not-found >/dev/null 2>&1 + for n in "${P}-hub" "$TLS_HUB" "${P}-proxy"; do + kubectl -n "$NS" delete deploy,svc "$n" --ignore-not-found >/dev/null 2>&1 + kubectl -n "$NS" delete configmap "${n}-script" --ignore-not-found >/dev/null 2>&1 + done + kubectl -n "$NS" delete secret "${P}-tls" --ignore-not-found >/dev/null 2>&1 + kubectl -n "$SYSTEM_NS" delete configmap "${P}-ca" --ignore-not-found >/dev/null 2>&1 kubectl delete namespace "$NSB" --ignore-not-found --wait=false >/dev/null 2>&1 - if [ -n "$ORIG_ENDPOINT" ]; then setting_set model-artifact-huggingface-endpoint "$ORIG_ENDPOINT"; else setting_unset model-artifact-huggingface-endpoint; fi - if [ -n "$ORIG_DELIVERY" ]; then setting_set model-artifact-delivery-mode "$ORIG_DELIVERY"; else setting_unset model-artifact-delivery-mode; fi + kubectl get csidriver model.csi.gpustack.ai >/dev/null 2>&1 || kubectl apply -f "$SCRATCH/csidriver.yaml" >/dev/null 2>&1 + settings_restore rm -rf "$SCRATCH" if [ "$PSA_SNAPSHOT" = 1 ]; then labels=(pod-security.kubernetes.io/enforce- pod-security.kubernetes.io/enforce-version-) @@ -124,6 +146,13 @@ if ! kubectl get csidriver model.csi.gpustack.ai >/dev/null 2>&1; then echo "[case-101] the model-manager plugin is not installed; NOTHING WAS VERIFIED" exit 2 fi +# The object to put back: its name, labels, annotations and spec, without what the API server set. +kubectl get csidriver model.csi.gpustack.ai -o json | python3 -c " +import json, sys +o = json.load(sys.stdin) +m = o['metadata'] +print(json.dumps({'apiVersion': o['apiVersion'], 'kind': o['kind'], 'spec': o['spec'], + 'metadata': {k: m[k] for k in ('name', 'labels', 'annotations') if k in m}}))" >"$SCRATCH/csidriver.yaml" for ns in "$NS" "$NSB"; do kubectl create namespace "$ns" --dry-run=client -o yaml | kubectl apply -f - >/dev/null || exit 1 @@ -145,6 +174,7 @@ print(json.dumps({ 'e2e/corrupt': {'files': {'config.json': {'size': 300}, 'model.safetensors': {'size': mib, 'lfs': True}}}, 'e2e/restart': {'files': {'config.json': {'size': 300}, 'model.safetensors': {'size': 8 * mib, 'lfs': True}}}, 'e2e/other': {'files': {'config.json': {'size': 400}}}, + 'e2e/guard': {'files': {'config.json': {'size': 100}}}, 'e2e/private': {'token': '${TOKEN}', 'files': {'config.json': {'size': 500}, 'w.bin': {'size': 70000, 'lfs': True}}}, }))")" HUB_URL="$(mh_deploy "$NS" "${P}-hub" "$REPOS")" @@ -329,10 +359,209 @@ else record FAIL "the token is in no output ($(grep -c -F "$TOKEN" "$SCRATCH/scan") matches)" "all" fi +# ---------------------------------------------------------------- the NodeModelStore status guard +SERVER="$(kubectl config view --minify --raw -o jsonpath='{.clusters[0].cluster.server}')" +kubectl config view --minify --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}' | base64 -d >"$SCRATCH/ca.crt" +# status_as TOKEN NODE: an update of NODE's NodeModelStore status as the bearer of TOKEN alone, the +# one verb the plugin's role holds on status and the one it sends, printing the API server's answer +# and succeeding on a 2xx. The object is read as the administrator, with a changed usedPercent, and +# put back as TOKEN. +status_as() { + local code + kubectl get nodemodelstores.v1alpha1.worker.gpustack.ai "$2" -o json | python3 -c " +import json, sys +o = json.load(sys.stdin) +o.setdefault('status', {}).setdefault('capacity', {})['usedPercent'] = 95 +print(json.dumps(o))" >"$SCRATCH/object" + printf 'Authorization: Bearer %s\n' "$1" >"$SCRATCH/auth" + code="$(curl -sS -o "$SCRATCH/answer" -w '%{http_code}' --cacert "$SCRATCH/ca.crt" -H @"$SCRATCH/auth" \ + -X PUT -H 'Content-Type: application/json' --data @"$SCRATCH/object" \ + "${SERVER}/apis/worker.gpustack.ai/v1alpha1/nodemodelstores/$2/status")" + rm -f "$SCRATCH/auth" + cat "$SCRATCH/answer" + case "$code" in 2*) return 0 ;; *) return 1 ;; esac +} +PLUGIN1="$(plugin_pod "$W1")" +BOUND="$(kubectl -n "$SYSTEM_NS" create token "$SA" --bound-object-kind Pod --bound-object-name "$PLUGIN1" --duration=10m)" +UNBOUND="$(kubectl -n "$SYSTEM_NS" create token "$SA" --duration=10m)" +patch='{"status":{"capacity":{"totalBytes":1,"storedBytes":1,"usedPercent":95}}}' +if status_as "$BOUND" "$W1" >"$SCRATCH/own" 2>&1; then + record PASS "the plugin's token bound to its Pod writes its own node's status (the baseline)" "$W1" +else + record FAIL "the plugin writes its own node's status" "$(tr -d "\n" <"$SCRATCH/own" | cut -c1-300)" +fi +for row in "$BOUND|$W2|the plugin writes only the status of the node its Pod runs on|the plugin on another node" \ + "$UNBOUND|$W1|the plugin's token must be bound to its Pod|a token bound to no Pod" \ + "|$W1|only the model-manager plugin writes a NodeModelStore's status|the cluster administrator"; do + IFS='|' read -r token node rule who <<<"$row" + if [ -n "$token" ]; then + status_as "$token" "$node" >"$SCRATCH/out" 2>&1 + else + kubectl patch nodemodelstores.v1alpha1.worker.gpustack.ai "$node" --subresource=status --type=merge -p "$patch" >"$SCRATCH/out" 2>&1 + fi + rc=$? + if [ "$rc" != 0 ] && grep -q -F "$rule" "$SCRATCH/out"; then + record PASS "${who} is refused: ${rule}" "$node" + else + record FAIL "${who} is refused: ${rule} (rc ${rc})" "$(tr -d "\n" <"$SCRATCH/out" | cut -c1-300)" + fi +done + +# ---------------------------------------------------------------- the ModelArtifact status guard +artifact "$NS" "${P}-guard" e2e/guard "" 101 +if [ -n "$(wait_resolved "$NS" "${P}-guard" 120)" ]; then + record PASS "the worker's own status writes are admitted: a new artifact resolves" "${NS}/${P}-guard" +else + record FAIL "the worker's own status writes are admitted" "${NS}/${P}-guard" +fi +kubectl -n "$NS" patch modelartifacts.v1alpha1.worker.gpustack.ai "${P}-guard" --subresource=status --type=merge \ + -p '{"status":{"resolved":{"manifestDigest":"sha256:0000000000000000000000000000000000000000000000000000000000000000"}}}' >"$SCRATCH/ma" 2>&1 +rc=$? +if [ "$rc" != 0 ] && grep -q -F "only the worker writes a ModelArtifact's status" "$SCRATCH/ma"; then + record PASS "the administrator's ModelArtifact status write is refused: only the worker writes it" "${NS}/${P}-guard" +else + record FAIL "the administrator's ModelArtifact status write is refused (rc ${rc})" "$(head -1 "$SCRATCH/ma")" +fi + +# ---------------------------------------------------------------- a Setting reaches every node +cur="$(setting_get model-store-download-concurrency)" +want=5 +[ "$cur" = 5 ] && want=6 +setting_set model-store-download-concurrency "$want" +converged=0 +for _ in $(seq 1 30); do + converged=1 + for n in $(kubectl get nodemodelstores.worker.gpustack.ai -o name); do + got="$(kubectl get "$n" -o jsonpath='{.spec.download.concurrency}|{.metadata.generation}|{.status.observedGeneration}')" + IFS='|' read -r c g o <<<"$got" + { [ "$c" = "$want" ] && [ "$g" = "$o" ]; } || converged=0 + done + [ "$converged" = 1 ] && break + sleep 2 +done +count="$(kubectl get nodemodelstores.worker.gpustack.ai -o name | wc -l | tr -d ' ')" +if [ "$converged" = 1 ] && [ "$count" -ge 2 ]; then + record PASS "model-store-download-concurrency=${want} reached ${count} nodes' spec and every plugin applied it" "all nodes" +else + record FAIL "the Setting reached every node's spec and observedGeneration within 60 s" "$(kubectl get nodemodelstores.worker.gpustack.ai -o jsonpath='{range .items[*]}{.metadata.name}={.spec.download.concurrency}/{.metadata.generation}/{.status.observedGeneration} {end}')" +fi +# The restore puts back every key the case saved, the endpoint among them, so the endpoint +# carries its own value again until the case points it at the TLS hub. +settings_restore + +# ---------------------------------------------------------------- Node delivery needs the CSIDriver +set_delivery() { + kubectl -n "$SYSTEM_NS" patch settings.gpustack.ai model-artifact-delivery-mode --type=merge \ + -p "{\"spec\":{\"value\":\"$1\"}}" >"$SCRATCH/set" 2>&1 +} +set_delivery Engine || record FAIL "Engine is accepted" "$(head -1 "$SCRATCH/set")" +kubectl delete csidriver model.csi.gpustack.ai >/dev/null +gone=0 +for _ in $(seq 1 30); do [ -z "$(kubectl get nodemodelstores.worker.gpustack.ai -o name)" ] && { gone=1; break; }; sleep 2; done +if [ "$gone" = 1 ]; then + record PASS "without the CSIDriver every NodeModelStore is removed" "all nodes" +else + record FAIL "without the CSIDriver every NodeModelStore is removed" "$(kubectl get nodemodelstores.worker.gpustack.ai -o name | tr '\n' ' ')" +fi +if set_delivery Node; then + record FAIL "Node is refused while the CSIDriver is absent" "model-artifact-delivery-mode" +else + if grep -q "does not exist" "$SCRATCH/set"; then + record PASS "Node is refused while the CSIDriver is absent" "model-artifact-delivery-mode" + else + record FAIL "Node is refused for the CSIDriver" "$(head -1 "$SCRATCH/set")" + fi +fi +kubectl apply -f "$SCRATCH/csidriver.yaml" >/dev/null +back=0 +for _ in $(seq 1 60); do [ "$(kubectl get nodemodelstores.worker.gpustack.ai -o name | wc -l | tr -d ' ')" -ge "$count" ] && { back=1; break; }; sleep 2; done +if [ "$back" = 1 ]; then + record PASS "with the CSIDriver back every NodeModelStore returns" "${count} nodes" +else + record FAIL "with the CSIDriver back every NodeModelStore returns" "$(kubectl get nodemodelstores.worker.gpustack.ai -o name | wc -l)" +fi +if set_delivery Node; then + record PASS "with the CSIDriver back Node is accepted" "model-artifact-delivery-mode" +else + record FAIL "with the CSIDriver back Node is accepted" "$(head -1 "$SCRATCH/set")" +fi +for _ in $(seq 1 30); do + [ "$(kubectl get nodemodelstore "$W1" -o jsonpath="{.status.conditions[?(@.type=='Ready')].status}" 2>/dev/null)" = True ] && break + sleep 2 +done + +# ---------------------------------------------------------------- an HTTPS proxy, with its control +HOST="${TLS_HUB}.${NS}.svc" +TLS_REPOS='{"e2e/proxied":{"files":{"config.json":{"size":200},"w.bin":{"size":300000,"lfs":true}}},"e2e/direct":{"files":{"config.json":{"size":210},"w.bin":{"size":310000,"lfs":true}}}}' +openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj "/CN=${HOST}" -addext "subjectAltName=DNS:${HOST}" \ + -keyout "$SCRATCH/tls.key" -out "$SCRATCH/tls.crt" >/dev/null 2>&1 +kubectl -n "$NS" create secret tls "${P}-tls" --cert="$SCRATCH/tls.crt" --key="$SCRATCH/tls.key" >/dev/null +kubectl -n "$SYSTEM_NS" create configmap "${P}-ca" --from-file=ca.crt="$SCRATCH/tls.crt" >/dev/null +TLS_URL="$(mh_deploy "$NS" "$TLS_HUB" "$TLS_REPOS" "${P}-tls")" +PROXY_URL="$(mh_proxy "$NS" "${P}-proxy")" +setting_set model-artifact-ca-bundle "${P}-ca" +setting_set model-artifact-no-proxy "" +setting_set model-artifact-https-proxy "$PROXY_URL" +setting_set model-artifact-huggingface-endpoint "$TLS_URL" +settings_settle + +# remotes REPO: the distinct client addresses the hub served REPO's file downloads to. +remotes() { + mh_log "$NS" "$TLS_HUB" | python3 -c " +import json, sys +print(' '.join(sorted({json.loads(l)['remote'] for l in sys.stdin if l.strip() and json.loads(l)['method'] == 'GET' + and json.loads(l)['path'].startswith('/$1/') and json.loads(l).get('status') in (200, 206)})))" +} +fetch() { # name repo pod + artifact "$NS" "$1" "$2" "" 101 + local d + d="$(wait_resolved "$NS" "$1" 120)" + [ -n "$d" ] || return 1 + consumer "$NS" "$3" "$W1" "$1" "$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "$1" -o jsonpath='{.metadata.uid}')" "$d" + pod_ready "$NS" "$3" 240 +} +PROXY_IP="$(mh_pod_ip "$NS" "${P}-proxy")" +PLUGIN_IP="$(kubectl -n "$SYSTEM_NS" get pod "$(plugin_pod "$W1")" -o jsonpath='{.status.podIP}')" +if fetch "${P}-proxied" e2e/proxied "${P}-pp"; then + via="$(remotes e2e/proxied)" + if [ "$via" = "$PROXY_IP" ]; then + record PASS "through the HTTPS proxy every file reached the hub from the proxy (${via})" "e2e/proxied" + else + record FAIL "every file reached the hub from the proxy ${PROXY_IP} (saw ${via})" "e2e/proxied" + fi + tunnels="$(mh_log "$NS" "${P}-proxy" | grep -c -F "\"connect\": \"${HOST}:8443\"")" + if [ "$tunnels" -ge 1 ]; then + record PASS "the proxy tunneled to the hub (${tunnels} tunnels)" "${P}-proxy" + else + record FAIL "the proxy tunneled to the hub" "${P}-proxy" + fi +else + record FAIL "a cold download through the HTTPS proxy and the CA bundle mounts" "$(pod_mount_events "$NS" "${P}-pp" | tail -1)" +fi +setting_set model-artifact-https-proxy "" +settings_settle +if fetch "${P}-direct" e2e/direct "${P}-pd"; then + direct="$(remotes e2e/direct)" + if [ "$PROXY_IP" = "$PLUGIN_IP" ]; then + record SKIP "undecidable: the proxy and the plugin share the address ${PLUGIN_IP}" "e2e/direct" + else + if [ "$direct" = "$PLUGIN_IP" ]; then + record PASS "without the proxy the files reached the hub from the plugin (${direct}), so the address tells the paths apart" "e2e/direct" + else + record FAIL "without the proxy the files reached the hub from the plugin ${PLUGIN_IP} (saw ${direct})" "e2e/direct" + fi + fi +else + record FAIL "a cold download without the proxy mounts" "$(pod_mount_events "$NS" "${P}-pd" | tail -1)" +fi + # ---------------------------------------------------------------- the real Hub if [ "${E2E_C101_OFFLINE:-0}" = 1 ]; then record SKIP "the real Hub (E2E_C101_OFFLINE=1): NOTHING WAS VERIFIED on it" "hf-internal-testing/tiny-random-bert" else + # The test CA bundle and the proxy would stand between the plugin and the real Hub, so both go + # before the endpoint moves. + setting_unset model-artifact-ca-bundle setting_set model-artifact-huggingface-endpoint https://huggingface.co settings_settle REAL=hf-internal-testing/tiny-random-bert diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-102.sh b/.agents/skills/gpustack-operator-e2e/cases/case-102.sh deleted file mode 100644 index 01ec52509..000000000 --- a/.agents/skills/gpustack-operator-e2e/cases/case-102.sh +++ /dev/null @@ -1,291 +0,0 @@ -#!/usr/bin/env bash -# -# CASE 102 — Only the plugin on a node writes that node's NodeModelStore status, only the worker writes -# a ModelArtifact's; a Setting reaches every node's effective configuration; Node delivery -# needs the CSIDriver; and an HTTPS proxy carries every download (MUTATING, self-recovering) -# -# case-102.sh -# -# Goal: Prove the control-plane contract of node delivery on a real API server, so that -# admission and kubelet-issued tokens are the real ones: each status guard admits its -# writer and refuses the others by the rule it names; a Setting change rewrites every -# node's spec and every plugin applies it; the delivery Setting refuses Node while the -# plugin is absent, the worker removes every NodeModelStore then and recreates them after; -# and a cold download through an HTTPS proxy reaches the hub only from the proxy. -# Environment: A cluster installed from this chart with modelManager.enabled, two or more workers, the -# stock python image pullable, and openssl on this machine. NO GPU. -# Inputs: MOCKED: a test hub over HTTPS with a self-signed certificate made here, handed to the -# plugin and the controller through the model-artifact-ca-bundle Setting, and a forward -# proxy. Real: the webhooks, the kubelet-issued tokens and the plugin's downloads. -# Expected: - the plugin's token bound to its Pod on node A updates node A's status, and is refused -# on node B by the node rule; a token bound to no Pod is refused by the binding rule; the -# cluster administrator is refused by the identity rule; -# - the administrator's ModelArtifact status update is refused by the worker rule, and the -# worker's own writes land: a new artifact resolves; -# - model-store-download-concurrency reaches every NodeModelStore's spec, and every plugin -# reports that generation, within 60 s; -# - without the CSIDriver, writing Node is refused and every NodeModelStore is removed; -# with it back, Node is accepted and every NodeModelStore returns; -# - through the HTTPS proxy every file the hub serves is fetched from the proxy's address, -# and without it from the plugin's; equal addresses make that row undecidable. -# Cleanup: Trap restores the Settings and the CSIDriver, and deletes the Pods, artifacts, hub, -# proxy, the CA ConfigMap and the TLS Secret. -set -uo pipefail - -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" -CASES_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=/dev/null -. "${CASES_DIR}/_rows-lib.sh" -# shellcheck source=/dev/null -. "${CASES_DIR}/_model-hub-lib.sh" - -NS="${1:?usage: case-102.sh }" -P=c102 -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } -SCRATCH="$(mktemp -d)" -SA="${E2E_MODEL_MANAGER_SA:-gpustack-operator-model-manager}" -KEYS="model-artifact-huggingface-endpoint model-artifact-https-proxy model-artifact-no-proxy - model-artifact-ca-bundle model-store-download-concurrency model-artifact-delivery-mode" -mkdir -p "$SCRATCH/orig" -for k in $KEYS; do - kubectl -n "$SYSTEM_NS" get secret gpustack-settings -o jsonpath="{.data.$k}" >"$SCRATCH/orig/$k" 2>/dev/null -done -# restore KEY: the value KEY had before the case, or no value when it had none. -restore() { - if [ -s "$SCRATCH/orig/$1" ]; then setting_set "$1" "$(base64 -d <"$SCRATCH/orig/$1")"; else setting_unset "$1"; fi -} - -cleanup() { - echo - echo "[case-102] cleanup" - kubectl get csidriver model.csi.gpustack.ai >/dev/null 2>&1 || kubectl apply -f "$SCRATCH/csidriver.yaml" >/dev/null 2>&1 - for k in $KEYS; do restore "$k"; done - kubectl -n "$NS" delete pods -l e2e.gpustack.ai/consumer=true --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 - kubectl -n "$NS" delete modelartifacts.worker.gpustack.ai -l e2e.gpustack.ai/case=102 --ignore-not-found >/dev/null 2>&1 - for n in "${P}-hub" "${P}-proxy"; do - kubectl -n "$NS" delete deploy,svc "$n" --ignore-not-found >/dev/null 2>&1 - kubectl -n "$NS" delete configmap "${n}-script" --ignore-not-found >/dev/null 2>&1 - done - kubectl -n "$NS" delete secret "${P}-tls" --ignore-not-found >/dev/null 2>&1 - kubectl -n "$SYSTEM_NS" delete configmap "${P}-ca" --ignore-not-found >/dev/null 2>&1 - rm -rf "$SCRATCH" -} -trap cleanup EXIT - -read -r -a WORKERS <<<"$(model_workers)" -[ "${#WORKERS[@]}" -ge 2 ] || { echo "[case-102] needs two workers; NOTHING WAS VERIFIED"; exit 2; } -W1="${WORKERS[0]}" W2="${WORKERS[1]}" -kubectl get csidriver model.csi.gpustack.ai >/dev/null 2>&1 \ - || { echo "[case-102] the model-manager plugin is not installed; NOTHING WAS VERIFIED"; exit 2; } -# The object to put back: its name, labels, annotations and spec, without what the API server set. -kubectl get csidriver model.csi.gpustack.ai -o json | python3 -c " -import json, sys -o = json.load(sys.stdin) -m = o['metadata'] -print(json.dumps({'apiVersion': o['apiVersion'], 'kind': o['kind'], 'spec': o['spec'], - 'metadata': {k: m[k] for k in ('name', 'labels', 'annotations') if k in m}}))" >"$SCRATCH/csidriver.yaml" -kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null - -# status_as TOKEN NODE: an update of NODE's NodeModelStore status as the bearer of TOKEN alone, the -# one verb the plugin's role holds on status and the one it sends, printing the API server's answer -# and succeeding on a 2xx. The object is read as the administrator, with a changed usedPercent, and -# put back as TOKEN. -SERVER="$(kubectl config view --minify --raw -o jsonpath='{.clusters[0].cluster.server}')" -kubectl config view --minify --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}' | base64 -d >"$SCRATCH/ca.crt" -status_as() { - local code - kubectl get nodemodelstores.v1alpha1.worker.gpustack.ai "$2" -o json | python3 -c " -import json, sys -o = json.load(sys.stdin) -o.setdefault('status', {}).setdefault('capacity', {})['usedPercent'] = 95 -print(json.dumps(o))" >"$SCRATCH/object" - printf 'Authorization: Bearer %s\n' "$1" >"$SCRATCH/auth" - code="$(curl -sS -o "$SCRATCH/answer" -w '%{http_code}' --cacert "$SCRATCH/ca.crt" -H @"$SCRATCH/auth" \ - -X PUT -H 'Content-Type: application/json' --data @"$SCRATCH/object" \ - "${SERVER}/apis/worker.gpustack.ai/v1alpha1/nodemodelstores/$2/status")" - rm -f "$SCRATCH/auth" - cat "$SCRATCH/answer" - case "$code" in 2*) return 0 ;; *) return 1 ;; esac -} - -# ---------------------------------------------------------------- the NodeModelStore status guard -PLUGIN1="$(plugin_pod "$W1")" -BOUND="$(kubectl -n "$SYSTEM_NS" create token "$SA" --bound-object-kind Pod --bound-object-name "$PLUGIN1" --duration=10m)" -UNBOUND="$(kubectl -n "$SYSTEM_NS" create token "$SA" --duration=10m)" -patch='{"status":{"capacity":{"totalBytes":1,"storedBytes":1,"usedPercent":95}}}' -if status_as "$BOUND" "$W1" >"$SCRATCH/own" 2>&1; then - record PASS "the plugin's token bound to its Pod writes its own node's status (the baseline)" "$W1" -else - record FAIL "the plugin writes its own node's status" "$(tr -d "\n" <"$SCRATCH/own" | cut -c1-300)" -fi -for row in "$BOUND|$W2|the plugin writes only the status of the node its Pod runs on|the plugin on another node" \ - "$UNBOUND|$W1|the plugin's token must be bound to its Pod|a token bound to no Pod" \ - "|$W1|only the model-manager plugin writes a NodeModelStore's status|the cluster administrator"; do - IFS='|' read -r token node rule who <<<"$row" - if [ -n "$token" ]; then - status_as "$token" "$node" >"$SCRATCH/out" 2>&1 - else - kubectl patch nodemodelstores.v1alpha1.worker.gpustack.ai "$node" --subresource=status --type=merge -p "$patch" >"$SCRATCH/out" 2>&1 - fi - rc=$? - if [ "$rc" != 0 ] && grep -q -F "$rule" "$SCRATCH/out"; then - record PASS "${who} is refused: ${rule}" "$node" - else - record FAIL "${who} is refused: ${rule} (rc ${rc})" "$(tr -d "\n" <"$SCRATCH/out" | cut -c1-300)" - fi -done - -# ---------------------------------------------------------------- the ModelArtifact status guard -REPOS='{"e2e/guard":{"files":{"config.json":{"size":100}}},"e2e/proxied":{"files":{"config.json":{"size":200},"w.bin":{"size":300000,"lfs":true}}},"e2e/direct":{"files":{"config.json":{"size":210},"w.bin":{"size":310000,"lfs":true}}}}' -PLAIN_URL="$(mh_deploy "$NS" "${P}-plain" "$REPOS")" -setting_set model-artifact-huggingface-endpoint "$PLAIN_URL" -settings_settle -artifact "$NS" "${P}-guard" e2e/guard "" 102 -if [ -n "$(wait_resolved "$NS" "${P}-guard" 120)" ]; then - record PASS "the worker's own status writes are admitted: a new artifact resolves" "${NS}/${P}-guard" -else - record FAIL "the worker's own status writes are admitted" "${NS}/${P}-guard" -fi -kubectl -n "$NS" patch modelartifacts.v1alpha1.worker.gpustack.ai "${P}-guard" --subresource=status --type=merge \ - -p '{"status":{"resolved":{"manifestDigest":"sha256:0000000000000000000000000000000000000000000000000000000000000000"}}}' >"$SCRATCH/ma" 2>&1 -rc=$? -if [ "$rc" != 0 ] && grep -q -F "only the worker writes a ModelArtifact's status" "$SCRATCH/ma"; then - record PASS "the administrator's ModelArtifact status write is refused: only the worker writes it" "${NS}/${P}-guard" -else - record FAIL "the administrator's ModelArtifact status write is refused (rc ${rc})" "$(head -1 "$SCRATCH/ma")" -fi -kubectl -n "$NS" delete deploy,svc "${P}-plain" --ignore-not-found >/dev/null 2>&1 -kubectl -n "$NS" delete configmap "${P}-plain-script" --ignore-not-found >/dev/null 2>&1 - -# ---------------------------------------------------------------- a Setting reaches every node -cur="$(setting_get model-store-download-concurrency)" -want=5 -[ "$cur" = 5 ] && want=6 -setting_set model-store-download-concurrency "$want" -converged=0 -for _ in $(seq 1 30); do - converged=1 - for n in $(kubectl get nodemodelstores.worker.gpustack.ai -o name); do - got="$(kubectl get "$n" -o jsonpath='{.spec.download.concurrency}|{.metadata.generation}|{.status.observedGeneration}')" - IFS='|' read -r c g o <<<"$got" - { [ "$c" = "$want" ] && [ "$g" = "$o" ]; } || converged=0 - done - [ "$converged" = 1 ] && break - sleep 2 -done -count="$(kubectl get nodemodelstores.worker.gpustack.ai -o name | wc -l | tr -d ' ')" -if [ "$converged" = 1 ] && [ "$count" -ge 2 ]; then - record PASS "model-store-download-concurrency=${want} reached ${count} nodes' spec and every plugin applied it" "all nodes" -else - record FAIL "the Setting reached every node's spec and observedGeneration within 60 s" "$(kubectl get nodemodelstores.worker.gpustack.ai -o jsonpath='{range .items[*]}{.metadata.name}={.spec.download.concurrency}/{.metadata.generation}/{.status.observedGeneration} {end}')" -fi -restore model-store-download-concurrency - -# ---------------------------------------------------------------- Node delivery needs the CSIDriver -set_delivery() { kubectl -n "$SYSTEM_NS" patch settings.gpustack.ai model-artifact-delivery-mode --type=merge -p "{\"spec\":{\"value\":\"$1\"}}" >"$SCRATCH/set" 2>&1; } -set_delivery Engine || record FAIL "Engine is accepted" "$(head -1 "$SCRATCH/set")" -kubectl delete csidriver model.csi.gpustack.ai >/dev/null -gone=0 -for _ in $(seq 1 30); do [ -z "$(kubectl get nodemodelstores.worker.gpustack.ai -o name)" ] && { gone=1; break; }; sleep 2; done -if [ "$gone" = 1 ]; then - record PASS "without the CSIDriver every NodeModelStore is removed" "all nodes" -else - record FAIL "without the CSIDriver every NodeModelStore is removed" "$(kubectl get nodemodelstores.worker.gpustack.ai -o name | tr '\n' ' ')" -fi -if set_delivery Node; then - record FAIL "Node is refused while the CSIDriver is absent" "model-artifact-delivery-mode" -else - if grep -q "does not exist" "$SCRATCH/set"; then - record PASS "Node is refused while the CSIDriver is absent" "model-artifact-delivery-mode" - else - record FAIL "Node is refused for the CSIDriver" "$(head -1 "$SCRATCH/set")" - fi -fi -kubectl apply -f "$SCRATCH/csidriver.yaml" >/dev/null -back=0 -for _ in $(seq 1 60); do [ "$(kubectl get nodemodelstores.worker.gpustack.ai -o name | wc -l | tr -d ' ')" -ge "$count" ] && { back=1; break; }; sleep 2; done -if [ "$back" = 1 ]; then - record PASS "with the CSIDriver back every NodeModelStore returns" "${count} nodes" -else - record FAIL "with the CSIDriver back every NodeModelStore returns" "$(kubectl get nodemodelstores.worker.gpustack.ai -o name | wc -l)" -fi -if set_delivery Node; then - record PASS "with the CSIDriver back Node is accepted" "model-artifact-delivery-mode" -else - record FAIL "with the CSIDriver back Node is accepted" "$(head -1 "$SCRATCH/set")" -fi -for _ in $(seq 1 30); do - [ "$(kubectl get nodemodelstore "$W1" -o jsonpath="{.status.conditions[?(@.type=='Ready')].status}" 2>/dev/null)" = True ] && break - sleep 2 -done - -# ---------------------------------------------------------------- an HTTPS proxy, with its control -HOST="${P}-hub.${NS}.svc" -openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj "/CN=${HOST}" -addext "subjectAltName=DNS:${HOST}" \ - -keyout "$SCRATCH/tls.key" -out "$SCRATCH/tls.crt" >/dev/null 2>&1 -kubectl -n "$NS" create secret tls "${P}-tls" --cert="$SCRATCH/tls.crt" --key="$SCRATCH/tls.key" >/dev/null -kubectl -n "$SYSTEM_NS" create configmap "${P}-ca" --from-file=ca.crt="$SCRATCH/tls.crt" >/dev/null -HUB_URL="$(mh_deploy "$NS" "${P}-hub" "$REPOS" "${P}-tls")" -PROXY_URL="$(mh_proxy "$NS" "${P}-proxy")" -setting_set model-artifact-ca-bundle "${P}-ca" -setting_set model-artifact-no-proxy "" -setting_set model-artifact-https-proxy "$PROXY_URL" -setting_set model-artifact-huggingface-endpoint "$HUB_URL" -settings_settle - -# remotes REPO: the distinct client addresses the hub served REPO's file downloads to. -remotes() { - mh_log "$NS" "${P}-hub" | python3 -c " -import json, sys -print(' '.join(sorted({json.loads(l)['remote'] for l in sys.stdin if l.strip() and json.loads(l)['method'] == 'GET' - and json.loads(l)['path'].startswith('/$1/') and json.loads(l).get('status') in (200, 206)})))" -} -fetch() { # name repo pod - artifact "$NS" "$1" "$2" "" 102 - local d - d="$(wait_resolved "$NS" "$1" 120)" - [ -n "$d" ] || return 1 - consumer "$NS" "$3" "$W1" "$1" "$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "$1" -o jsonpath='{.metadata.uid}')" "$d" - pod_ready "$NS" "$3" 240 -} -PROXY_IP="$(mh_pod_ip "$NS" "${P}-proxy")" -PLUGIN_IP="$(kubectl -n "$SYSTEM_NS" get pod "$(plugin_pod "$W1")" -o jsonpath='{.status.podIP}')" -if fetch "${P}-proxied" e2e/proxied "${P}-pp"; then - via="$(remotes e2e/proxied)" - if [ "$via" = "$PROXY_IP" ]; then - record PASS "through the HTTPS proxy every file reached the hub from the proxy (${via})" "e2e/proxied" - else - record FAIL "every file reached the hub from the proxy ${PROXY_IP} (saw ${via})" "e2e/proxied" - fi - tunnels="$(mh_log "$NS" "${P}-proxy" | grep -c -F "\"connect\": \"${HOST}:8443\"")" - if [ "$tunnels" -ge 1 ]; then - record PASS "the proxy tunneled to the hub (${tunnels} tunnels)" "${P}-proxy" - else - record FAIL "the proxy tunneled to the hub" "${P}-proxy" - fi -else - record FAIL "a cold download through the HTTPS proxy and the CA bundle mounts" "$(pod_mount_events "$NS" "${P}-pp" | tail -1)" -fi -setting_set model-artifact-https-proxy "" -settings_settle -if fetch "${P}-direct" e2e/direct "${P}-pd"; then - direct="$(remotes e2e/direct)" - if [ "$PROXY_IP" = "$PLUGIN_IP" ]; then - record SKIP "undecidable: the proxy and the plugin share the address ${PLUGIN_IP}" "e2e/direct" - else - if [ "$direct" = "$PLUGIN_IP" ]; then - record PASS "without the proxy the files reached the hub from the plugin (${direct}), so the address tells the paths apart" "e2e/direct" - else - record FAIL "without the proxy the files reached the hub from the plugin ${PLUGIN_IP} (saw ${direct})" "e2e/direct" - fi - fi -else - record FAIL "a cold download without the proxy mounts" "$(pod_mount_events "$NS" "${P}-pd" | tail -1)" -fi - -print_rows -[ "$FAILS" -eq 0 ] || { echo "[case-102] ${FAILS} check(s) FAILED"; exit 1; } -echo "[case-102] PASS" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-103.sh b/.agents/skills/gpustack-operator-e2e/cases/case-103.sh index e06cc8f2b..c6c6cf8a6 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/case-103.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-103.sh @@ -1,10 +1,12 @@ #!/usr/bin/env bash # -# CASE 103 — The node cache's lifecycle: a plugin roll leaves mounted Pods reading, collection removes -# the oldest unreferenced digest and never a referenced one, a filtered artifact downloads -# only its files, and an Instance and a ModelDeployment mount a hub artifact +# CASE 103 — The node cache's lifecycle and the node's own record of it: a plugin roll leaves mounted +# Pods reading, collection removes the oldest unreferenced digest and never a referenced +# one, a filtered artifact downloads only its files, an Instance and a ModelDeployment +# mount a hub artifact, the node's entries survive a plugin restart and a deleted +# object, and the record goes with the Node it belongs to # (MUTATING, self-recovering; AUTO-SKIPS the collection rows without docker access to the -# kind nodes) +# kind nodes, and the Node row without it as well) # # case-103.sh # @@ -14,13 +16,17 @@ # removes by age of last use down to the low watermark, keeps a digest a Pod references # even when told to go to 5%; allow patterns select the files downloaded, and a filtered # artifact cannot be Engine-delivered; an Instance and a ModelDeployment on Node delivery -# render the plugin's volume and run. +# render the plugin's volume and run; the node's own record of its content is rewritten +# from the node's disk after the plugin restarts and after the record itself is deleted, +# and deleting the Node collects the record with it. # Environment: A kind cluster installed from this chart with modelManager.enabled, two or more # workers, a materialized scheduling chain (run case-1 first) with an InstanceType, # docker on this machine reaching the kind node containers, and the stock python image # pullable. NO GPU. The collection rows mount a tmpfs over the second worker's cache -# root; without docker they SKIP. The whole case takes about fifteen minutes: the -# collector keeps a digest ten minutes after its last use. +# root and the Node row stops the second worker's kubelet while its Node object is +# deleted; without docker they SKIP. The whole case takes about half an hour: the +# collector keeps a digest ten minutes after its last use, and the node's entries are +# compared across a plugin restart. # Inputs: MOCKED: a test hub over HTTP with generated repositories, and a tmpfs as the second # worker's cache filesystem. Real: the plugin, kubelet, the collector and the worker. # Override with E2E_MD_INSTANCE_TYPE, E2E_MD_IMAGE. @@ -38,14 +44,20 @@ # and status.model.delivery Node; # - with a 100 MiB cache holding three 20 MiB digests, two unreferenced past their # grace and one referenced: watermarks 50/45 remove only the oldest; 10/5 remove the -# second and keep the referenced one. +# second and keep the referenced one; +# - after the plugin Pod on the first worker is replaced, the node's entries are the +# ones it held before; after the node's NodeModelStore is deleted, it comes back and +# the plugin rewrites the same entries from the node's disk; +# - after the second worker's Node object is deleted, its NodeModelStore is gone; after +# kubelet registers the node again, a new one exists with a new UID. # Cleanup: Trap deletes the Pods, deployments, Instances, artifacts and the hub, puts the # Settings back, restores the CSIDriver if it is missing, unmounts the tmpfs and -# restarts that node's plugin. +# restarts that node's plugin, and starts kubelet on a node whose Node object it +# deleted if that node is still missing. set -uo pipefail -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" CASES_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null . "${CASES_DIR}/_rows-lib.sh" @@ -59,25 +71,21 @@ IT="${E2E_MD_INSTANCE_TYPE:-}" IMAGE="${E2E_MD_IMAGE:-registry.k8s.io/pause:3.10}" DS=gpustack-operator-model-manager CACHE_ROOT=/var/lib/gpustack/models -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } SCRATCH="$(mktemp -d)" TMPFS_NODE="" +DELETED_NODE="" -KEYS="model-artifact-huggingface-endpoint model-artifact-delivery-mode model-store-high-watermark model-store-low-watermark" -mkdir -p "$SCRATCH/orig" -for k in $KEYS; do - kubectl -n "$SYSTEM_NS" get secret gpustack-settings -o jsonpath="{.data.$k}" >"$SCRATCH/orig/$k" 2>/dev/null -done -# restore KEY: the value KEY had before the case, or no value when it had none. -restore() { - if [ -s "$SCRATCH/orig/$1" ]; then setting_set "$1" "$(base64 -d <"$SCRATCH/orig/$1")"; else setting_unset "$1"; fi -} +# Remember what the store carried before the case touched it, so the cleanup puts every key back. +settings_save model-artifact-huggingface-endpoint model-artifact-delivery-mode \ + model-store-high-watermark model-store-low-watermark cleanup() { echo echo "[case-103] cleanup" + # A node whose Node object the case deleted gets its kubelet back, so it registers itself again. + if [ -n "$DELETED_NODE" ] && ! kubectl get node "$DELETED_NODE" >/dev/null 2>&1; then + docker exec "$DELETED_NODE" systemctl start kubelet >/dev/null 2>&1 + fi kubectl -n "$MDNS" delete modeldeployments.worker.gpustack.ai,instances.worker.gpustack.ai --all --ignore-not-found --wait=false >/dev/null 2>&1 for ns in "$NS" "$MDNS"; do kubectl -n "$ns" delete pods -l e2e.gpustack.ai/consumer=true --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 @@ -89,9 +97,7 @@ cleanup() { kubectl -n "$NS" delete deploy,svc "${P}-hub" --ignore-not-found >/dev/null 2>&1 kubectl -n "$NS" delete configmap "${P}-hub-script" --ignore-not-found >/dev/null 2>&1 kubectl delete namespace "$MDNS" --ignore-not-found --wait=false >/dev/null 2>&1 - for k in model-store-high-watermark model-store-low-watermark model-artifact-delivery-mode model-artifact-huggingface-endpoint; do - restore "$k" - done + settings_restore [ -s "$SCRATCH/csidriver.json" ] && ! kubectl get csidriver model.csi.gpustack.ai >/dev/null 2>&1 \ && kubectl apply -f "$SCRATCH/csidriver.json" >/dev/null 2>&1 if [ -n "$TMPFS_NODE" ]; then @@ -131,6 +137,30 @@ weights() { kubectl -n "$MDNS" get modeldeployments.worker.gpustack.ai "$1" \ -o jsonpath="{.status.conditions[?(@.type=='WeightsReady')].status}|{.status.conditions[?(@.type=='WeightsReady')].reason}" 2>/dev/null } +# entries NODE: the node's entries as sorted "digest state size referenced" lines. +entries() { + kubectl get nodemodelstores.worker.gpustack.ai "$1" -o json 2>/dev/null | python3 -c " +import json, sys +for m in sorted(json.load(sys.stdin).get('status', {}).get('models', []), key=lambda m: m['digest']): + print(m['digest'], m['state'], m.get('sizeBytes', 0), m.get('referenced', False))" +} +# wait_entries NODE WANT BOUND: waits until the node's entries equal WANT; 0 once they do. +wait_entries() { + for _ in $(seq 1 $(( $3 / 3 ))); do + [ "$(entries "$1")" = "$2" ] && return 0 + sleep 3 + done + return 1 +} +plugin_ready() { # plugin_ready NODE BOUND + local pod + for _ in $(seq 1 $(( $2 / 3 ))); do + pod="$(plugin_pod "$1")" + [ -n "$pod" ] && [ "$(kubectl -n "$SYSTEM_NS" get pod "$pod" -o jsonpath='{.status.containerStatuses[?(@.name=="main")].ready}')" = true ] && return 0 + sleep 3 + done + return 1 +} read -r -a WORKERS <<<"$(model_workers)" [ "${#WORKERS[@]}" -ge 2 ] || { echo "[case-103] needs two workers; NOTHING WAS VERIFIED"; exit 2; } @@ -141,6 +171,9 @@ kubectl get csidriver model.csi.gpustack.ai >/dev/null 2>&1 \ [ -n "$IT" ] || { echo "[case-103] no InstanceType; run case-1 first" >&2; exit 2; } for ns in "$NS" "$MDNS"; do kubectl create namespace "$ns" --dry-run=client -o yaml | kubectl apply -f - >/dev/null; done +# One docker gate for the whole case: the tmpfs and the kubelet both need it. +if docker exec "$W2" true >/dev/null 2>&1; then DOCKER_W2=1; else DOCKER_W2=0; fi + REPOS="$(python3 -c " import json mib = 1 << 20 @@ -155,6 +188,8 @@ print(json.dumps({ 'e2e/instance': {'files': {'config.json': {'size': 400}}}, 'e2e/md': {'files': {'config.json': {'size': 500}, 'w.bin': {'size': 4 * mib, 'lfs': True}}}, 'e2e/drv': {'files': {'config.json': {'size': 600}}}, + 'e2e/lifecycle-model': {'files': {'config.json': {'size': 300}, + 'model.safetensors': {'size': mib, 'lfs': True}}}, }))")" HUB_URL="$(mh_deploy "$NS" "${P}-hub" "$REPOS")" setting_set model-artifact-huggingface-endpoint "$HUB_URL" @@ -167,7 +202,7 @@ echo "[case-103] hub at ${HUB_URL}; workers ${W1}, ${W2}; InstanceType ${IT}" # released, a first, c held. The grace starts at each release, so the rows that follow run while it # passes, all of them on the first worker. GC=0 D_a="" D_b="" D_c="" -if docker exec "$W2" mount -t tmpfs -o size=100m tmpfs "$CACHE_ROOT" >/dev/null 2>&1; then +if [ "$DOCKER_W2" = 1 ] && docker exec "$W2" mount -t tmpfs -o size=100m tmpfs "$CACHE_ROOT" >/dev/null 2>&1; then TMPFS_NODE="$W2" kubectl -n "$SYSTEM_NS" delete pod "$(plugin_pod "$W2")" --wait=true >/dev/null total="" @@ -431,6 +466,54 @@ else fi kubectl -n "$MDNS" delete modeldeployments.worker.gpustack.ai "${P}-drv" --wait=false >/dev/null +# ---------------------------------------------------------------- the node's own entries +# The first worker's record of its content is rewritten from the node's disk after the plugin Pod on +# it is replaced, and again after the record itself is deleted and the worker creates it anew. It +# runs here, while the collection grace below runs out, so the two share their wall clock. +LIFE_ART="${P}-life" +LIFE_POD="${P}-life-pod" +artifact "$NS" "$LIFE_ART" e2e/lifecycle-model "" 103 +LIFE_DIGEST="$(mount_node "$NS" "$LIFE_ART" "$W1" "$LIFE_POD")" +LIFE_OK=1 +if [ -z "$LIFE_DIGEST" ] || ! pod_ready "$NS" "$LIFE_POD" 300; then + record FAIL "the consumer runs on ${W1}" "$(pod_mount_events "$NS" "$LIFE_POD" | tail -1)" + LIFE_OK=0 +fi +if [ "$LIFE_OK" = 1 ]; then + BEFORE="" + for _ in $(seq 1 20); do + BEFORE="$(entries "$W1")" + echo "$BEFORE" | grep -q "^${LIFE_DIGEST} Ready [0-9]* True$" && break + sleep 3 + done + echo "[case-103] ${W1} holds:" + echo "$BEFORE" + # The plugin Pod on the node is replaced: the new one reads the node's disk again. + kubectl -n "$SYSTEM_NS" delete pod "$(plugin_pod "$W1")" --wait=true --timeout=120s >/dev/null 2>&1 + if plugin_ready "$W1" 180 && wait_entries "$W1" "$BEFORE" 120; then + record PASS "after the plugin restarts the node's entries are rewritten as before" "$W1" + else + record FAIL "the plugin rewrites the same entries after a restart" "${W1}: $(entries "$W1" | tr '\n' ';')" + fi + # The record itself goes: the worker creates it again, and the plugin writes the same entries. + LIFE_OLD_UID="$(kubectl get nodemodelstores.worker.gpustack.ai "$W1" -o jsonpath='{.metadata.uid}')" + kubectl delete nodemodelstores.v1alpha1.worker.gpustack.ai "$W1" --wait=true >/dev/null 2>&1 + LIFE_NEW_UID="" + for _ in $(seq 1 40); do + LIFE_NEW_UID="$(kubectl get nodemodelstores.worker.gpustack.ai "$W1" -o jsonpath='{.metadata.uid}' 2>/dev/null)" + [ -n "$LIFE_NEW_UID" ] && [ "$LIFE_NEW_UID" != "$LIFE_OLD_UID" ] && break + sleep 3 + done + if [ -n "$LIFE_NEW_UID" ] && [ "$LIFE_NEW_UID" != "$LIFE_OLD_UID" ] && wait_entries "$W1" "$BEFORE" 420; then + record PASS "a deleted object comes back and the plugin rewrites the same entries from disk" "${W1}: ${LIFE_OLD_UID:0:8} -> ${LIFE_NEW_UID:0:8}" + else + record FAIL "the object is recreated with the node's entries" "${W1}: uid ${LIFE_NEW_UID:-}, $(entries "$W1" | tr '\n' ';')" + fi +else + record FAIL "the plugin rewrites the same entries after a restart" "${W1}: the consumer Pod did not mount" + record FAIL "the object is recreated with the node's entries" "${W1}: the consumer Pod did not mount" +fi + # ---------------------------------------------------------------- collection, part two: collect if [ "$GC" = 1 ]; then left=$(( RELEASED + 660 - $(date +%s) )) @@ -460,6 +543,49 @@ if [ "$GC" = 1 ]; then fi fi +# The tmpfs goes before the node does: the second worker's own cache root is back, and its Node +# object is deleted against that root. +if [ -n "$TMPFS_NODE" ]; then + docker exec "$TMPFS_NODE" umount -l "$CACHE_ROOT" >/dev/null 2>&1 +fi + +# ---------------------------------------------------------------- the Node goes, its record with it +# The second worker's Node object is deleted while its kubelet is stopped, so the node does not +# register itself again before the garbage collector reaches the record that belongs to it. +if [ "$DOCKER_W2" = 1 ]; then + W2_OLD_UID="$(kubectl get nodemodelstores.worker.gpustack.ai "$W2" -o jsonpath='{.metadata.uid}')" + DELETED_NODE="$W2" + # kubelet stops first: a running kubelet registers its Node again within seconds, and the worker + # then adopts the record for the new Node before the garbage collector reaches it. + docker exec "$W2" systemctl stop kubelet >/dev/null 2>&1 + kubectl delete node "$W2" --wait=true >/dev/null 2>&1 + STORE_GONE="" + for _ in $(seq 1 40); do + kubectl get nodemodelstores.worker.gpustack.ai "$W2" >/dev/null 2>&1 || { STORE_GONE=1; break; } + sleep 3 + done + if [ -n "$W2_OLD_UID" ] && [ -n "$STORE_GONE" ]; then + record PASS "deleting the Node collects its NodeModelStore" "$W2" + else + record FAIL "the NodeModelStore goes with its Node" "${W2}: uid ${W2_OLD_UID:-} still there" + fi + docker exec "$W2" systemctl start kubelet >/dev/null 2>&1 + W2_NEW_UID="" + for _ in $(seq 1 80); do + W2_NEW_UID="$(kubectl get nodemodelstores.worker.gpustack.ai "$W2" -o jsonpath='{.metadata.uid}' 2>/dev/null)" + [ -n "$W2_NEW_UID" ] && break + sleep 3 + done + if [ -n "$W2_NEW_UID" ] && [ "$W2_NEW_UID" != "$W2_OLD_UID" ]; then + record PASS "the node registered again gets a new NodeModelStore" "${W2}: ${W2_NEW_UID:0:8}" + DELETED_NODE="" + else + record FAIL "the node registered again gets its object back" "${W2}: $(kubectl get node "$W2" -o name 2>&1)" + fi +else + record SKIP "the NodeModelStore goes with its Node" "no docker access to the node ${W2}" +fi + print_rows [ "$FAILS" -eq 0 ] || { echo "[case-103] ${FAILS} check(s) FAILED"; exit 1; } echo "[case-103] PASS" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-104.sh b/.agents/skills/gpustack-operator-e2e/cases/case-104.sh index d0fb770fe..2ec0a7e0e 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/case-104.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-104.sh @@ -36,8 +36,8 @@ # in the node's cache; the collector removes it once it is unreferenced and needed. set -uo pipefail -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" CASES_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null . "${CASES_DIR}/_rows-lib.sh" @@ -56,10 +56,6 @@ P=c104 SERVED=e2e/c104 LABEL="e2e.gpustack.ai/case=104" -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } - cleanup() { echo echo "[case-104] cleanup" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-105.sh b/.agents/skills/gpustack-operator-e2e/cases/case-105.sh deleted file mode 100644 index 53fa34c00..000000000 --- a/.agents/skills/gpustack-operator-e2e/cases/case-105.sh +++ /dev/null @@ -1,342 +0,0 @@ -#!/usr/bin/env bash -# -# CASE 105 — An Instance's persistent claims place it where the volume can attach: a bound claim -# runs on its PV's node every time, waits without quota while that node is full, and a -# claim nothing binds creates no Pod (MUTATING, self-recovering; AUTO-SKIPS without two workers) -# -# case-105.sh -# -# Goal: Prove on a real API server, scheduler, kubelet and Kueue that an Instance's -# spec.volume.persistent and spec.additionalVolumes[].persistent follow the claim -# placement rules: a bound claim's PV node affinity reaches the Pod and TAS places it on -# the PV's node on every start; while that node is full the Workload waits without -# reserving quota and admits itself once room appears; a claim provisioned on the first -# Pod's node gets no affinity and binds there; an unbound claim on an immediate class -# creates no Pod and says why; the escape Setting instance-persistent-volume-placement -# restores the render that knew only the claim names. -# Environment: A cluster with at least two schedulable workers (else SKIP: on one node TAS has no -# wrong node to pick, so NOTHING WOULD BE VERIFIED), a materialized scheduling chain (run -# case-1 first), an InstanceType whose pool covers the workers and, in , the pool's -# entrance LocalQueue. The WaitForFirstConsumer row needs a default StorageClass that -# binds on the first Pod's node with a provisioner (kind's local-path), else that row -# SKIPs. NO GPU: every Pod runs the pause image. -# Inputs: All real. Two hostPath PVs pinned to the last worker by node affinity and bound to -# their claims through spec.volumeName; a claim on an immediate class whose provisioner -# does not exist, so it stays Pending; a placeholder Pod assigned to the PV's node by -# spec.nodeName, outside any queue, that takes the node's free CPU. Override the type -# with E2E_MD_INSTANCE_TYPE and the image with E2E_MD_IMAGE. -# Expected: - a workspace claim: over three starts, every Pod carries the PV's affinity and runs on -# the PV's node; -# - an additional-volume claim: the same; -# - the PV's node full: the Pod is not scheduled, its Workload has QuotaReserved not True -# and no admission; once the placeholder goes, the Pod runs on the PV's node; -# - a WaitForFirstConsumer claim: the Pod carries no affinity, the claim binds and the Pod -# runs; -# - an unbound claim on an immediate class: no Pod, phase Starting, and the phase message -# names the claim as Pending; -# - the Setting off: that Instance's Pod is created, and a bound claim's Pod carries no -# affinity. -# Cleanup: Trap deletes every Instance and Pod carrying the case label, then the claims, the PVs -# and the StorageClass, and puts the Setting back to the value it found. -set -uo pipefail - -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" -# shellcheck source=/dev/null -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" - -NS="${1:?usage: case-105.sh }" -IT="${E2E_MD_INSTANCE_TYPE:-}" -IMAGE="${E2E_MD_IMAGE:-registry.k8s.io/pause:3.10}" -SYSTEM_NS="${E2E_SYSTEM_NS:-gpustack-system}" -P=c105 -LABEL="e2e.gpustack.ai/case=105" -SETTING=instance-persistent-volume-placement - -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } - -# The Setting is read through the worker's thirty-second cache, so every change waits it out. -ORIG_SETTING="$(kubectl -n "$SYSTEM_NS" get secret gpustack-settings -o jsonpath="{.data.${SETTING}}" 2>/dev/null)" -set_placement() { # true|false - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"${SETTING}\":\"$(printf '%s' "$1" | base64)\"}}" >/dev/null - sleep 35 -} -restore_setting() { - local v=null - [ -n "$ORIG_SETTING" ] && v="\"${ORIG_SETTING}\"" - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge -p "{\"data\":{\"${SETTING}\":${v}}}" >/dev/null 2>&1 -} - -cleanup() { - echo - echo "[case-105] cleanup" - kubectl -n "$NS" delete instances.worker.gpustack.ai,pods -l "$LABEL" --ignore-not-found --wait=false >/dev/null 2>&1 - for _ in $(seq 1 24); do - [ -z "$(kubectl -n "$NS" get pods -o name 2>/dev/null | command grep "/${P}-")" ] && break - sleep 5 - done - kubectl -n "$NS" delete pvc -l "$LABEL" --ignore-not-found --wait=false >/dev/null 2>&1 - kubectl delete pv "${P}-pv-ws" "${P}-pv-add" --ignore-not-found --wait=false >/dev/null 2>&1 - kubectl delete storageclass "${P}-static" "${P}-immediate" --ignore-not-found >/dev/null 2>&1 - restore_setting -} -trap cleanup EXIT - -if [ -z "$IT" ]; then - IT="$(kubectl get instancetypes.worker.gpustack.ai -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)" -fi -[ -n "$IT" ] || { echo "[case-105] no InstanceType; run case-1 first" >&2; exit 2; } -WORKERS="$(kubectl get nodes -l '!node-role.kubernetes.io/control-plane' -o jsonpath='{range .items[*]}{.metadata.labels.kubernetes\.io/hostname}{"\n"}{end}' | sort)" -PV_NODE="$(printf '%s\n' "$WORKERS" | sed -n '$p')" -if [ "$(printf '%s\n' "$WORKERS" | command grep -c .)" -lt 2 ]; then - record SKIP "the placement rows" "fewer than two workers: TAS has no wrong node to pick; NOTHING WAS VERIFIED" - print_rows - exit 0 -fi -PV_NODE_NAME="$(kubectl get nodes -l "kubernetes.io/hostname=${PV_NODE}" -o jsonpath='{.items[0].metadata.name}')" - -inst() { # name volume-yaml [additional-volumes-yaml] - kubectl apply -f - >/dev/null </dev/null </dev/null </dev/null; } -affinity_of() { pod_field "$1" '{.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[?(@.key=="kubernetes.io/hostname")].values[0]}'; } - -wait_running() { # pod bound -> node, or "" - local node="" - for _ in $(seq 1 $(( $2 / 3 ))); do - if [ "$(pod_field "$1" '{.status.phase}')" = Running ]; then - node="$(pod_field "$1" '{.spec.nodeName}')" - break - fi - sleep 3 - done - printf '%s' "$node" -} - -wait_gone() { # pod - for _ in $(seq 1 40); do - kubectl -n "$NS" get pod "$1" >/dev/null 2>&1 || return 0 - sleep 3 - done - return 1 -} - -workload_of() { # pod -> quotaReserved|admitted - kubectl -n "$NS" get workloads.kueue.x-k8s.io -o json 2>/dev/null | python3 -c ' -import json,sys -pod=sys.argv[1] -for w in json.load(sys.stdin)["items"]: - if any(o.get("kind")=="Pod" and o.get("name")==pod for o in w["metadata"].get("ownerReferences",[])): - c={x["type"]:x["status"] for x in w.get("status",{}).get("conditions",[])} - print("%s|%s" % (c.get("QuotaReserved",""), "admission" if w.get("status",{}).get("admission") else "none")) - break -' "$1" -} - -placed() { # check pod - local node aff - node="$(wait_running "$2" 120)" - aff="$(affinity_of "$2")" - if [ "$node" = "$PV_NODE_NAME" ] && [ "$aff" = "$PV_NODE" ]; then - record PASS "$1" "node=${node} affinity=${aff}" - else - record FAIL "$1" "node=${node:-} affinity=${aff:-} pv=${PV_NODE}" - fi -} - -echo "== fixtures: two PVs pinned to ${PV_NODE}, bound claims, an immediate class ==" -kubectl apply -f - >/dev/null </dev/null)" = Bound ] && break - sleep 3 - done -done - -echo "== 1. a workspace claim runs on the PV's node on every start ==" -inst "${P}-ws" "{persistent: {name: ${P}-ws}}" -for round in 1 2 3; do - placed "start ${round}: the workspace claim's Pod carries the PV's affinity and runs there" "${P}-ws" - [ "$round" = 3 ] && break - kubectl -n "$NS" patch instances.worker.gpustack.ai "${P}-ws" --type=merge -p '{"spec":{"stop":true}}' >/dev/null - wait_gone "${P}-ws" || record FAIL "start ${round}: the Pod goes on stop" "still present after 120s" - # Admission starts only a Stopped Instance, which the phase says a pass after the Pod is gone. - for _ in $(seq 1 20); do - [ "$(kubectl -n "$NS" get instances.worker.gpustack.ai "${P}-ws" -o jsonpath='{.status.phase}')" = Stopped ] && break - sleep 3 - done - kubectl -n "$NS" patch instances.worker.gpustack.ai "${P}-ws" --type=merge -p '{"spec":{"stop":false}}' >/dev/null -done -REQ="$(pod_field "${P}-ws" '{.spec.containers[0].resources.requests.cpu}')" - -echo "== 2. an additional-volume claim runs on the PV's node ==" -inst "${P}-add" "{ephemeral: {capacity: 1Gi}}" "[{mountPath: /mnt/data, persistent: {name: ${P}-add}}]" -placed "the additional claim's Pod carries the PV's affinity and runs there" "${P}-add" - -echo "== 3. with the PV's node full, the Workload waits without quota ==" -kubectl -n "$NS" delete instances.worker.gpustack.ai "${P}-ws" "${P}-add" --wait=false >/dev/null 2>&1 -wait_gone "${P}-ws" -wait_gone "${P}-add" -FREE="$(kubectl get pods -A --field-selector "spec.nodeName=${PV_NODE_NAME}" -o json | python3 -c ' -import json,sys -def milli(q): - q=str(q) - return int(q[:-1]) if q.endswith("m") else int(float(q)*1000) -alloc=milli(sys.argv[1]); used=0 -for p in json.load(sys.stdin)["items"]: - if p["status"].get("phase") in ("Succeeded","Failed"): continue - for c in p["spec"]["containers"]: - used+=milli(c.get("resources",{}).get("requests",{}).get("cpu","0")) -print(alloc-used) -' "$(kubectl get node "$PV_NODE_NAME" -o jsonpath='{.status.allocatable.cpu}')")" -REQ_M="$(printf '%s' "${REQ:-0}" | python3 -c 'import sys; q=sys.stdin.read().strip() or "0"; print(int(q[:-1]) if q.endswith("m") else int(float(q)*1000))')" -FILL=$(( FREE - REQ_M / 2 )) -if [ "$REQ_M" -le 0 ] || [ "$FILL" -le 0 ]; then - record FAIL "the PV's node can be filled" "free=${FREE}m instance request=${REQ:-}" -else - kubectl apply -f - >/dev/null </dev/null - inst "${P}-full" "{persistent: {name: ${P}-ws}}" - sleep 45 - node="$(pod_field "${P}-full" '{.spec.nodeName}')" - wl="$(workload_of "${P}-full")" - if [ -n "$(pod_field "${P}-full" '{.metadata.uid}')" ] && [ -z "$node" ] && [ "${wl%%|*}" != True ] && [ "${wl#*|}" = none ]; then - record PASS "the full node's Workload waits without reserving quota" "fill=${FILL}m workload=${wl}" - else - record FAIL "the full node's Workload waits without reserving quota" "node=${node:-} workload=${wl:-}" - fi - kubectl -n "$NS" delete pod "${P}-fill" --wait=false >/dev/null 2>&1 - placed "once room appears the Pod runs on the PV's node" "${P}-full" -fi - -echo "== 4. a WaitForFirstConsumer claim binds where its first Pod lands ==" -default_class="$(kubectl get storageclass -o jsonpath='{range .items[?(@.metadata.annotations.storageclass\.kubernetes\.io/is-default-class=="true")]}{.metadata.name}|{.volumeBindingMode}|{.provisioner}{"\n"}{end}' | head -1)" -case "$default_class" in - *'|WaitForFirstConsumer|kubernetes.io/no-provisioner'|'') record SKIP "a WaitForFirstConsumer claim binds on the first Pod's node" "no default class provisioning on the first Pod's node: '${default_class}'" ;; - *'|WaitForFirstConsumer|'*) - pvc "${P}-wffc" "" - inst "${P}-wffc" "{persistent: {name: ${P}-wffc}}" - node="$(wait_running "${P}-wffc" 180)" - aff="$(affinity_of "${P}-wffc")" - phase="$(kubectl -n "$NS" get pvc "${P}-wffc" -o jsonpath='{.status.phase}' 2>/dev/null)" - if [ -n "$node" ] && [ -z "$aff" ] && [ "$phase" = Bound ]; then - record PASS "a WaitForFirstConsumer claim binds on the first Pod's node" "node=${node} claim=${phase}" - else - record FAIL "a WaitForFirstConsumer claim binds on the first Pod's node" "node=${node:-} affinity=${aff:-} claim=${phase:-}" - fi ;; - *) record SKIP "a WaitForFirstConsumer claim binds on the first Pod's node" "the default class binds immediately: '${default_class}'" ;; -esac - -echo "== 5. an unbound claim on an immediate class creates no Pod ==" -pvc "${P}-imm" "${P}-immediate" -inst "${P}-imm" "{persistent: {name: ${P}-imm}}" -sleep 40 -phase="$(kubectl -n "$NS" get instances.worker.gpustack.ai "${P}-imm" -o jsonpath='{.status.phase}')" -msg="$(kubectl -n "$NS" get instances.worker.gpustack.ai "${P}-imm" -o jsonpath='{.status.phaseMessage}')" -if ! kubectl -n "$NS" get pod "${P}-imm" >/dev/null 2>&1 && [ "$phase" = Starting ] && - printf '%s' "$msg" | command grep -qF "PersistentVolumeClaim \"${P}-imm\" is Pending"; then - record PASS "an unbound immediate claim creates no Pod and says why" "${phase}: ${msg}" -else - record FAIL "an unbound immediate claim creates no Pod and says why" "pod=$(pod_field "${P}-imm" '{.metadata.name}') phase=${phase:-} message=${msg:-}" -fi - -echo "== 6. the escape Setting restores the render from claim names alone ==" -set_placement false -pod="" -for _ in $(seq 1 20); do - pod="$(pod_field "${P}-imm" '{.metadata.name}')" - [ -n "$pod" ] && break - sleep 3 -done -if [ -n "$pod" ] && [ -z "$(affinity_of "${P}-imm")" ]; then - record PASS "with the Setting off the waiting Instance's Pod is created" "pod=${pod}" -else - record FAIL "with the Setting off the waiting Instance's Pod is created" "pod=${pod:-} within 60s" -fi -inst "${P}-off" "{ephemeral: {capacity: 1Gi}}" "[{mountPath: /mnt/data, persistent: {name: ${P}-add}}]" -pod="" -for _ in $(seq 1 20); do - pod="$(pod_field "${P}-off" '{.metadata.name}')" - [ -n "$pod" ] && break - sleep 3 -done -aff="$(affinity_of "${P}-off")" -if [ -n "$pod" ] && [ -z "$aff" ]; then - record PASS "with the Setting off a bound claim's Pod carries no affinity" "pod=${pod}" -else - record FAIL "with the Setting off a bound claim's Pod carries no affinity" "pod=${pod:-} affinity=${aff:-}" -fi -restore_setting - -print_rows -[ "$FAILS" -eq 0 ] || { echo "[case-105] ${FAILS} check(s) FAILED"; exit 1; } diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-106.sh b/.agents/skills/gpustack-operator-e2e/cases/case-106.sh deleted file mode 100755 index 2b35c715c..000000000 --- a/.agents/skills/gpustack-operator-e2e/cases/case-106.sh +++ /dev/null @@ -1,226 +0,0 @@ -#!/usr/bin/env bash -# -# CASE 106 — With instance-type-derived-from-node off, a P/D deployment on one administrator-authored -# InstanceType is still admitted as a set (MUTATING, self-cleaning) -# -# case-106.sh -# -# Goal: In the mode where the administrator authors every InstanceType, the queues used to -# reference no AdmissionCheck at all. Every replica of every role is its own Workload, so -# a short pool admitted the role that fit and left the other waiting: a prefiller serving -# nothing while its decoder never arrives. The joint-admission check is now referenced -# from every operator-owned queue whatever the setting says. This case proves it on a -# queue created while the setting is off, then reruns CASE 50's shortage on that queue. -# -# IT RUNS CASE 50 RATHER THAN COPYING IT. Case 50 already measures a short pool the only -# way that holds on any cluster -- a probe asks Kueue how many replicas the queue takes, -# and a filler leaves room for exactly one -- and it names the joint check on the role -# that fit. What this case adds is the mode and a queue born in it. -# -# Environment: Any cluster with a materialized scheduling chain (run case-1 first) and an InstanceType -# whose general group, OS and arch select a CPU pool. NO GPU is needed. The namespace must -# receive the new type's entrance LocalQueue, which the operator creates in every -# namespace. EXITS 2 (input required) when the cluster has no usable InstanceType. Needs -# case-50.sh beside this file. -# -# Inputs: Real objects only. The instance-type-derived-from-node key of the gpustack-settings -# Secret in E2E_SYSTEM_NS (default gpustack-system), set to false for the run and put back -# to the value it had; one CPU-only InstanceType `case106-authored` on the working type's -# own CPU pool, created while the setting is off so its ClusterQueue is first filled in -# that mode; and CASE 50's three ModelDeployments on that type. Override the working -# type with E2E_MD_INSTANCE_TYPE. -# -# Expected: - the gpustack-model-deployment-joint AdmissionCheck is Active (precondition); -# - the authored type's ClusterQueue references it while the setting is off; -# - every CASE 50 row passes on the authored type -- in particular, the role that fits a -# short pool stays gated, and its Workload's joint check is Pending naming the other -# role, and releasing the pool admits both. -# Against an operator without this change, the queue row and CASE 50's two gating rows -# FAIL: the role that fit is admitted on its own, which is the defect this case pins. -# -# Cleanup: A trap deletes the InstanceType and restores the Setting. CASE 50 cleans up its own -# deployments. Idempotent, runs on pass AND fail, safe to re-run. -# -set -uo pipefail - -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" -CASES_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=/dev/null -. "${CASES_DIR}/_rows-lib.sh" - -NS="${1:?usage: case-106.sh }" -IT="${E2E_MD_INSTANCE_TYPE:-}" -SYSTEM_NS="${E2E_SYSTEM_NS:-gpustack-system}" -SETTING=instance-type-derived-from-node -AUTHORED=case106-authored -JOINT=gpustack-model-deployment-joint -SETTLE="${E2E_MD_SETTLE:-120}" - -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } - -wait_for() { # wait_for - local deadline=$((SECONDS + $1)) - shift - while [ $SECONDS -lt "$deadline" ]; do - "$@" && return 0 - sleep 5 - done - - return 1 -} - -# The first InstanceType a deployment can name: not terminating, not Inactive, and not this case's own -# type left over from a previous run. See case-68 for why the first one listed is not good enough. -usable_instance_type() { - kubectl get instancetypes.worker.gpustack.ai \ - -o jsonpath='{range .items[*]}{.metadata.name}|{.metadata.deletionTimestamp}|{.spec.inactive}{"\n"}{end}' \ - 2>/dev/null \ - | while IFS='|' read -r name deleting inactive; do - [ -n "$name" ] || continue - [ "$name" = "$AUTHORED" ] && continue - [ -z "$deleting" ] || continue - [ "$inactive" = true ] && continue - echo "$name" - break - done -} - -if [ -z "$IT" ]; then - IT="$(usable_instance_type)" -fi -if [ -z "$IT" ]; then - echo "[case-106] no usable InstanceType in the cluster; run case-1 first" >&2 - exit 2 -fi -read -r GROUP OS ARCH <<<"$(kubectl get instancetypes.worker.gpustack.ai "$IT" \ - -o jsonpath='{.spec.generalGroup} {.spec.os} {.spec.arch}' 2>/dev/null)" -if [ -z "$GROUP" ] || [ -z "$OS" ] || [ -z "$ARCH" ]; then - echo "[case-106] InstanceType ${IT} names no general group, OS and arch to author a type on" >&2 - exit 2 -fi - -# The Setting is read through the worker's thirty-second cache, so every change waits it out. -ORIG_SETTING="$(kubectl -n "$SYSTEM_NS" get secret gpustack-settings -o jsonpath="{.data.${SETTING}}" 2>/dev/null)" -set_derived() { # true|false - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"${SETTING}\":\"$(printf '%s' "$1" | base64)\"}}" >/dev/null - sleep 35 -} -restore_setting() { - local v=null - [ -n "$ORIG_SETTING" ] && v="\"${ORIG_SETTING}\"" - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge -p "{\"data\":{\"${SETTING}\":${v}}}" >/dev/null 2>&1 -} - -cleanup() { - kubectl delete instancetypes.worker.gpustack.ai "$AUTHORED" --ignore-not-found --wait=false >/dev/null 2>&1 - restore_setting -} -trap cleanup EXIT - -# ------------------------------------------------------------------ precondition: the check is Active. -ACTIVE="$(kubectl get admissionchecks.kueue.x-k8s.io "$JOINT" \ - -o jsonpath='{.status.conditions[?(@.type=="Active")].status}' 2>/dev/null)" -if [ "$ACTIVE" != True ]; then - record FAIL "the joint check is Active" "${JOINT} Active=[${ACTIVE:-absent}]; a queue references only an Active check" - print_rows - exit 1 -fi -record PASS "the joint check is Active" "$JOINT" - -# ----------------------------------------------- a queue born while the administrator authors types. -# -# THE TYPE IS CREATED AFTER THE FLIP, NOT BEFORE. Flipping the setting re-reads nothing on an existing -# queue until its resource groups are refilled, so a queue filled before the flip says what the -# default mode wrote and nothing about this one. -set_derived false -kubectl delete instancetypes.worker.gpustack.ai "$AUTHORED" --ignore-not-found --wait=true >/dev/null 2>&1 -if ! kubectl apply -f - >/dev/null 2>&1 </dev/null)" ] -} -queue_references_joint() { - kubectl get clusterqueue "$AUTHORED" \ - -o jsonpath='{range .spec.admissionChecksStrategy.admissionChecks[*]}{.name}{"\n"}{end}' 2>/dev/null \ - | grep -qxF "$JOINT" -} -entrance_ready() { - local lq - lq="$(kubectl get instancetypes.worker.gpustack.ai "$AUTHORED" -o jsonpath='{.status.entrance}' 2>/dev/null)" - [ -n "$lq" ] && kubectl -n "$NS" get localqueue "$lq" >/dev/null 2>&1 -} - -if ! wait_for "$SETTLE" queue_filled; then - record FAIL "the authored type's queue is filled" \ - "ClusterQueue ${AUTHORED} carries no flavor after ${SETTLE}s, so nothing below can reserve quota" - print_rows - exit 1 -fi -record PASS "the authored type's queue is filled" "$(kubectl get clusterqueue "$AUTHORED" \ - -o jsonpath='{.spec.resourceGroups[*].flavors[*].name}' 2>/dev/null)" - -# THE QUEUE ROW IS WHAT DISCRIMINATES THIS MODE. A queue filled while the setting is off carried no -# AdmissionCheck at all before this change, and the check is referenced only once it is Active -- -# which the precondition above established -- so a missing reference here is the operator's. -if wait_for 30 queue_references_joint; then - record PASS "the authored type's queue references the joint check" "setting off, ${JOINT} referenced" -else - record FAIL "the authored type's queue references the joint check" \ - "references [$(kubectl get clusterqueue "$AUTHORED" \ - -o jsonpath='{.spec.admissionChecksStrategy.admissionChecks[*].name}' 2>/dev/null)] with the setting off" -fi - -if ! wait_for "$SETTLE" entrance_ready; then - record FAIL "the authored type's entrance LocalQueue reaches ${NS}" "no LocalQueue after ${SETTLE}s" - print_rows - exit 1 -fi - -# --------------------------------------------------------------------- CASE 50 on the authored type. -# -# A SKIP IN CASE 50 EXITS 0, and a skipped shortage is exactly the run in which nothing was gated. -# So its table is read as well as its exit code, and a skipped row makes this row a SKIP rather than -# a PASS: the claim was not measured. -echo "[case-106] running case-50 on ${AUTHORED} with ${SETTING}=false" -C50_OUT="$(mktemp)" -E2E_MD_INSTANCE_TYPE="$AUTHORED" bash "${CASES_DIR}/case-50.sh" "$NS" 2>&1 | tee "$C50_OUT" -C50_RC=${PIPESTATUS[0]} -C50_SKIPS="$(grep -c '^SKIP | ' "$C50_OUT" || true)" -rm -f "$C50_OUT" -if [ "$C50_RC" -ne 0 ]; then - record FAIL "case-50 passes on the authored type" "case-50 exited ${C50_RC}; see its table above" -elif [ "${C50_SKIPS:-0}" -gt 0 ]; then - record SKIP "case-50 passes on the authored type" \ - "case-50 skipped ${C50_SKIPS} row(s), so the shortage was not measured; see its table above" -else - record PASS "case-50 passes on the authored type" "every row of its table above passed" -fi - -echo -echo "== case-106: authored-mode P/D admitted as a set ==" -print_rows -[ "$FAILS" -eq 0 ] || { echo "[case-106] ${FAILS} check(s) FAILED"; exit 1; } -echo "[case-106] all checks passed" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-107.sh b/.agents/skills/gpustack-operator-e2e/cases/case-107.sh index 628bc780f..7087eb7bf 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/case-107.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-107.sh @@ -45,8 +45,8 @@ # and the deployments' namespace, and puts back the Settings it changed. set -uo pipefail -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" CASES_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null . "${CASES_DIR}/_rows-lib.sh" @@ -61,20 +61,10 @@ IMAGE="${E2E_MD_IMAGE:-registry.k8s.io/pause:3.10}" TRIALS="${E2E_C107_TRIALS:-20}" FULL_TRIALS="${E2E_C107_FULL_TRIALS:-10}" SEED="${E2E_C107_SEED:-$(date +%s)}" -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } SCRATCH="$(mktemp -d)" -KEYS="model-artifact-huggingface-endpoint model-artifact-delivery-mode" -mkdir -p "$SCRATCH/orig" -for k in $KEYS; do - kubectl -n "$SYSTEM_NS" get secret gpustack-settings -o jsonpath="{.data.$k}" >"$SCRATCH/orig/$k" 2>/dev/null -done -# restore KEY: the value KEY had before the case, or no value when it had none. -restore() { - if [ -s "$SCRATCH/orig/$1" ]; then setting_set "$1" "$(base64 -d <"$SCRATCH/orig/$1")"; else setting_unset "$1"; fi -} +# Remember what the store carried before the case touched it, so the cleanup puts every key back. +settings_save model-artifact-huggingface-endpoint model-artifact-delivery-mode cleanup() { echo @@ -87,7 +77,7 @@ cleanup() { kubectl -n "$NS" delete deploy,svc "${P}-hub" --ignore-not-found >/dev/null 2>&1 kubectl -n "$NS" delete configmap "${P}-hub-script" --ignore-not-found >/dev/null 2>&1 kubectl delete namespace "$MDNS" --ignore-not-found --wait=false >/dev/null 2>&1 - for k in $KEYS; do restore "$k"; done + settings_restore rm -rf "$SCRATCH" } trap cleanup EXIT diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-108.sh b/.agents/skills/gpustack-operator-e2e/cases/case-108.sh index 0afcdffbd..a79639493 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/case-108.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-108.sh @@ -1,43 +1,61 @@ #!/usr/bin/env bash # -# CASE 108 — Download progress: a node writes its progress on thresholds and nothing at rest, the -# artifact counts its nodes in 5% steps, and the v1 progress subresource answers live, -# to its own namespace only, naming no node (MUTATING, self-recovering) +# CASE 108 — One test hub, three node-delivery legs: an expected digest asserts the content, a +# prefetch warms a node under a store and a grant, and a slow download writes its progress +# on thresholds and nothing at rest, with the v1 views read-only to the worker +# (MUTATING, self-recovering; the peer leg AUTO-SKIPS on one worker or with +# E2E_OFFLINE=1) # # case-108.sh # -# Goal: Prove the progress contract end to end on a download slow enough to show it: the -# node's NodeModelStore carries downloadedBytes on 30-second, 5% thresholds and writes -# nothing once the content is published; the ModelArtifact's status.nodes moves in -# multiples of 5 and ends with the node counted ready; the v1 progress subresource -# reads the running download live, is refused to a subject without its RBAC rule and -# for another namespace, and names no node; the cluster-scoped NodeModelStore names no -# tenant; the v1 views write nothing the worker's identity would. -# Environment: A cluster installed from this chart with modelManager.enabled (the default), a -# schedulable worker, and the stock python image pullable. NO GPU and no InstanceType: -# the consumer is a bare Pod. -# Inputs: MOCKED: a test hub (_model-hub.py) in standing in for the Hugging Face Hub. The -# node's bandwidth Setting is lowered to 128 KiB/s so a 24 MiB file downloads for about -# three minutes. Real: the plugin, kubelet, the worker, the aggregated API server and -# RBAC, with two service accounts impersonated. -# Expected: - at least two writes carry an intermediate downloadedBytes; no two progress-only -# writes are closer than 30 s; at most 20 progress-only writes; -# - no write for one report interval (5 min) after the Pod runs; -# - the artifact's status.nodes.downloadingPercent is a multiple of 5 while downloading, -# and ends ready >= 1 with no percent; -# - two progress reads during the download: live 1 and non-decreasing downloadingBytes; -# - progress: allowed with get modelartifacts/progress in ; refused for -b's -# artifact and to a subject with only get modelartifacts; the allowed answer carries -# no node name; -# - no namespace, artifact, repository or Pod name of this case in any NodeModelStore; -# - an update of status through the v1 ModelArtifact leaves the stored status; the v1 -# ModelArtifact has no status subresource; the v1 NodeModelStore refuses an update. -# Cleanup: Trap stops the watch and deletes the Pods, artifacts, RBAC, the second namespace and -# the hub, and puts back the Settings it changed. +# Goal: Prove the node-delivery contract end to end against one controlled hub: the digest an +# artifact asserts is the identity the hub serves, a foreign anchor is refused naming +# both digests, a hub that cannot be reached hands the identity to the anchor, an +# anchored artifact is held under Engine delivery and served from a peer's published +# tree under Node delivery; a prefetch warms a node through a store and a grant and +# leaves the tree to the plugin's own grace; a download slow enough to show it writes +# progress on 30-second, 5% thresholds and nothing once the content is published, and +# the v1 progress subresource answers live, to its own namespace only, naming no node. +# Environment: A cluster installed from this chart with modelManager.enabled (the default), the stock +# python image pullable, and a schedulable worker (two for the peer leg). NO GPU: the +# consumer is a bare Pod, and an InstanceType is needed only by the engine-block row. +# The peer leg AUTO-SKIPS on one worker or with E2E_OFFLINE=1; with no worker at +# all, or with no model.csi.gpustack.ai CSIDriver, nothing is verified and the case +# exits 2. +# Inputs: MOCKED: one test hub (_model-hub.py) in standing in for the Hugging Face Hub, +# serving the four repositories the three legs resolve. Real: the plugin, kubelet, the +# worker, the aggregated API server and RBAC, with two service accounts impersonated. +# The node's bandwidth Setting is lowered to 128 KiB/s for the download leg alone. +# Expected: - the anchor leg: a twin resolves and publishes on the first worker; a matching anchor +# resolves with digestSource Hub and a commit; a foreign anchor stays refused with +# DigestMismatch naming both digests; with the hub unreachable the anchor becomes the +# identity (digestSource Expected, no revision) with the hub's request log flat; under +# Engine delivery both anchored artifacts are held with AnchorNeedsNodeDelivery and +# no Pod; on a second worker the Expected identity mounts from the peer, hub log flat; +# - the prefetch leg: the node's spec names the store and takes its watermark; the +# prefetch reaches Available with honest counts, its warm-up pod node-pinned and free +# of the queue-name label, the digest Ready on the node; a projection past the budget +# is refused at admission; pinning is refused without the grant's permission and +# lands once the grant allows it; deleting the prefetch removes its pod and its pin +# and leaves the tree to the grace; +# - the download leg: at least two writes carry an intermediate downloadedBytes, no two +# progress-only writes are closer than 30 s, there are at most 20 of them, and nothing +# is written for a report interval after the Pod ran; the artifact's status.nodes moves +# in 5% steps and ends with the node ready and no percent; two progress reads during +# the download are live and non-decreasing; progress is allowed with get +# modelartifacts/progress in the namespace and refused for another namespace and to a +# subject holding only get modelartifacts; the allowed answer names no node; no +# NodeModelStore names this case's namespace, artifact, repository or Pod; a status +# update through the v1 ModelArtifact leaves the stored status, the v1 ModelArtifact +# has no status subresource, the v1 NodeModelStore refuses an update, and both v1 +# kinds print their columns. +# Cleanup: Trap deletes the pods, artifacts, prefetches, store, binding, deployments, RBAC, the +# second namespace and the hub, and puts back the Settings, the node label and the +# namespace labels it changed. Idempotent, on pass AND fail. set -uo pipefail -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" CASES_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null . "${CASES_DIR}/_rows-lib.sh" @@ -47,20 +65,42 @@ CASES_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" NS="${1:?usage: case-108.sh }" NSB="${NS}-b" P=c108 +P110=c110 +P114=c114 +HUB="${P}-hub" LABEL="e2e.gpustack.ai/case=108" -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } +LABEL110="e2e.gpustack.ai/case=110" +LABEL114="e2e.gpustack.ai/case=114" +POOL_LABEL="e2e.gpustack.ai/warm-pool" +COMMIT_BOUND=120 +ANCHOR_BOUND=240 +POD_BOUND=300 +WRONG="sha256:$(printf 'f%.0s' $(seq 64))" SCRATCH="$(mktemp -d)" WATCH_PID="" -ORIG_ENDPOINT="$(setting_get model-artifact-huggingface-endpoint)" -ORIG_DELIVERY="$(setting_get model-artifact-delivery-mode)" -ORIG_BANDWIDTH="$(setting_get model-store-download-bandwidth)" +W1="" +NODE_WAS_LABELED="" +ORIG_PSS_ENFORCE="" +ORIG_PSS_VERSION="" READER="system:serviceaccount:${NS}:${P}-reader" PLAIN="system:serviceaccount:${NS}:${P}-plain" -restore() { # restore KEY VALUE - if [ -n "$2" ]; then setting_set "$1" "$2"; else setting_unset "$1"; fi +# Remember what the store carried before the case touched it, so the cleanup puts every key back. +settings_save model-artifact-huggingface-endpoint model-artifact-delivery-mode \ + model-store-download-bandwidth + +# restore_label KIND NAME KEY VALUE: put a label back, removing it when the object carried none. +restore_label() { + if [ -n "$4" ]; then + kubectl label "$1" "$2" "$3=$4" --overwrite >/dev/null 2>&1 + else + kubectl label "$1" "$2" "$3-" >/dev/null 2>&1 + fi +} +# ns_label NS KEY: the namespace's label value, empty when it carries none. +ns_label() { + kubectl get namespace "$1" -o json 2>/dev/null \ + | python3 -c 'import json,sys; print(json.load(sys.stdin).get("metadata",{}).get("labels",{}).get(sys.argv[1],""))' "$2" } cleanup() { echo @@ -68,15 +108,29 @@ cleanup() { [ -n "$WATCH_PID" ] && kill "$WATCH_PID" >/dev/null 2>&1 for ns in "$NS" "$NSB"; do kubectl -n "$ns" delete pods -l e2e.gpustack.ai/consumer=true --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 - kubectl -n "$ns" delete modelartifacts.worker.gpustack.ai -l "$LABEL" --ignore-not-found >/dev/null 2>&1 + for lbl in "$LABEL" "$LABEL110" "$LABEL114"; do + kubectl -n "$ns" delete modelartifacts.worker.gpustack.ai -l "$lbl" --ignore-not-found >/dev/null 2>&1 + done done + kubectl -n "$NS" delete pods "${P114}-seed" "${P114}-peer" --ignore-not-found --wait=true --timeout=180s >/dev/null 2>&1 + kubectl -n "$NS" delete modeldeployments.worker.gpustack.ai "${P114}-md-match" "${P114}-md-expected" --ignore-not-found >/dev/null 2>&1 + kubectl -n "$NS" delete modelprefetches.worker.gpustack.ai -l "$LABEL110" --ignore-not-found --wait=false >/dev/null 2>&1 + kubectl -n "$NS" delete modelprefetches.worker.gpustack.ai "${P110}-warm" "${P110}-pin" "${P110}-big" --ignore-not-found --wait=false >/dev/null 2>&1 + kubectl -n "$NS" delete modelstorebindings.worker.gpustack.ai -l "$LABEL110" --ignore-not-found --wait=false >/dev/null 2>&1 + kubectl delete modelstores.worker.gpustack.ai -l "$LABEL110" --ignore-not-found --wait=false >/dev/null 2>&1 + kubectl -n "$NS" delete pods -l worker.gpustack.ai/model-prefetch --ignore-not-found --wait=false >/dev/null 2>&1 kubectl -n "$NS" delete role,rolebinding,serviceaccount -l "$LABEL" --ignore-not-found >/dev/null 2>&1 kubectl -n "$NS" delete deploy,svc,configmap -l e2e.gpustack.ai/model-hub=true --ignore-not-found >/dev/null 2>&1 - kubectl -n "$NS" delete configmap "${P}-hub-script" --ignore-not-found >/dev/null 2>&1 + kubectl -n "$NS" delete deploy "${HUB}" --ignore-not-found >/dev/null 2>&1 + kubectl -n "$NS" delete svc "${HUB}" --ignore-not-found >/dev/null 2>&1 + kubectl -n "$NS" delete configmap "${HUB}-script" --ignore-not-found >/dev/null 2>&1 kubectl delete namespace "$NSB" --ignore-not-found --wait=false >/dev/null 2>&1 - restore model-artifact-huggingface-endpoint "$ORIG_ENDPOINT" - restore model-artifact-delivery-mode "$ORIG_DELIVERY" - restore model-store-download-bandwidth "$ORIG_BANDWIDTH" + if [ "$NODE_WAS_LABELED" = 1 ]; then + kubectl label node "$W1" "$POOL_LABEL-" >/dev/null 2>&1 + fi + restore_label namespace "$NS" pod-security.kubernetes.io/enforce "$ORIG_PSS_ENFORCE" + restore_label namespace "$NS" pod-security.kubernetes.io/enforce-version "$ORIG_PSS_VERSION" + settings_restore rm -rf "$SCRATCH" } trap cleanup EXIT @@ -89,9 +143,40 @@ nodes_of() { } # field JSON KEY: a top-level number of a progress answer, or "". field() { python3 -c "import json,sys; print(json.loads(sys.argv[1]).get(sys.argv[2], ''))" "$1" "$2" 2>/dev/null; } +# has_columns HEADER COLUMN...: whether the header line names every column, in order. +has_columns() { + local header="$1" col rest + shift + rest=" $(echo "$header" | tr -s ' ') " + for col in "$@"; do + case "$rest" in *" $col "*) rest="${rest#* "$col" }"; rest=" $rest" ;; *) return 1 ;; esac + done +} +# wait_prefetch_available NS NAME [SECONDS] [POD_CAPTURE]: until the prefetch's Available condition is +# True, snapshotting its warm-up pod into POD_CAPTURE at first sight — the pod is transient, deleted +# once its work is done, so the first sighting is the evidence. +wait_prefetch_available() { + local ns="$1" name="$2" seconds="${3:-300}" capture="${4:-}" + for _ in $(seq 1 "$(( seconds / 3 ))"); do + if [ -n "$capture" ] && ! [ -s "$capture" ]; then + if kubectl -n "$ns" get pod -l "worker.gpustack.ai/model-prefetch=$name" -o json > "$capture.tmp" 2>/dev/null; then + python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); sys.exit(0 if d.get("items") else 1)' "$capture.tmp" \ + && mv "$capture.tmp" "$capture" + fi + fi + [ "$(kubectl -n "$ns" get modelprefetches.worker.gpustack.ai "$name" \ + -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>/dev/null)" = True ] && return 0 + sleep 3 + done + return 1 +} +if [ "$NS" = "$SYSTEM_NS" ]; then + echo "[case-108] refusing the system namespace ${SYSTEM_NS}; usage: case-108.sh " >&2 + exit 2 +fi read -r -a WORKERS <<<"$(model_workers)" -[ "${#WORKERS[@]}" -ge 1 ] || { echo "[case-108] needs a worker; NOTHING WAS VERIFIED"; exit 2; } +[ "${#WORKERS[@]}" -ge 1 ] || { echo "[case-108] needs a worker, found none; NOTHING WAS VERIFIED"; exit 2; } W1="${WORKERS[0]}" if ! kubectl get csidriver model.csi.gpustack.ai >/dev/null 2>&1; then echo "[case-108] the model-manager plugin is not installed; NOTHING WAS VERIFIED" @@ -101,115 +186,447 @@ fi for ns in "$NS" "$NSB"; do kubectl create namespace "$ns" --dry-run=client -o yaml | kubectl apply -f - >/dev/null done -kubectl apply -f - >/dev/null </dev/null +if kubectl label node "$W1" "$POOL_LABEL=true" >/dev/null 2>&1; then + NODE_WAS_LABELED=1 +else + echo "[case-108] could not label ${W1} as the warm pool; NOTHING WAS VERIFIED" + exit 2 +fi REPOS="$(python3 -c " import json mib = 1 << 20 -print(json.dumps({'e2e/progress-model': {'files': {'config.json': {'size': 300}, - 'model.safetensors': {'size': 24 * mib, 'lfs': True}}}}))")" -HUB_URL="$(mh_deploy "$NS" "${P}-hub" "$REPOS")" -setting_set model-artifact-huggingface-endpoint "$HUB_URL" -setting_set model-artifact-delivery-mode Node -setting_set model-store-download-bandwidth 128Ki +print(json.dumps({ + 'e2e/anchored': {'files': {'config.json': {'size': 300}, 'tokenizer.json': {'size': 3000}}}, + 'e2e/small': {'files': {'config.json': {'size': 300}, 'tokenizer.json': {'size': 3000}, + 'model.safetensors': {'size': 5 * mib, 'lfs': True}}}, + 'e2e/big': {'files': {'config.json': {'size': 300}, 'model.safetensors': {'size': 50 * mib, 'lfs': True}}}, + 'e2e/progress-model': {'files': {'config.json': {'size': 300}, + 'model.safetensors': {'size': 24 * mib, 'lfs': True}}}, +}))")" +HUB_URL="$(mh_deploy "$NS" "$HUB" "$REPOS")" +setting_set model-artifact-huggingface-endpoint "$HUB_URL" >/dev/null +setting_set model-artifact-delivery-mode Node >/dev/null settings_settle echo "[case-108] hub at ${HUB_URL}; worker ${W1}" +# ---------------------------------------------------------------- the anchor: an expected digest +echo "== the anchor: an expected digest asserts the content ==" +artifact "$NS" "${P114}-twin" e2e/anchored "" 114 +DIGEST="$(wait_resolved "$NS" "${P114}-twin" "$COMMIT_BOUND")" +ANCHOR_OK=1 +if [ -n "$DIGEST" ]; then + record PASS "twin resolved" "digest ${DIGEST:0:19}…" +else + record FAIL "twin resolved" "no digest within ${COMMIT_BOUND}s: $(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P114}-twin" -o jsonpath='{.status.conditions[?(@.type=="Resolved")].message}' 2>/dev/null)" + ANCHOR_OK=0 +fi + +if [ "$ANCHOR_OK" = 1 ]; then + TWIN_UID=$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P114}-twin" -o jsonpath='{.metadata.uid}') + consumer "$NS" "${P114}-seed" "${WORKERS[0]}" "${P114}-twin" "$TWIN_UID" "$DIGEST" + if pod_ready "$NS" "${P114}-seed" "$POD_BOUND"; then + record PASS "seed node" "the first worker holds the published tree" + else + record FAIL "seed node" "not Running within ${POD_BOUND}s: $(pod_mount_events "$NS" "${P114}-seed" | head -1)" + fi + + echo "== a matching anchor resolves as a hub identity ==" + artifact "$NS" "${P114}-match" e2e/anchored "" 114 " + expectedDigest: $DIGEST" + MATCH_DIGEST="$(wait_resolved "$NS" "${P114}-match" "$COMMIT_BOUND")" + MATCH_SRC="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P114}-match" -o jsonpath='{.status.resolved.digestSource}' 2>/dev/null)" + MATCH_REV="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P114}-match" -o jsonpath='{.status.resolved.revision}' 2>/dev/null)" + if [ "$MATCH_DIGEST" = "$DIGEST" ] && [ "$MATCH_SRC" = Hub ] && [ -n "$MATCH_REV" ]; then + record PASS "match resolves" "digestSource Hub at commit ${MATCH_REV:0:12}…" + else + record FAIL "match resolves" "digest '${MATCH_DIGEST:0:19}…' source '$MATCH_SRC' revision '$MATCH_REV'" + fi + + echo "== a foreign anchor is refused with both digests ==" + artifact "$NS" "${P114}-mismatch" e2e/anchored "" 114 " + expectedDigest: $WRONG" + MISMATCH_MSG="" + for _ in $(seq 1 45); do + R="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P114}-mismatch" \ + -o jsonpath='{.status.conditions[?(@.type=="Resolved")].reason}' 2>/dev/null)" + [ "$R" = DigestMismatch ] && break + sleep 2 + done + MISMATCH_MSG="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P114}-mismatch" -o jsonpath='{.status.conditions[?(@.type=="Resolved")].message}' 2>/dev/null)" + if [ "$R" = DigestMismatch ] && printf '%s' "$MISMATCH_MSG" | grep -qF "$WRONG" && printf '%s' "$MISMATCH_MSG" | grep -qF "$DIGEST"; then + record PASS "mismatch refused" "DigestMismatch names the anchor and the hub's digest" + else + record FAIL "mismatch refused" "reason '$R', message '${MISMATCH_MSG:0:160}'" + fi + + echo "== a twice-unreachable hub hands the identity to the anchor ==" + LOG_BEFORE="$(mh_log "$NS" "$HUB" | wc -l | tr -d ' ')" + setting_set model-artifact-huggingface-endpoint "http://${P114}-absent.${NS}.svc:8080" >/dev/null + settings_settle + artifact "$NS" "${P114}-expected" e2e/anchored "" 114 " + expectedDigest: $DIGEST" + EXPECTED_DIGEST="$(wait_resolved "$NS" "${P114}-expected" "$ANCHOR_BOUND")" + EXPECTED_SRC="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P114}-expected" -o jsonpath='{.status.resolved.digestSource}' 2>/dev/null)" + EXPECTED_REV="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P114}-expected" -o jsonpath='{.status.resolved.revision}' 2>/dev/null)" + LOG_DURING="$(mh_log "$NS" "$HUB" | wc -l | tr -d ' ')" + if [ "$EXPECTED_DIGEST" = "$DIGEST" ] && [ "$EXPECTED_SRC" = Expected ] && [ -z "$EXPECTED_REV" ] && [ "$LOG_DURING" = "$LOG_BEFORE" ]; then + record PASS "anchored identity" "digestSource Expected, no revision, hub log flat (${LOG_BEFORE} → ${LOG_DURING})" + else + record FAIL "anchored identity" "digest '${EXPECTED_DIGEST:0:19}…' source '$EXPECTED_SRC' revision '$EXPECTED_REV' log ${LOG_BEFORE} → ${LOG_DURING}" + fi + setting_set model-artifact-huggingface-endpoint "$HUB_URL" >/dev/null + settings_settle + + echo "== Engine delivery is refused for anchored artifacts ==" + IT="$(kubectl get instancetypes.worker.gpustack.ai -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)" + if [ -z "$IT" ]; then + record SKIP "engine block" "no InstanceType exists to reference; the render is unit-covered" + else + setting_set model-artifact-delivery-mode Engine >/dev/null + settings_settle + for pair in "${P114}-match:${P114}-md-match:cannot anchor-verify" "${P114}-expected:${P114}-md-expected:no resolved commit"; do + ART="${pair%%:*}"; REST="${pair#*:}"; MD="${REST%%:*}"; WANT="${REST#*:}" + kubectl apply -f - >/dev/null </dev/null)" + [ "$REASON" = AnchorNeedsNodeDelivery ] && break + sleep 2 + done + MSG="$(kubectl -n "$NS" get modeldeployments.worker.gpustack.ai "$MD" -o jsonpath='{.status.conditions[?(@.type=="WeightsReady")].message}' 2>/dev/null)" + PODS="$(kubectl -n "$NS" get pods -l "worker.gpustack.ai/model-deployment=$MD" --no-headers 2>/dev/null | wc -l | tr -d ' ')" + if [ "$REASON" = AnchorNeedsNodeDelivery ] && printf '%s' "$MSG" | grep -qF "$WANT" && [ "$PODS" = 0 ]; then + record PASS "engine block" "$MD held with AnchorNeedsNodeDelivery ('$WANT'), no Pod" + else + record FAIL "engine block" "$MD reason '$REASON', pods $PODS, message '${MSG:0:120}'" + fi + kubectl -n "$NS" delete modeldeployments.worker.gpustack.ai "$MD" --ignore-not-found >/dev/null 2>&1 + done + setting_set model-artifact-delivery-mode Node >/dev/null + settings_settle + fi + + echo "== the Expected identity is served from the peer ==" + if [ "${#WORKERS[@]}" -lt 2 ]; then + record SKIP "peer mount" "needs two schedulable workers, found ${#WORKERS[@]}" + elif [ "${E2E_OFFLINE:-0}" = 1 ]; then + record SKIP "peer mount" "E2E_OFFLINE=1" + else + EXPECTED_UID=$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P114}-expected" -o jsonpath='{.metadata.uid}') + kubectl -n "$SYSTEM_NS" delete pod -l app.kubernetes.io/component=model-manager \ + --field-selector "spec.nodeName=${WORKERS[1]}" --wait=true --timeout=120s >/dev/null 2>&1 + for _ in $(seq 1 30); do + PP="$(plugin_pod "${WORKERS[1]}")" + [ -n "$PP" ] && [ "$(kubectl -n "$SYSTEM_NS" get pod "$PP" -o jsonpath='{.status.phase}' 2>/dev/null)" = Running ] && break + sleep 3 + done + LOG_BEFORE="$(mh_log "$NS" "$HUB" | wc -l | tr -d ' ')" + PEER_BEFORE="$(plugin_metric "${WORKERS[1]}" "gpustack_model_manager_download_bytes_total{source=\"peer\"}")" + HUB_BEFORE="$(plugin_metric "${WORKERS[1]}" "gpustack_model_manager_download_bytes_total{source=\"hub\"}")" + consumer "$NS" "${P114}-peer" "${WORKERS[1]}" "${P114}-expected" "$EXPECTED_UID" "$DIGEST" + if pod_ready "$NS" "${P114}-peer" "$POD_BOUND"; then + LOG_AFTER="$(mh_log "$NS" "$HUB" | wc -l | tr -d ' ')" + PEER_AFTER="$(plugin_metric "${WORKERS[1]}" "gpustack_model_manager_download_bytes_total{source=\"peer\"}")" + HUB_AFTER="$(plugin_metric "${WORKERS[1]}" "gpustack_model_manager_download_bytes_total{source=\"hub\"}")" + if [ "$LOG_AFTER" != "$LOG_BEFORE" ]; then + record FAIL "peer mount" "the hub was asked (${LOG_BEFORE} → ${LOG_AFTER}) though the identity is Expected" + elif [ "$PEER_AFTER" -gt "$PEER_BEFORE" ] && [ "$HUB_AFTER" -le "$HUB_BEFORE" ]; then + record PASS "peer mount" "the second node took ${PEER_AFTER} peer bytes and ${HUB_AFTER} hub bytes" + elif [ "$PEER_AFTER" -le "$PEER_BEFORE" ] && [ "$HUB_AFTER" -le "$HUB_BEFORE" ]; then + record PASS "peer mount" "the mount is a hit on pre-warmed content (bytes flat); the hub log stayed flat" + else + record FAIL "peer mount" "the second node pulled from the hub (${HUB_BEFORE} → ${HUB_AFTER})" + fi + else + record FAIL "peer mount" "not Running within ${POD_BOUND}s: $(pod_mount_events "$NS" "${P114}-peer" | head -1)" + fi + fi +else + record SKIP "seed node" "the twin did not resolve" + record SKIP "match resolves" "the twin did not resolve" + record SKIP "mismatch refused" "the twin did not resolve" + record SKIP "anchored identity" "the twin did not resolve" + record SKIP "engine block" "the twin did not resolve" + record SKIP "peer mount" "the twin did not resolve" +fi + +# ---------------------------------------------------------------- the prefetch: a store and a grant +echo "== the prefetch: a store, a grant and a warm-up pod ==" +kubectl apply -f - >/dev/null </dev/null </dev/null)" = "${P110}-store" ] && { STORE_ON_NODE=1; break; } + sleep 3 +done +HIGH="$(kubectl get nodemodelstores.worker.gpustack.ai "$W1" \ + -o jsonpath='{.spec.watermarks.highPercent}' 2>/dev/null)" +if [ "$STORE_ON_NODE" = 1 ] && [ "$HIGH" = 95 ]; then + record PASS "the node's spec names the store and takes its watermark" "${P110}-store high=${HIGH}" +else + record FAIL "the node's spec names the store and takes its watermark" "store=$(kubectl get nodemodelstores.worker.gpustack.ai "$W1" -o jsonpath='{.spec.store}') high=${HIGH}" +fi + +if [ "$STORE_OK" = 1 ]; then + echo "== warm: available with honest counts ==" + kubectl -n "$NS" apply -f - >/dev/null </dev/null)" + QUEUED="$(python3 -c ' +import json, sys +d = json.load(open(sys.argv[1])) +items = d.get("items") or [] +print((items[0].get("metadata", {}).get("labels") or {}).get("kueue.x-k8s.io/queue-name", "") if items else "")' "$POD_CAPTURE" 2>/dev/null)" + READY="$(kubectl get nodemodelstores.worker.gpustack.ai "$W1" \ + -o jsonpath='{.status.models[?(@.digest=="'"$DIGEST_SMALL"'")].state}' 2>/dev/null)" + if [ "$WANT" = "1 1" ] && [ "$POD_NODE" = "$W1" ] && [ -z "$QUEUED" ] && [ "$READY" = Ready ]; then + record PASS "the prefetch is Available, the warm-up pod is node-pinned and label-free, the digest is Ready" "warmup@${POD_NODE:-captured}" + else + record FAIL "the prefetch is Available with an honest, label-free delivery" "counts=${WANT:-none} node=${POD_NODE:-none} queued=${QUEUED:-unset} state=${READY:-none}" + fi + else + record FAIL "the prefetch reaches Available" "$(kubectl -n "$NS" get modelprefetches.worker.gpustack.ai "${P110}-warm" -o jsonpath='{.status.conditions[?(@.type=="Progressing")].message}' 2>/dev/null)" + fi + + echo "== over budget: refused at admission ==" + BUDGET_ERR="$(kubectl -n "$NS" apply -f - 2>&1 <&1 </dev/null + if kubectl -n "$NS" apply -f - >/dev/null 2>&1 </dev/null)" in + *${DIGEST_SMALL}*) PINNED=1; break ;; + esac + sleep 3 + done + if [ "$PINNED" = 1 ]; then + record PASS "under the grant, the digest is pinned on the node" "${W1} pinned=${DIGEST_SMALL:0:19}..." + else + record FAIL "under the grant, the digest is pinned on the node" "pinned=$(kubectl get nodemodelstores.worker.gpustack.ai "$W1" -o jsonpath='{.spec.pinned}')" + fi + else + record FAIL "under the grant, the pinned prefetch is admitted" "$(echo "$PIN_ERR" | tail -1)" + fi + + echo "== delete: pod gone, pin gone, tree stays ==" + HEX="${DIGEST_SMALL#sha256:}" + POD_GONE="" UNPINNED="" TREE="$(docker exec "$W1" sh -c "test -d /var/lib/gpustack/models/published/$HEX/tree && echo present" 2>/dev/null)" + kubectl -n "$NS" delete modelprefetches.worker.gpustack.ai "${P110}-pin" --wait=false >/dev/null 2>&1 + for _ in $(seq 1 40); do + [ -z "$(kubectl -n "$NS" get pod "${P110}-pin-warmup-${W1}" -o name 2>/dev/null)" ] && POD_GONE=1 + case "$(kubectl get nodemodelstores.worker.gpustack.ai "$W1" \ + -o jsonpath='{.spec.pinned}' 2>/dev/null)" in + *${DIGEST_SMALL}*) : ;; + *) UNPINNED=1 ;; + esac + [ "$POD_GONE" = 1 ] && [ "$UNPINNED" = 1 ] && break + sleep 3 + done + if [ "$POD_GONE" = 1 ] && [ "$UNPINNED" = 1 ] && [ -n "$TREE" ]; then + record PASS "deleting the prefetch removes its pod and its pin, and leaves the tree to the grace" "tree kept: ${TREE:0:19}..." + else + record FAIL "deleting the prefetch removes its pod and its pin, and leaves the tree to the grace" "podGone=${POD_GONE:-0} unpinned=${UNPINNED:-0} tree=${TREE:-none}" + fi +else + record SKIP "the prefetch is Available, the warm-up pod is node-pinned and label-free, the digest is Ready" "the prefetch artifacts did not resolve" + record SKIP "the prefetch reaches Available" "the prefetch artifacts did not resolve" + record SKIP "a projection past the budget is refused at admission" "the prefetch artifacts did not resolve" + record SKIP "pinning without the grant's permission is refused" "the prefetch artifacts did not resolve" + record SKIP "under the grant, the digest is pinned on the node" "the prefetch artifacts did not resolve" + record SKIP "deleting the prefetch removes its pod and its pin, and leaves the tree to the grace" "the prefetch artifacts did not resolve" +fi + +# The store layer goes before the slow download: its watermarks would reclaim the content the +# download leg watches. +kubectl -n "$NS" delete modelprefetches.worker.gpustack.ai -l "$LABEL110" --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 +kubectl -n "$NS" delete modelprefetches.worker.gpustack.ai "${P110}-pin" "${P110}-big" --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 +kubectl -n "$NS" delete modelstorebindings.worker.gpustack.ai -l "$LABEL110" --ignore-not-found >/dev/null 2>&1 +kubectl delete modelstores.worker.gpustack.ai -l "$LABEL110" --ignore-not-found >/dev/null 2>&1 +kubectl -n "$NS" delete modelartifacts.worker.gpustack.ai -l "$LABEL110" --ignore-not-found >/dev/null 2>&1 +for _ in $(seq 1 20); do + [ -z "$(kubectl get nodemodelstores.worker.gpustack.ai "$W1" -o jsonpath='{.spec.store}' 2>/dev/null)" ] && break + sleep 3 +done + +# ---------------------------------------------------------------- the download: progress on thresholds +echo "== the download: progress on thresholds and nothing at rest ==" +setting_set model-store-download-bandwidth 128Ki >/dev/null +settings_settle artifact "$NS" "${P}-art" e2e/progress-model "" 108 artifact "$NSB" "${P}-other" e2e/progress-model "" 108 DIGEST="$(wait_resolved "$NS" "${P}-art" 120)" [ -n "$(wait_resolved "$NSB" "${P}-other" 120)" ] || record FAIL "the second namespace's artifact resolves" "${NSB}/${P}-other" +DOWNLOAD_OK=1 if [ -z "$DIGEST" ]; then record FAIL "the artifact resolves against the test hub" "${P}-art" - print_rows - exit 1 + DOWNLOAD_OK=0 fi SIZE="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-art" -o jsonpath='{.status.resolved.sizeBytes}')" -# ---------------------------------------------------------------- the download, watched -WATCH_OUT="${SCRATCH}/nms-watch.jsonl" -WATCH_PID="$(nms_watch "$W1" "$WATCH_OUT")" -sleep 3 -consumer "$NS" "${P}-pod" "$W1" "${P}-art" "$(get_uid "$NS" "${P}-art")" "$DIGEST" - -percents="" reads=() -for _ in $(seq 1 90); do - sleep 5 - [ "$(kubectl -n "$NS" get pod "${P}-pod" -o jsonpath='{.status.phase}' 2>/dev/null)" = Running ] && break - read -r _ dl _ pct <<<"$(nodes_of "$NS" "${P}-art")" - [ "${dl:-0}" = 1 ] && [ -n "${pct:-}" ] && percents="${percents} ${pct}" - if [ "${#reads[@]}" -lt 2 ] && [ "${dl:-0}" = 1 ]; then - ans="$(progress_as "$NS" "${P}-art" "$READER")" && reads+=("$ans") +if [ "$DOWNLOAD_OK" = 1 ]; then + WATCH_OUT="${SCRATCH}/nms-watch.jsonl" + WATCH_PID="$(nms_watch "$W1" "$WATCH_OUT")" + sleep 3 + consumer "$NS" "${P}-pod" "$W1" "${P}-art" "$(get_uid "$NS" "${P}-art")" "$DIGEST" + + percents="" reads=() + for _ in $(seq 1 90); do sleep 5 - fi -done -pod_ready "$NS" "${P}-pod" 120 || record FAIL "the consumer runs after its download" "$(pod_mount_events "$NS" "${P}-pod" | tail -1)" -RAN_AT=$(date +%s) + [ "$(kubectl -n "$NS" get pod "${P}-pod" -o jsonpath='{.status.phase}' 2>/dev/null)" = Running ] && break + read -r _ dl _ pct <<<"$(nodes_of "$NS" "${P}-art")" + [ "${dl:-0}" = 1 ] && [ -n "${pct:-}" ] && percents="${percents} ${pct}" + if [ "${#reads[@]}" -lt 2 ] && [ "${dl:-0}" = 1 ]; then + ans="$(progress_as "$NS" "${P}-art" "$READER")" && reads+=("$ans") + sleep 5 + fi + done + pod_ready "$NS" "${P}-pod" 120 || record FAIL "the consumer runs after its download" "$(pod_mount_events "$NS" "${P}-pod" | tail -1)" + RAN_AT=$(date +%s) -# ---------------------------------------------------------------- the artifact's nodes -bad="$(for p in $percents; do [ $((p % 5)) -eq 0 ] || echo "$p"; done)" -if [ -n "$percents" ] && [ -z "$bad" ]; then - record PASS "the artifact's downloading percent moved in steps of 5 while one node downloaded" "${P}-art:${percents}" -else - record FAIL "the artifact's percent is a multiple of 5 while downloading" "${P}-art: seen '${percents}', off-step '${bad}'" -fi -final="" -for _ in $(seq 1 20); do - final="$(nodes_of "$NS" "${P}-art")" + bad="$(for p in $percents; do [ $((p % 5)) -eq 0 ] || echo "$p"; done)" + if [ -n "$percents" ] && [ -z "$bad" ]; then + record PASS "the artifact's downloading percent moved in steps of 5 while one node downloaded" "${P}-art:${percents}" + else + record FAIL "the artifact's percent is a multiple of 5 while downloading" "${P}-art: seen '${percents}', off-step '${bad}'" + fi + final="" + for _ in $(seq 1 20); do + final="$(nodes_of "$NS" "${P}-art")" + read -r rd dl _ pct <<<"$final" + [ "${rd:-0}" -ge 1 ] && [ "${dl:-1}" = 0 ] && [ -z "${pct:-}" ] && break + sleep 3 + done read -r rd dl _ pct <<<"$final" - [ "${rd:-0}" -ge 1 ] && [ "${dl:-1}" = 0 ] && [ -z "${pct:-}" ] && break - sleep 3 -done -read -r rd dl _ pct <<<"$final" -if [ "${rd:-0}" -ge 1 ] && [ "${dl:-1}" = 0 ] && [ -z "${pct:-}" ]; then - record PASS "after publication the artifact counts the node ready and no percent" "${P}-art: ${final}" -else - record FAIL "after publication the node is counted ready (got '${final}')" "${P}-art" -fi + if [ "${rd:-0}" -ge 1 ] && [ "${dl:-1}" = 0 ] && [ -z "${pct:-}" ]; then + record PASS "after publication the artifact counts the node ready and no percent" "${P}-art: ${final}" + else + record FAIL "after publication the node is counted ready (got '${final}')" "${P}-art" + fi -# ---------------------------------------------------------------- progress, live -if [ "${#reads[@]}" -eq 2 ]; then - l1="$(field "${reads[0]}" live)" l2="$(field "${reads[1]}" live)" - b1="$(field "${reads[0]}" downloadingBytes)" b2="$(field "${reads[1]}" downloadingBytes)" - if [ "$l1" = 1 ] && [ "$l2" = 1 ] && [ "${b2:-0}" -ge "${b1:-1}" ] && [ "${b1:-0}" -gt 0 ]; then - record PASS "two progress reads during the download are live and never go back" "live ${l1},${l2}; bytes ${b1} -> ${b2}" + if [ "${#reads[@]}" -eq 2 ]; then + l1="$(field "${reads[0]}" live)" l2="$(field "${reads[1]}" live)" + b1="$(field "${reads[0]}" downloadingBytes)" b2="$(field "${reads[1]}" downloadingBytes)" + if [ "$l1" = 1 ] && [ "$l2" = 1 ] && [ "${b2:-0}" -ge "${b1:-1}" ] && [ "${b1:-0}" -gt 0 ]; then + record PASS "two progress reads during the download are live and never go back" "live ${l1},${l2}; bytes ${b1} -> ${b2}" + else + record FAIL "progress reads are live and non-decreasing" "live ${l1:-?},${l2:-?}; bytes ${b1:-?} -> ${b2:-?}" + fi else - record FAIL "progress reads are live and non-decreasing" "live ${l1:-?},${l2:-?}; bytes ${b1:-?} -> ${b2:-?}" + record FAIL "two progress reads were taken during the download" "${#reads[@]} taken: ${reads[*]:-none}" fi else - record FAIL "two progress reads were taken during the download" "${#reads[@]} taken: ${reads[*]:-none}" + record SKIP "the consumer runs after its download" "the artifact did not resolve" + record SKIP "the artifact's downloading percent moved in steps of 5 while one node downloaded" "the artifact did not resolve" + record SKIP "after publication the artifact counts the node ready and no percent" "the artifact did not resolve" + record SKIP "two progress reads during the download are live and never go back" "the artifact did not resolve" fi # ---------------------------------------------------------------- progress, authorization @@ -275,15 +692,6 @@ if [ "$rc" -ne 0 ] && [ -z "$(kubectl get nodemodelstores.worker.gpustack.ai "$W else record FAIL "the v1 NodeModelStore is read-only (rc ${rc})" "$(echo "$out" | head -c 200)" fi -# has_columns HEADER COLUMN...: whether the header line names every column, in order. -has_columns() { - local header="$1" col rest - shift - rest=" $(echo "$header" | tr -s ' ') " - for col in "$@"; do - case "$rest" in *" $col "*) rest="${rest#* "$col" }"; rest=" $rest" ;; *) return 1 ;; esac - done -} cols="$(kubectl -n "$NS" get modelartifacts.v1.worker.gpustack.ai 2>&1 | head -1)" if has_columns "$cols" NAME SOURCE REVISION SIZE READY DOWNLOADING RESOLVED; then record PASS "the v1 ModelArtifact prints its columns" "$(echo "$cols" | tr -s ' ')" @@ -298,30 +706,36 @@ else fi # ---------------------------------------------------------------- the writes, then the quiet node -QUIET=330 -left=$(( RAN_AT + QUIET - $(date +%s) )) -[ "$left" -gt 0 ] && sleep "$left" -stats="$(nms_write_stats "$WATCH_OUT" "$DIGEST")" -kill "$WATCH_PID" >/dev/null 2>&1 -WATCH_PID="" -progress="${stats#progress=}" progress="${progress%% *}" -mingap="${stats#*mingap=}" mingap="${mingap%% *}" -inter="${stats#*intermediate=}" inter="${inter%% *}" -last="${stats##*last=}" -if [ "${inter:-0}" -ge 2 ]; then - record PASS "the node's writes carried at least two intermediate progress values" "${W1}: ${stats}" -else - record FAIL "at least two intermediate progress values were written" "${W1}: ${stats} (size ${SIZE})" -fi -if [ "${progress:-99}" -le 20 ] && { [ "${mingap:--1}" -lt 0 ] || [ "${mingap}" -ge 29 ]; }; then - record PASS "progress-only writes: at most 20, none closer than 30 s" "${W1}: ${stats}" -else - record FAIL "progress writes keep their thresholds" "${W1}: ${stats}" -fi -if [ "${last:-0}" -gt 0 ] && [ "$last" -le $((RAN_AT + 30)) ]; then - record PASS "the node wrote nothing for a report interval after the Pod ran" "${W1}: last write $((last - RAN_AT))s after, quiet ${QUIET}s" +if [ "$DOWNLOAD_OK" = 1 ]; then + QUIET=330 + left=$(( RAN_AT + QUIET - $(date +%s) )) + [ "$left" -gt 0 ] && sleep "$left" + stats="$(nms_write_stats "$WATCH_OUT" "$DIGEST")" + kill "$WATCH_PID" >/dev/null 2>&1 + WATCH_PID="" + progress="${stats#progress=}" progress="${progress%% *}" + mingap="${stats#*mingap=}" mingap="${mingap%% *}" + inter="${stats#*intermediate=}" inter="${inter%% *}" + last="${stats##*last=}" + if [ "${inter:-0}" -ge 2 ]; then + record PASS "the node's writes carried at least two intermediate progress values" "${W1}: ${stats}" + else + record FAIL "at least two intermediate progress values were written" "${W1}: ${stats} (size ${SIZE})" + fi + if [ "${progress:-99}" -le 20 ] && { [ "${mingap:--1}" -lt 0 ] || [ "${mingap}" -ge 29 ]; }; then + record PASS "progress-only writes: at most 20, none closer than 30 s" "${W1}: ${stats}" + else + record FAIL "progress writes keep their thresholds" "${W1}: ${stats}" + fi + if [ "${last:-0}" -gt 0 ] && [ "$last" -le $((RAN_AT + 30)) ]; then + record PASS "the node wrote nothing for a report interval after the Pod ran" "${W1}: last write $((last - RAN_AT))s after, quiet ${QUIET}s" + else + record FAIL "a quiet node writes nothing" "${W1}: last write $((last - RAN_AT))s after the Pod ran" + fi else - record FAIL "a quiet node writes nothing" "${W1}: last write $((last - RAN_AT))s after the Pod ran" + record SKIP "the node's writes carried at least two intermediate progress values" "the artifact did not resolve" + record SKIP "progress-only writes: at most 20, none closer than 30 s" "the artifact did not resolve" + record SKIP "the node wrote nothing for a report interval after the Pod ran" "the artifact did not resolve" fi print_rows diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-109.sh b/.agents/skills/gpustack-operator-e2e/cases/case-109.sh deleted file mode 100644 index 0c55a2eb2..000000000 --- a/.agents/skills/gpustack-operator-e2e/cases/case-109.sh +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env bash -# -# CASE 109 — The NodeModelStore's lifecycle: it goes with its Node, and the plugin rewrites it from -# the node's disk after a restart and after the object is deleted (MUTATING, -# self-recovering; the Node row AUTO-SKIPS without docker access to the kind nodes) -# -# case-109.sh -# -# Goal: Prove the object is reconstructible and owned: a plugin restart and a deletion of the -# object each end with the entries the node held before, rewritten by the plugin from -# disk rather than from the previous status; deleting the Node object collects its -# NodeModelStore through the owner reference, and the node registered again gets a new -# one. -# Environment: A kind cluster installed from this chart with modelManager.enabled (the default), two -# workers, the stock python image pullable, and docker reaching the node containers for -# the Node row (kubelet is stopped there while its Node is deleted, then started to -# register the node again). -# Inputs: MOCKED: a test hub (_model-hub.py) in . Real: the plugin, kubelet, the worker and -# the garbage collector. -# Expected: - after the plugin Pod on a node is deleted and replaced, the node's entries (digest, -# state, size, referenced) equal the ones before; -# - after the node's NodeModelStore is deleted, the worker creates it again and the -# plugin rewrites the same entries; -# - after a worker's Node object is deleted, its NodeModelStore is gone; after kubelet -# registers the node again, a new one exists with a new UID. -# Cleanup: Trap deletes the Pods, the artifact and the hub, puts back the Settings it changed, -# and starts kubelet on a node whose Node object it deleted if it is still missing. -set -uo pipefail - -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" -CASES_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=/dev/null -. "${CASES_DIR}/_rows-lib.sh" -# shellcheck source=/dev/null -. "${CASES_DIR}/_model-hub-lib.sh" - -NS="${1:?usage: case-109.sh }" -P=c109 -LABEL="e2e.gpustack.ai/case=109" -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } -ORIG_ENDPOINT="$(setting_get model-artifact-huggingface-endpoint)" -ORIG_DELIVERY="$(setting_get model-artifact-delivery-mode)" -DELETED_NODE="" - -cleanup() { - echo - echo "[case-109] cleanup" - if [ -n "$DELETED_NODE" ] && ! kubectl get node "$DELETED_NODE" >/dev/null 2>&1; then - docker exec "$DELETED_NODE" systemctl start kubelet >/dev/null 2>&1 - fi - kubectl -n "$NS" delete pods -l e2e.gpustack.ai/consumer=true --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 - kubectl -n "$NS" delete modelartifacts.worker.gpustack.ai -l "$LABEL" --ignore-not-found >/dev/null 2>&1 - kubectl -n "$NS" delete deploy,svc,configmap -l e2e.gpustack.ai/model-hub=true --ignore-not-found >/dev/null 2>&1 - kubectl -n "$NS" delete configmap "${P}-hub-script" --ignore-not-found >/dev/null 2>&1 - if [ -n "$ORIG_ENDPOINT" ]; then setting_set model-artifact-huggingface-endpoint "$ORIG_ENDPOINT"; else setting_unset model-artifact-huggingface-endpoint; fi - if [ -n "$ORIG_DELIVERY" ]; then setting_set model-artifact-delivery-mode "$ORIG_DELIVERY"; else setting_unset model-artifact-delivery-mode; fi -} -trap cleanup EXIT - -# entries NODE: the node's entries as sorted "digest state size referenced" lines. -entries() { - kubectl get nodemodelstores.worker.gpustack.ai "$1" -o json 2>/dev/null | python3 -c " -import json, sys -for m in sorted(json.load(sys.stdin).get('status', {}).get('models', []), key=lambda m: m['digest']): - print(m['digest'], m['state'], m.get('sizeBytes', 0), m.get('referenced', False))" -} -# wait_entries NODE WANT BOUND: waits until the node's entries equal WANT; 0 once they do. -wait_entries() { - for _ in $(seq 1 $(( $3 / 3 ))); do - [ "$(entries "$1")" = "$2" ] && return 0 - sleep 3 - done - return 1 -} -plugin_ready() { # plugin_ready NODE BOUND - local pod - for _ in $(seq 1 $(( $2 / 3 ))); do - pod="$(plugin_pod "$1")" - [ -n "$pod" ] && [ "$(kubectl -n "$SYSTEM_NS" get pod "$pod" -o jsonpath='{.status.containerStatuses[?(@.name=="main")].ready}')" = true ] && return 0 - sleep 3 - done - return 1 -} - -read -r -a WORKERS <<<"$(model_workers)" -[ "${#WORKERS[@]}" -ge 2 ] || { echo "[case-109] needs two workers, found ${#WORKERS[@]}; NOTHING WAS VERIFIED"; exit 2; } -W1="${WORKERS[0]}" W2="${WORKERS[1]}" -if ! kubectl get csidriver model.csi.gpustack.ai >/dev/null 2>&1; then - echo "[case-109] the model-manager plugin is not installed; NOTHING WAS VERIFIED" - exit 2 -fi -kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null - -REPOS='{"e2e/lifecycle-model": {"files": {"config.json": {"size": 300}, "model.safetensors": {"size": 1048576, "lfs": true}}}}' -HUB_URL="$(mh_deploy "$NS" "${P}-hub" "$REPOS")" -setting_set model-artifact-huggingface-endpoint "$HUB_URL" -setting_set model-artifact-delivery-mode Node -settings_settle - -artifact "$NS" "${P}-art" e2e/lifecycle-model "" 109 -DIGEST="$(wait_resolved "$NS" "${P}-art" 120)" -UID_ART="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-art" -o jsonpath='{.metadata.uid}')" -consumer "$NS" "${P}-pod" "$W1" "${P}-art" "$UID_ART" "$DIGEST" -if [ -z "$DIGEST" ] || ! pod_ready "$NS" "${P}-pod" 300; then - record FAIL "the consumer runs on ${W1}" "$(pod_mount_events "$NS" "${P}-pod" | tail -1)" - print_rows - exit 1 -fi -BEFORE="" -for _ in $(seq 1 20); do - BEFORE="$(entries "$W1")" - echo "$BEFORE" | grep -q "^${DIGEST} Ready [0-9]* True$" && break - sleep 3 -done -echo "[case-109] ${W1} holds:" -echo "$BEFORE" - -# ---------------------------------------------------------------- plugin restart -kubectl -n "$SYSTEM_NS" delete pod "$(plugin_pod "$W1")" --wait=true --timeout=120s >/dev/null 2>&1 -if plugin_ready "$W1" 180 && wait_entries "$W1" "$BEFORE" 120; then - record PASS "after the plugin restarts the node's entries are rewritten as before" "$W1" -else - record FAIL "the plugin rewrites the same entries after a restart" "${W1}: $(entries "$W1" | tr '\n' ';')" -fi - -# ---------------------------------------------------------------- object deleted -OLD_UID="$(kubectl get nodemodelstores.worker.gpustack.ai "$W1" -o jsonpath='{.metadata.uid}')" -kubectl delete nodemodelstores.v1alpha1.worker.gpustack.ai "$W1" --wait=true >/dev/null 2>&1 -NEW_UID="" -for _ in $(seq 1 40); do - NEW_UID="$(kubectl get nodemodelstores.worker.gpustack.ai "$W1" -o jsonpath='{.metadata.uid}' 2>/dev/null)" - [ -n "$NEW_UID" ] && [ "$NEW_UID" != "$OLD_UID" ] && break - sleep 3 -done -if [ -n "$NEW_UID" ] && [ "$NEW_UID" != "$OLD_UID" ] && wait_entries "$W1" "$BEFORE" 420; then - record PASS "a deleted object comes back and the plugin rewrites the same entries from disk" "${W1}: ${OLD_UID:0:8} -> ${NEW_UID:0:8}" -else - record FAIL "the object is recreated with the node's entries" "${W1}: uid ${NEW_UID:-}, $(entries "$W1" | tr '\n' ';')" -fi - -# ---------------------------------------------------------------- the Node goes, its object with it -if ! docker exec "$W2" true >/dev/null 2>&1; then - record SKIP "the NodeModelStore goes with its Node" "no docker access to the node ${W2}" -else - OLD_UID="$(kubectl get nodemodelstores.worker.gpustack.ai "$W2" -o jsonpath='{.metadata.uid}')" - DELETED_NODE="$W2" - # kubelet stops first: a running kubelet registers its Node again within seconds, and the worker - # then adopts the object for the new Node before the garbage collector reaches it. - docker exec "$W2" systemctl stop kubelet >/dev/null 2>&1 - kubectl delete node "$W2" --wait=true >/dev/null 2>&1 - gone="" - for _ in $(seq 1 40); do - kubectl get nodemodelstores.worker.gpustack.ai "$W2" >/dev/null 2>&1 || { gone=1; break; } - sleep 3 - done - if [ -n "$OLD_UID" ] && [ -n "$gone" ]; then - record PASS "deleting the Node collects its NodeModelStore" "$W2" - else - record FAIL "the NodeModelStore goes with its Node" "${W2}: uid ${OLD_UID:-} still there" - fi - docker exec "$W2" systemctl start kubelet >/dev/null 2>&1 - NEW_UID="" - for _ in $(seq 1 80); do - NEW_UID="$(kubectl get nodemodelstores.worker.gpustack.ai "$W2" -o jsonpath='{.metadata.uid}' 2>/dev/null)" - [ -n "$NEW_UID" ] && break - sleep 3 - done - if [ -n "$NEW_UID" ] && [ "$NEW_UID" != "$OLD_UID" ]; then - record PASS "the node registered again gets a new NodeModelStore" "${W2}: ${NEW_UID:0:8}" - DELETED_NODE="" - else - record FAIL "the node registered again gets its object back" "${W2}: $(kubectl get node "$W2" -o name 2>&1)" - fi -fi - -print_rows -[ "$FAILS" -eq 0 ] || { echo "[case-109] ${FAILS} check(s) FAILED"; exit 1; } -echo "[case-109] PASS" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-11.sh b/.agents/skills/gpustack-operator-e2e/cases/case-11.sh index ecc36398f..a8e41e72f 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/case-11.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-11.sh @@ -49,8 +49,11 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" + +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_pool-lib.sh" NS="${1:?usage: case-11.sh }" SLICED=nvidia.com/gpu.sliced @@ -90,26 +93,7 @@ echo "[case-11] target: ${SLICE_NODE} with ${NCARDS} logically sliceable card(s) # its per-card memory (MiB). This must match the target node, or the queue would route the slice to a # different pool's nodes. Selected by whether the pool BACKS this node — see the index's note on pool # lookup — plus the accelerator group the gate above targeted. -read -r IT LQ CARDMEM MANUF <<<"$(kubectl get instancetypes.worker.gpustack.ai -o json 2>/dev/null | NODE_GID="$GROUPID" NODE_JSON="$(kubectl get node "$SLICE_NODE" -o json 2>/dev/null)" python3 -c " -import json,sys,os -gid=os.environ.get('NODE_GID','') -nl=(json.loads(os.environ.get('NODE_JSON') or '{}').get('metadata',{}) or {}).get('labels',{}) or {} -# A pool BACKS this node when every discriminator it carries is a label the node carries too. Those are -# the schedule labels the webhook stamps from PoolScheduleLabels, so this is the pool's whole identity — -# os/arch, the acceleratable boolean, the accelerator group and, under CPU-aware grouping, the general -# group. schedule.gpustack.ai/* is the pool's own bookkeeping and never a node label. -def backs(it): - d={k:v for k,v in (it['metadata'].get('labels') or {}).items() if not k.startswith('schedule.gpustack.ai/')} - return bool(d) and all(nl.get(k)==v for k,v in d.items()) -def in_group(s): - return bool(gid) and (s.get('acceleratorGroup') or '').endswith('-'+gid) -for it in json.load(sys.stdin).get('items',[]): - s=it.get('spec',{}); name=it['metadata']['name']; st=it.get('status',{}); d=st.get('detail',{}); sd=d.get('slicedDetail',{}) - # LOGICALLY sliceable only: a hardware-partitioned card serves no logical slice. - sliceable=(sd.get('logical',{}).get('count',0) or 0)>0 - if s.get('acceleratable') and sliceable and in_group(s) and backs(it): - print(name, st.get('entrance',''), d.get('memory',''), d.get('manufacturer','')); break -")" +read -r IT LQ CARDMEM MANUF <<<"$(discover_node_pool "$SLICE_NODE" "$GROUPID" --sliceable name entrance memory manufacturer)" [ -n "$IT" ] && [ -n "$LQ" ] || { echo "no sliceable accelerated InstanceType with an entrance LocalQueue backs ${SLICE_NODE} in group '${GROUPID}'"; exit 1; } PHYS_MIB=$(python3 -c " import re @@ -153,9 +137,6 @@ restore() { } trap restore EXIT -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } # verdict verdict() { [ "$1" -eq 0 ] && record PASS "$2" "$3" || record FAIL "$2" "$4"; } diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-110.sh b/.agents/skills/gpustack-operator-e2e/cases/case-110.sh deleted file mode 100644 index 331a52cfc..000000000 --- a/.agents/skills/gpustack-operator-e2e/cases/case-110.sh +++ /dev/null @@ -1,290 +0,0 @@ -#!/usr/bin/env bash -# -# CASE 110 — A ModelPrefetch warms an artifact onto its target node through a per-node warm-up pod: -# admission holds the line (budget, pin permission), the delivery reaches Available with -# honest counts, the pin lands only under its grant and leaves with the prefetch, and the -# node keeps the tree until its own grace decides (MUTATING, self-recovering) -# -# case-110.sh -# -# Goal: Prove the prefetch path end to end: a warm-up pod pinned to the target node mounts -# the artifact's CSI volume under a restricted namespace and never carries a queue-name -# label; the prefetch reports Available only when its nodes hold the digest; a -# projection past the budget is refused at admission; pinning without the grant's -# permission is refused and lands on the node once the grant allows it; deleting the -# prefetch deletes its pod, releases the pin, and leaves reclamation to the node's own -# grace; the store layer names itself in the node's spec. -# Environment: A cluster installed from this chart with modelManager.enabled (the default), one -# schedulable worker to label as the warm pool (two for a sharper story), and the stock -# python image pullable. No GPU, no engine, no InstanceType. -# Inputs: MOCKED: a test hub (_model-hub.py) in standing in for the Hugging Face Hub, whose -# repositories are generated from their names, so every digest is known. -# Expected: - the node's spec names the store and takes its watermarks; -# - the prefetch reaches Available with desired/ready honest, its warm-up pod pinned to -# the node, label-free of the queue-name label, and the digest Ready on the node; -# - a second prefetch whose artifact alone would pass the budget is refused with the -# budget message; -# - pinning is refused while the grant says no, and after the grant turns, the digest -# is pinned on the node; -# - deleting the prefetch deletes its pod and unpins the node, while the tree stays — -# reclamation belongs to the grace the plugin already owns. -# Cleanup: Trap deletes the prefetches, the binding, the store, the artifacts, the hub, the -# node label, and puts back the Settings it changed. -set -uo pipefail - -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" -CASES_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=/dev/null -. "${CASES_DIR}/_rows-lib.sh" -# shellcheck source=/dev/null -. "${CASES_DIR}/_model-hub-lib.sh" - -NS="${1:?usage: case-110.sh }" -P=c110 -POOL_LABEL="e2e.gpustack.ai/warm-pool" -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } -SCRATCH="$(mktemp -d)" -ORIG_ENDPOINT="$(setting_get model-artifact-huggingface-endpoint)" -ORIG_DELIVERY="$(setting_get model-artifact-delivery-mode)" -NODE_WAS_LABELED="" - -cleanup() { - echo - echo "[case-110] cleanup" - kubectl -n "$NS" delete modelprefetches.worker.gpustack.ai -l e2e.gpustack.ai/case=110 --ignore-not-found --wait=false >/dev/null 2>&1 - kubectl -n "$NS" delete modelstorebindings.worker.gpustack.ai -l e2e.gpustack.ai/case=110 --ignore-not-found --wait=false >/dev/null 2>&1 - kubectl delete modelstores.worker.gpustack.ai -l e2e.gpustack.ai/case=110 --ignore-not-found --wait=false >/dev/null 2>&1 - kubectl -n "$NS" delete modelartifacts.worker.gpustack.ai -l e2e.gpustack.ai/case=110 --ignore-not-found >/dev/null 2>&1 - kubectl -n "$NS" delete pods -l worker.gpustack.ai/model-prefetch --ignore-not-found --wait=false >/dev/null 2>&1 - kubectl -n "$NS" delete deploy,svc,configmap -l e2e.gpustack.ai/model-hub=true --ignore-not-found >/dev/null 2>&1 - if [ "$NODE_WAS_LABELED" = 1 ]; then - kubectl label node "$W1" "$POOL_LABEL-" >/dev/null 2>&1 - fi - if [ -n "$ORIG_ENDPOINT" ]; then setting_set model-artifact-huggingface-endpoint "$ORIG_ENDPOINT"; else setting_unset model-artifact-huggingface-endpoint; fi - if [ -n "$ORIG_DELIVERY" ]; then setting_set model-artifact-delivery-mode "$ORIG_DELIVERY"; else setting_unset model-artifact-delivery-mode; fi - rm -rf "$SCRATCH" -} -trap cleanup EXIT - -read -r -a WORKERS <<<"$(model_workers)" -[ "${#WORKERS[@]}" -ge 1 ] || { echo "[case-110] needs a worker, found none; NOTHING WAS VERIFIED"; exit 2; } -W1="${WORKERS[0]}" -if ! kubectl get csidriver model.csi.gpustack.ai >/dev/null 2>&1; then - echo "[case-110] the model-manager plugin is not installed; NOTHING WAS VERIFIED" - exit 2 -fi - -kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null -kubectl label namespace "$NS" --overwrite pod-security.kubernetes.io/enforce=restricted \ - pod-security.kubernetes.io/enforce-version=latest >/dev/null -if kubectl label node "$W1" "$POOL_LABEL=true" >/dev/null 2>&1; then - NODE_WAS_LABELED=1 -else - echo "[case-110] could not label ${W1} as the warm pool; NOTHING WAS VERIFIED" - exit 2 -fi - -REPOS="$(python3 -c " -import json -mib = 1 << 20 -print(json.dumps({ - 'e2e/small': {'files': {'config.json': {'size': 300}, 'tokenizer.json': {'size': 3000}, - 'model.safetensors': {'size': 5 * mib, 'lfs': True}}}, - 'e2e/big': {'files': {'config.json': {'size': 300}, 'model.safetensors': {'size': 50 * mib, 'lfs': True}}}, -}))")" -HUB_URL="$(mh_deploy "$NS" "${P}-hub" "$REPOS")" -setting_set model-artifact-huggingface-endpoint "$HUB_URL" -setting_set model-artifact-delivery-mode Node -settings_settle -echo "[case-110] hub at ${HUB_URL}; warm pool node ${W1}" - -kubectl apply -f - >/dev/null </dev/null < "$capture.tmp" 2>/dev/null; then - python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); sys.exit(0 if d.get("items") else 1)' "$capture.tmp" \ - && mv "$capture.tmp" "$capture" - fi - fi - [ "$(kubectl -n "$ns" get modelprefetches.worker.gpustack.ai "$name" \ - -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>/dev/null)" = True ] && return 0 - sleep 3 - done - return 1 -} - -# ---------------------------------------------------------------- the store layer reaches the node -STORE_ON_NODE="" -for _ in $(seq 1 20); do - [ "$(kubectl get nodemodelstores.worker.gpustack.ai "$W1" \ - -o jsonpath='{.spec.store}' 2>/dev/null)" = "${P}-store" ] && { STORE_ON_NODE=1; break; } - sleep 3 -done -HIGH="$(kubectl get nodemodelstores.worker.gpustack.ai "$W1" \ - -o jsonpath='{.spec.watermarks.highPercent}' 2>/dev/null)" -if [ "$STORE_ON_NODE" = 1 ] && [ "$HIGH" = 95 ]; then - record PASS "the node's spec names the store and takes its watermark" "${P}-store high=${HIGH}" -else - record FAIL "the node's spec names the store and takes its watermark" "store=$(kubectl get nodemodelstores.worker.gpustack.ai "$W1" -o jsonpath='{.spec.store}') high=${HIGH}" -fi - -# ---------------------------------------------------------------- warm: available with honest counts -kubectl -n "$NS" apply -f - >/dev/null </dev/null)" - QUEUED="$(python3 -c ' -import json, sys -d = json.load(open(sys.argv[1])) -items = d.get("items") or [] -print((items[0].get("metadata", {}).get("labels") or {}).get("kueue.x-k8s.io/queue-name", "") if items else "")' "$POD_CAPTURE" 2>/dev/null)" - READY="$(kubectl get nodemodelstores.worker.gpustack.ai "$W1" \ - -o jsonpath='{.status.models[?(@.digest=="'"$DIGEST_SMALL"'")].state}' 2>/dev/null)" - if [ "$WANT" = "1 1" ] && [ "$POD_NODE" = "$W1" ] && [ -z "$QUEUED" ] && [ "$READY" = Ready ]; then - record PASS "the prefetch is Available, the warm-up pod is node-pinned and label-free, the digest is Ready" "warmup@${POD_NODE:-captured}" - else - record FAIL "the prefetch is Available with an honest, label-free delivery" "counts=${WANT:-none} node=${POD_NODE:-none} queued=${QUEUED:-unset} state=${READY:-none}" - fi -else - record FAIL "the prefetch reaches Available" "$(kubectl -n "$NS" get modelprefetches.worker.gpustack.ai "${P}-warm" -o jsonpath='{.status.conditions[?(@.type=="Progressing")].message}' 2>/dev/null)" -fi - -# ---------------------------------------------------------------- over budget: refused at admission -BUDGET_ERR="$(kubectl -n "$NS" apply -f - 2>&1 <&1 </dev/null -if kubectl -n "$NS" apply -f - >/dev/null 2>&1 </dev/null)" in - *${DIGEST_SMALL}*) PINNED=1; break ;; - esac - sleep 3 - done - if [ "$PINNED" = 1 ]; then - record PASS "under the grant, the digest is pinned on the node" "${W1} pinned=${DIGEST_SMALL:0:19}..." - else - record FAIL "under the grant, the digest is pinned on the node" "pinned=$(kubectl get nodemodelstores.worker.gpustack.ai "$W1" -o jsonpath='{.spec.pinned}')" - fi -else - record FAIL "under the grant, the pinned prefetch is admitted" "$(echo "$PIN_ERR" | tail -1)" -fi - -# ---------------------------------------------------------------- delete: pod gone, pin gone, tree stays -HEX="${DIGEST_SMALL#sha256:}" -POD_GONE="" UNPINNED="" TREE="$(docker exec "$W1" sh -c "test -d /var/lib/gpustack/models/published/$HEX/tree && echo present" 2>/dev/null)" -kubectl -n "$NS" delete modelprefetches.worker.gpustack.ai "${P}-pin" --wait=false >/dev/null 2>&1 -for _ in $(seq 1 40); do - [ -z "$(kubectl -n "$NS" get pod "${P}-pin-warmup-${W1}" -o name 2>/dev/null)" ] && POD_GONE=1 - case "$(kubectl get nodemodelstores.worker.gpustack.ai "$W1" \ - -o jsonpath='{.spec.pinned}' 2>/dev/null)" in - *${DIGEST_SMALL}*) : ;; - *) UNPINNED=1 ;; - esac - [ "$POD_GONE" = 1 ] && [ "$UNPINNED" = 1 ] && break - sleep 3 -done -if [ "$POD_GONE" = 1 ] && [ "$UNPINNED" = 1 ] && [ -n "$TREE" ]; then - record PASS "deleting the prefetch removes its pod and its pin, and leaves the tree to the grace" "tree kept: ${TREE:0:19}..." -else - record FAIL "deleting the prefetch removes its pod and its pin, and leaves the tree to the grace" "podGone=${POD_GONE:-0} unpinned=${UNPINNED:-0} tree=${TREE:-none}" -fi - -print_rows -[ "$FAILS" -eq 0 ] || { echo "[case-110] ${FAILS} check(s) FAILED"; exit 1; } -echo "[case-110] PASS" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-111.sh b/.agents/skills/gpustack-operator-e2e/cases/case-111.sh index bfdc6add5..caf8d3ef4 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-111.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-111.sh @@ -15,7 +15,8 @@ # Environment: A cluster installed from this chart with modelManager.port set to a non-zero port # (the feature's switch; the default 0 turns the listener off and this case is # meaningless), the model-manager NetworkPolicy enabled, at least two schedulable -# workers, and the stock busybox image pullable. No GPU, no engine. +# workers, and the stock busybox image and the model's python image (MH_PYTHON) +# pullable. No GPU, no engine. # Inputs: MOCKED: a test hub (_model-hub.py) in standing in for the Hugging Face Hub, # whose repositories are generated from their names, so every digest is known. # Expected: - the seed node's entry is Ready with source=Hub and the hub byte counter for the @@ -32,8 +33,6 @@ # pod it killed comes back on its own (DaemonSet), and puts back the Setting. set -uo pipefail -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" CASES_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null . "${CASES_DIR}/_rows-lib.sh" @@ -43,9 +42,9 @@ CASES_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" NS="${1:?usage: case-111.sh }" P=c111 PEER_PORT=32446 -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } +POD_BOUND=600 +PEER_METRIC='gpustack_model_manager_download_bytes_total{source="peer"}' +HUB_METRIC='gpustack_model_manager_download_bytes_total{source="hub"}' ORIG_PEER_SYNC="$(setting_get model-store-peer-sync)" cleanup() { @@ -53,10 +52,14 @@ cleanup() { echo "[case-111] cleanup" kubectl -n "$NS" delete modelartifacts.worker.gpustack.ai -l e2e.gpustack.ai/case=111 --ignore-not-found --wait=false >/dev/null 2>&1 kubectl -n "$NS" delete pods -l e2e.gpustack.ai/case=111 --ignore-not-found --wait=false --force --grace-period=0 >/dev/null 2>&1 + kubectl -n "$NS" delete pod "c111-mount-${COLD:-}" --ignore-not-found --wait=false --force --grace-period=0 >/dev/null 2>&1 + kubectl -n "$NS" delete pod "c111-mount-${SEED:-}" --ignore-not-found --wait=false --force --grace-period=0 >/dev/null 2>&1 kubectl -n "$NS" delete deploy,svc,configmap -l e2e.gpustack.ai/model-hub=true --ignore-not-found >/dev/null 2>&1 if [ -n "$ORIG_PEER_SYNC" ]; then setting_set model-store-peer-sync "$ORIG_PEER_SYNC"; else setting_unset model-store-peer-sync; fi rm -rf "$SCRATCH" } +SEED="" +COLD="" SCRATCH="$(mktemp -d)" trap cleanup EXIT @@ -71,49 +74,16 @@ if [ "$(setting_get model-store-peer-sync)" != "true" ]; then fi kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null -counter() { # counter — the byte counter of the plugin pod ON that node. - local node="$1" source="$2" pod - pod="$(kubectl -n "$NS" get pods -l app.kubernetes.io/component=model-manager --field-selector "spec.nodeName=$node" -o jsonpath='{.items[0].metadata.name}')" - kubectl -n "$NS" exec "$pod" -c main -- curl -sk "https://127.0.0.1:32444/metrics" 2>/dev/null | - grep "download_bytes_total{source=\"$source\"}" | awk '{print $2}' | head -1 -} -wait_ready() { # wait_ready — until the node lists the artifact's digest Ready. +wait_ready() { # wait_ready — until the node lists the artifact's digest Ready. local digest="$1" node="$2" i=0 state="" while [ $i -lt 120 ]; do - state="$(kubectl get nodemodelstores.v1alpha1.worker.gpustack.ai "$node" -o jsonpath="{.status.models[?(@.digest=='$digest')].state}" 2>/dev/null || true)" + state="$(store_state "$node" "$digest")" + state="${state%%|*}" [ "$state" = "Ready" ] && return 0 i=$((i + 1)); sleep 5 done return 1 } -mount_pod() { # mount_pod — a plain consumer that drives a materialization. - local node="$1" artifact="$2" uid digest - uid="$(kubectl -n "$NS" get modelartifacts.v1alpha1.worker.gpustack.ai "$artifact" -o jsonpath='{.metadata.uid}')" - digest="$(kubectl -n "$NS" get modelartifacts.v1alpha1.worker.gpustack.ai "$artifact" -o jsonpath='{.status.resolved.manifestDigest}')" - [ -n "$digest" ] || { echo "[case-111] the artifact never resolved"; return 1; } - kubectl -n "$NS" delete pod "c111-mount-$node" --ignore-not-found --wait=true >/dev/null 2>&1 - kubectl apply -n "$NS" -f - >/dev/null </dev/null 2>&1 -SEED_PLUGIN="$(kubectl -n "$NS" get pods -l app.kubernetes.io/component=model-manager --field-selector "spec.nodeName=$SEED" -o jsonpath='{.items[0].metadata.name}')" +SEED_PLUGIN="$(plugin_pod "$SEED")" HEX="$(kubectl -n "$NS" get modelartifacts.v1alpha1.worker.gpustack.ai ${P}-weights -o jsonpath='{.status.resolved.manifestDigest}' | sed 's/^sha256://')" -if ! kubectl -n "$NS" exec "$SEED_PLUGIN" -c main -- rm -rf "/var/lib/gpustack/models/published/$HEX"; then +if ! kubectl -n "$SYSTEM_NS" exec "$SEED_PLUGIN" -c main -- rm -rf "/var/lib/gpustack/models/published/$HEX"; then echo "[case-111] could not clear the seed's published tree; NOTHING WAS VERIFIED" exit 2 fi -kubectl -n "$NS" delete pod "$SEED_PLUGIN" >/dev/null 2>&1 +kubectl -n "$SYSTEM_NS" delete pod "$SEED_PLUGIN" >/dev/null 2>&1 for i in $(seq 1 30); do - SEED_PLUGIN="$(kubectl -n "$NS" get pods -l app.kubernetes.io/component=model-manager --field-selector "spec.nodeName=$SEED" -o jsonpath='{.items[0].metadata.name}')" - kubectl -n "$NS" get pod "$SEED_PLUGIN" -o jsonpath='{.status.containerStatuses[0].ready}' 2>/dev/null | grep -q true && break + SEED_PLUGIN="$(plugin_pod "$SEED")" + kubectl -n "$SYSTEM_NS" get pod "$SEED_PLUGIN" -o jsonpath='{.status.containerStatuses[0].ready}' 2>/dev/null | grep -q true && break sleep 5 done -mount_pod "$SEED" "${P}-weights" -DIGEST="$(kubectl -n "$NS" get modelartifacts.v1alpha1.worker.gpustack.ai ${P}-weights -o jsonpath='{.status.resolved.manifestDigest}')" +SEED_UID="$(kubectl -n "$NS" get modelartifacts.v1alpha1.worker.gpustack.ai "${P}-weights" -o jsonpath='{.metadata.uid}')" +DIGEST="$(kubectl -n "$NS" get modelartifacts.v1alpha1.worker.gpustack.ai "${P}-weights" -o jsonpath='{.status.resolved.manifestDigest}')" +if [ -n "$DIGEST" ]; then + consumer "$NS" "c111-mount-$SEED" "$SEED" "${P}-weights" "$SEED_UID" "$DIGEST" +else + echo "[case-111] the artifact never resolved" +fi if wait_ready "$DIGEST" "$SEED"; then record PASS "seed" "the seed node pulled from the hub and is Ready" else record FAIL "seed" "the seed node never reached Ready" fi -SEED_HUB="$(counter "$SEED" hub)" -SEED_PEER="$(counter "$SEED" peer)" +SEED_HUB="$(plugin_metric "$SEED" "$HUB_METRIC")" +SEED_PEER="$(plugin_metric "$SEED" "$PEER_METRIC")" TOTAL_BYTES="$(kubectl get nodemodelstores.v1alpha1.worker.gpustack.ai "$SEED" -o jsonpath="{.status.models[?(@.digest=='$DIGEST')].sizeBytes}" 2>/dev/null)" # A plugin restart resets the counter to zero mid-measurement; in that case the attempt's # pull is exactly what the reset counter now shows. @@ -162,10 +137,16 @@ else fi echo "[case-111] pull: $COLD materializes the same digest" -PEER_BASE="$(counter "$COLD" peer)"; PEER_BASE="${PEER_BASE:-0}" -HUB_BASE="$(counter "$COLD" hub)"; HUB_BASE="${HUB_BASE:-0}" -mount_pod "$COLD" "${P}-weights" -COLD_POD="$(kubectl -n "$NS" get pods -l e2e.gpustack.ai/case=111 --field-selector "spec.nodeName=$COLD" -o jsonpath='{.items[0].metadata.name}')" +ART_UID="$(kubectl -n "$NS" get modelartifacts.v1alpha1.worker.gpustack.ai "${P}-weights" -o jsonpath='{.metadata.uid}')" +# The cold node's plugin pod is deleted before the mount, so its cache starts empty and the +# digest can only come from the peer; the helper samples the byte counters around the mount. +COUNTERS="$(mh_peer_sync_check "$NS" "c111-mount-$COLD" "$COLD" "${P}-weights" "$ART_UID" "$DIGEST" "$POD_BOUND")" +read -r PEER_BASE HUB_BASE COLD_PEER COLD_HUB <<< "$COUNTERS" +PEER_BASE="${PEER_BASE:-0}" +HUB_BASE="${HUB_BASE:-0}" +COLD_PEER="${COLD_PEER:-0}" +COLD_HUB="${COLD_HUB:-0}" +COLD_POD="c111-mount-$COLD" if wait_ready "$DIGEST" "$COLD"; then record PASS "pull" "the cold node reached Ready" else @@ -177,8 +158,6 @@ if [ "$COLD_STATE" = "Peer" ]; then else record FAIL "source" "the cold node's entry says $COLD_STATE, want Peer" fi -COLD_PEER="$(counter "$COLD" peer)" -COLD_HUB="$(counter "$COLD" hub)" PEER_DELTA="$(awk -v a="$COLD_PEER" -v b="$PEER_BASE" 'BEGIN{print a-b}')" HUB_DELTA="$(awk -v a="$COLD_HUB" -v b="$HUB_BASE" 'BEGIN{print a-b}')" if [ -n "$COLD_PEER" ] && awk -v a="$PEER_DELTA" -v b="${TOTAL_BYTES:-0}" 'BEGIN{exit !(a+0 >= b*0.999 && a+0 <= b*1.001)}'; then @@ -204,7 +183,6 @@ fi kubectl -n "$NS" delete pod c111-tenant --force --grace-period=0 --ignore-not-found >/dev/null 2>&1 echo -echo "[case-111] rows" -for r in "${ROWS[@]}"; do echo " $r"; done +print_rows echo "[case-111] seed hub bytes: ${SEED_HUB:-0}, seed peer bytes: ${SEED_PEER:-0}; cold hub bytes: ${COLD_HUB:-0}, cold peer bytes: ${COLD_PEER:-0}" [ "$FAILS" -eq 0 ] || exit 1 diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-112.sh b/.agents/skills/gpustack-operator-e2e/cases/case-112.sh index 473cabff1..b840f8887 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-112.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-112.sh @@ -34,8 +34,8 @@ # and forwarder, and the scratch dir. set -uo pipefail -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" CASES_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null . "${CASES_DIR}/_rows-lib.sh" @@ -44,9 +44,6 @@ NS="${1:?usage: case-112.sh }" P=c112 REG_PORT=35000 # the node's loopback port the forwarder serves the registry on PUSH_PORT=5550 # the runner's loopback port port-forward pushes through -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } cleanup() { echo @@ -340,7 +337,6 @@ else record PASS "prefetch-refused" "the prefetch was refused: nothing to warm" fi -echo echo "[case-112] rows" -for r in "${ROWS[@]}"; do echo " $r"; done +print_rows [ "$FAILS" -eq 0 ] || exit 1 diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-113.sh b/.agents/skills/gpustack-operator-e2e/cases/case-113.sh index d37514bed..a5116ba58 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-113.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-113.sh @@ -30,8 +30,6 @@ # Cleanup: Trap deletes the Pods and the artifact. set -uo pipefail -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" CASES_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null . "${CASES_DIR}/_rows-lib.sh" @@ -47,9 +45,6 @@ P=c113 REPO="qwen/Qwen2.5-0.5B-Instruct" COMMIT_BOUND=120 POD_BOUND=300 -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } WORKERS=() for _w in $(model_workers); do WORKERS+=("$_w"); done @@ -132,23 +127,12 @@ else fi echo "== 3. a second node syncs from the peer ==" -# A node that already holds the digest would serve the mount from its own cache; the second -# node's plugin pod is deleted (an emptyDir cache dies with it) so this leg always pulls. -kubectl -n "$SYSTEM_NS" delete pod -l app.kubernetes.io/component=model-manager \ - --field-selector "spec.nodeName=${WORKERS[1]}" --wait=true --timeout=120s >/dev/null 2>&1 -for _ in $(seq 1 30); do - PP="$(plugin_pod "${WORKERS[1]}")" - [ -n "$PP" ] && [ "$(kubectl -n "$SYSTEM_NS" get pod "$PP" -o jsonpath='{.status.phase}' 2>/dev/null)" = Running ] && break - sleep 3 -done -PEER_BEFORE="$(plugin_metric "${WORKERS[1]}" "gpustack_model_manager_download_bytes_total{source=\"peer\"}")" -HUB_BEFORE="$(plugin_metric "${WORKERS[1]}" "gpustack_model_manager_download_bytes_total{source=\"hub\"}")" -consumer "$NS" "${P}-peer" "${WORKERS[1]}" "${P}-qwen" "$ART_UID" "$DIGEST" -if pod_ready "$NS" "${P}-peer" "$POD_BOUND"; then +# The second node's plugin pod is deleted before the mount, so its cache starts empty and this +# leg always pulls; the helper samples the byte counters around the mount. +if COUNTERS="$(mh_peer_sync_check "$NS" "${P}-peer" "${WORKERS[1]}" "${P}-qwen" "$ART_UID" "$DIGEST" "$POD_BOUND")"; then + read -r PEER_BEFORE HUB_BEFORE PEER_AFTER HUB_AFTER <<< "$COUNTERS" GOT="$(kubectl -n "$NS" logs "${P}-peer" 2>/dev/null | sort)" WANT="$(sort "$MS_SHA_JSON")" - PEER_AFTER="$(plugin_metric "${WORKERS[1]}" "gpustack_model_manager_download_bytes_total{source=\"peer\"}")" - HUB_AFTER="$(plugin_metric "${WORKERS[1]}" "gpustack_model_manager_download_bytes_total{source=\"hub\"}")" if [ "$GOT" != "$WANT" ]; then record FAIL "peer sync" "the second Pod's hashes differ from the hub's listing" elif [ "$PEER_AFTER" -le "$PEER_BEFORE" ] && [ "$HUB_AFTER" -le "$HUB_BEFORE" ]; then diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-114.sh b/.agents/skills/gpustack-operator-e2e/cases/case-114.sh deleted file mode 100755 index 8d8aa1c24..000000000 --- a/.agents/skills/gpustack-operator-e2e/cases/case-114.sh +++ /dev/null @@ -1,241 +0,0 @@ -#!/usr/bin/env bash -# -# CASE 114 — An expectedDigest asserts the content: a match resolves, a mismatch refuses naming both -# digests, a twice-unreachable hub hands the identity to the anchor, and an anchored -# artifact is delivered by a node alone (MUTATING, self-recovering; the peer leg -# AUTO-SKIPS on one worker or with E2E_C114_OFFLINE=1) -# -# case-114.sh -# -# Goal: Pin the anchor end to end against a controlled hub: the digest a resolution -# produces is exactly the assertion, a hub serving other content is refused with both -# digests in the message, a hub that twice cannot be reached writes the anchor as the -# identity (digestSource Expected, no revision) and is never asked again, and an -# anchored artifact is held under Engine delivery and served from the peer's published -# tree under Node delivery. -# Environment: A cluster installed from this chart with modelManager.enabled (the default), the -# stock python image pullable, and two schedulable workers for the peer leg. NO GPU -# and no internet: the hub is this case's own in-cluster test hub, so the refusal and -# the confirmed outage are exact. -# Inputs: MOCKED: the hub (_model-hub.py) serving one small repository, and the worker's hub -# endpoint Setting pointed at it for the run and restored afterwards. Everything the -# anchor reads — resolution, the staircase, the ledger — is the real code path. -# Expected: - an unanchored twin resolves, and its digest is published on the first worker; -# - an artifact anchored to that digest resolves with digestSource Hub and a commit; -# - an artifact anchored to another digest stays Resolved=False, reason -# DigestMismatch, message carrying both digests; -# - with the hub unreachable, an artifact anchored to the first digest waits out one -# blip, then resolves to the anchor (digestSource Expected, revision empty), while -# the hub's request log stays flat from the flip onward; -# - under Engine delivery both anchored artifacts are held with -# AnchorNeedsNodeDelivery — the Expected one's message naming the missing commit — -# and no Pod is created; -# - a consumer on the second worker mounts the Expected identity, the hub's request -# log still flat, its peer bytes risen (or the node pre-warmed). -# Cleanup: Trap restores both Settings, deletes the Pods, artifacts, deployments and the hub. -set -uo pipefail - -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" -CASES_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=/dev/null -. "${CASES_DIR}/_rows-lib.sh" -# shellcheck source=/dev/null -. "${CASES_DIR}/_model-hub-lib.sh" - -NS="${1:?usage: case-114.sh }" -if [ "$NS" = "$SYSTEM_NS" ]; then - echo "[case-114] refusing system namespace ${SYSTEM_NS}; usage: case-114.sh " >&2 - exit 2 -fi -P=c114 -COMMIT_BOUND=120 -ANCHOR_BOUND=240 -POD_BOUND=300 -WRONG="sha256:$(printf 'f%.0s' $(seq 64))" -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } - -WORKERS=() -for _w in $(model_workers); do WORKERS+=("$_w"); done -if [ "${#WORKERS[@]}" -lt 1 ]; then - echo "[case-114] SKIP: needs a schedulable worker, found none" >&2 - exit 0 -fi - -# The two Settings this case flips, saved so the trap restores whatever the cluster carried. -EP_SAVED="$(setting_get model-artifact-huggingface-endpoint)" -DELIVERY_SAVED="$(setting_get model-artifact-delivery-mode)" - -# shellcheck disable=SC2317 # the trap keeps it reachable; the checker cannot see the EXIT -cleanup() { - echo - echo "[case-114] cleanup" - if [ -n "$EP_SAVED" ]; then setting_set model-artifact-huggingface-endpoint "$EP_SAVED" >/dev/null; else setting_unset model-artifact-huggingface-endpoint >/dev/null; fi - if [ -n "$DELIVERY_SAVED" ]; then setting_set model-artifact-delivery-mode "$DELIVERY_SAVED" >/dev/null; else setting_unset model-artifact-delivery-mode >/dev/null; fi - kubectl -n "$NS" delete pod "${P}-seed" "${P}-peer" --ignore-not-found --wait=true --timeout=180s >/dev/null 2>&1 - kubectl -n "$NS" delete modeldeployments.worker.gpustack.ai "${P}-md-match" "${P}-md-expected" --ignore-not-found >/dev/null 2>&1 - kubectl -n "$NS" delete modelartifacts.worker.gpustack.ai "${P}-twin" "${P}-match" "${P}-mismatch" "${P}-expected" --ignore-not-found >/dev/null 2>&1 - kubectl -n "$NS" delete deploy "${P}-hub" --ignore-not-found >/dev/null 2>&1 - kubectl -n "$NS" delete svc "${P}-hub" --ignore-not-found >/dev/null 2>&1 - kubectl -n "$NS" delete configmap "${P}-hub-script" --ignore-not-found >/dev/null 2>&1 -} -trap cleanup EXIT - -REPOS="$(python3 -c " -import json -print(json.dumps({ - 'e2e/anchored': {'files': {'config.json': {'size': 300}, 'tokenizer.json': {'size': 3000}}}, -}))")" -HUB_URL="$(mh_deploy "$NS" "${P}-hub" "$REPOS")" -setting_set model-artifact-huggingface-endpoint "$HUB_URL" >/dev/null -setting_set model-artifact-delivery-mode Node >/dev/null -settings_settle -echo "[case-114] hub at ${HUB_URL}; workers ${WORKERS[*]}" - -echo "== 1. the twin resolves, its digest publishes on the first worker ==" -artifact "$NS" "${P}-twin" e2e/anchored "" "$P" -DIGEST="$(wait_resolved "$NS" "${P}-twin" "$COMMIT_BOUND")" -if [ -n "$DIGEST" ]; then - record PASS "twin resolved" "digest ${DIGEST:0:19}…" -else - record FAIL "twin resolved" "no digest within ${COMMIT_BOUND}s: $(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-twin" -o jsonpath='{.status.conditions[?(@.type=="Resolved")].message}' 2>/dev/null)" -fi -[ -n "$DIGEST" ] || { print_rows "${ROWS[@]}"; exit 1; } - -TWIN_UID=$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-twin" -o jsonpath='{.metadata.uid}') -consumer "$NS" "${P}-seed" "${WORKERS[0]}" "${P}-twin" "$TWIN_UID" "$DIGEST" -if pod_ready "$NS" "${P}-seed" "$POD_BOUND"; then - record PASS "seed node" "the first worker holds the published tree" -else - record FAIL "seed node" "not Running within ${POD_BOUND}s: $(pod_mount_events "$NS" "${P}-seed" | head -1)" -fi - -echo "== 2. a matching anchor resolves as a hub identity ==" -artifact "$NS" "${P}-match" e2e/anchored "" "$P" " - expectedDigest: $DIGEST" -MATCH_DIGEST="$(wait_resolved "$NS" "${P}-match" "$COMMIT_BOUND")" -MATCH_SRC="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-match" -o jsonpath='{.status.resolved.digestSource}' 2>/dev/null)" -MATCH_REV="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-match" -o jsonpath='{.status.resolved.revision}' 2>/dev/null)" -if [ "$MATCH_DIGEST" = "$DIGEST" ] && [ "$MATCH_SRC" = Hub ] && [ -n "$MATCH_REV" ]; then - record PASS "match resolves" "digestSource Hub at commit ${MATCH_REV:0:12}…" -else - record FAIL "match resolves" "digest '${MATCH_DIGEST:0:19}…' source '$MATCH_SRC' revision '$MATCH_REV'" -fi - -echo "== 3. a foreign anchor is refused with both digests ==" -artifact "$NS" "${P}-mismatch" e2e/anchored "" "$P" " - expectedDigest: $WRONG" -MISMATCH_MSG="" -for _ in $(seq 1 45); do - R="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-mismatch" \ - -o jsonpath='{.status.conditions[?(@.type=="Resolved")].reason}' 2>/dev/null)" - [ "$R" = DigestMismatch ] && break - sleep 2 -done -MISMATCH_MSG="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-mismatch" -o jsonpath='{.status.conditions[?(@.type=="Resolved")].message}' 2>/dev/null)" -if [ "$R" = DigestMismatch ] && printf '%s' "$MISMATCH_MSG" | grep -qF "$WRONG" && printf '%s' "$MISMATCH_MSG" | grep -qF "$DIGEST"; then - record PASS "mismatch refused" "DigestMismatch names the anchor and the hub's digest" -else - record FAIL "mismatch refused" "reason '$R', message '${MISMATCH_MSG:0:160}'" -fi - -echo "== 4. a twice-unreachable hub hands the identity to the anchor ==" -LOG_BEFORE="$(mh_log "$NS" "${P}-hub" | wc -l | tr -d ' ')" -setting_set model-artifact-huggingface-endpoint "http://${P}-absent.${NS}.svc:8080" >/dev/null -settings_settle -artifact "$NS" "${P}-expected" e2e/anchored "" "$P" " - expectedDigest: $DIGEST" -EXPECTED_DIGEST="$(wait_resolved "$NS" "${P}-expected" "$ANCHOR_BOUND")" -EXPECTED_SRC="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-expected" -o jsonpath='{.status.resolved.digestSource}' 2>/dev/null)" -EXPECTED_REV="$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-expected" -o jsonpath='{.status.resolved.revision}' 2>/dev/null)" -LOG_DURING="$(mh_log "$NS" "${P}-hub" | wc -l | tr -d ' ')" -if [ "$EXPECTED_DIGEST" = "$DIGEST" ] && [ "$EXPECTED_SRC" = Expected ] && [ -z "$EXPECTED_REV" ] && [ "$LOG_DURING" = "$LOG_BEFORE" ]; then - record PASS "anchored identity" "digestSource Expected, no revision, hub log flat (${LOG_BEFORE} → ${LOG_DURING})" -else - record FAIL "anchored identity" "digest '${EXPECTED_DIGEST:0:19}…' source '$EXPECTED_SRC' revision '$EXPECTED_REV' log ${LOG_BEFORE} → ${LOG_DURING}" -fi -if [ -n "$EP_SAVED" ]; then setting_set model-artifact-huggingface-endpoint "$EP_SAVED" >/dev/null; else setting_unset model-artifact-huggingface-endpoint >/dev/null; fi -settings_settle - -echo "== 5. Engine delivery is refused for anchored artifacts ==" -IT="$(kubectl get instancetypes.worker.gpustack.ai -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)" -if [ -z "$IT" ]; then - record SKIP "engine block" "no InstanceType exists to reference; the render is unit-covered" -else - setting_set model-artifact-delivery-mode Engine >/dev/null - settings_settle - for pair in "${P}-match:${P}-md-match:cannot anchor-verify" "${P}-expected:${P}-md-expected:no resolved commit"; do - ART="${pair%%:*}"; REST="${pair#*:}"; MD="${REST%%:*}"; WANT="${REST#*:}" - kubectl apply -f - >/dev/null </dev/null)" - [ "$REASON" = AnchorNeedsNodeDelivery ] && break - sleep 2 - done - MSG="$(kubectl -n "$NS" get modeldeployments.worker.gpustack.ai "$MD" -o jsonpath='{.status.conditions[?(@.type=="WeightsReady")].message}' 2>/dev/null)" - PODS="$(kubectl -n "$NS" get pods -l "worker.gpustack.ai/model-deployment=$MD" --no-headers 2>/dev/null | wc -l | tr -d ' ')" - if [ "$REASON" = AnchorNeedsNodeDelivery ] && printf '%s' "$MSG" | grep -qF "$WANT" && [ "$PODS" = 0 ]; then - record PASS "engine block" "$MD held with AnchorNeedsNodeDelivery ('$WANT'), no Pod" - else - record FAIL "engine block" "$MD reason '$REASON', pods $PODS, message '${MSG:0:120}'" - fi - kubectl -n "$NS" delete modeldeployments.worker.gpustack.ai "$MD" --ignore-not-found >/dev/null 2>&1 - done - if [ -n "$DELIVERY_SAVED" ]; then setting_set model-artifact-delivery-mode "$DELIVERY_SAVED" >/dev/null; else setting_unset model-artifact-delivery-mode >/dev/null; fi - settings_settle -fi - -echo "== 6. the Expected identity is served from the peer ==" -if [ "${#WORKERS[@]}" -lt 2 ]; then - record SKIP "peer mount" "needs two schedulable workers, found ${#WORKERS[@]}" -elif [ "${E2E_C114_OFFLINE:-0}" = 1 ]; then - record SKIP "peer mount" "E2E_C114_OFFLINE=1" -else - EXPECTED_UID=$(kubectl -n "$NS" get modelartifacts.worker.gpustack.ai "${P}-expected" -o jsonpath='{.metadata.uid}') - kubectl -n "$SYSTEM_NS" delete pod -l app.kubernetes.io/component=model-manager \ - --field-selector "spec.nodeName=${WORKERS[1]}" --wait=true --timeout=120s >/dev/null 2>&1 - for _ in $(seq 1 30); do - PP="$(plugin_pod "${WORKERS[1]}")" - [ -n "$PP" ] && [ "$(kubectl -n "$SYSTEM_NS" get pod "$PP" -o jsonpath='{.status.phase}' 2>/dev/null)" = Running ] && break - sleep 3 - done - LOG_BEFORE="$(mh_log "$NS" "${P}-hub" | wc -l | tr -d ' ')" - PEER_BEFORE="$(plugin_metric "${WORKERS[1]}" "gpustack_model_manager_download_bytes_total{source=\"peer\"}")" - HUB_BEFORE="$(plugin_metric "${WORKERS[1]}" "gpustack_model_manager_download_bytes_total{source=\"hub\"}")" - consumer "$NS" "${P}-peer" "${WORKERS[1]}" "${P}-expected" "$EXPECTED_UID" "$DIGEST" - if pod_ready "$NS" "${P}-peer" "$POD_BOUND"; then - LOG_AFTER="$(mh_log "$NS" "${P}-hub" | wc -l | tr -d ' ')" - PEER_AFTER="$(plugin_metric "${WORKERS[1]}" "gpustack_model_manager_download_bytes_total{source=\"peer\"}")" - HUB_AFTER="$(plugin_metric "${WORKERS[1]}" "gpustack_model_manager_download_bytes_total{source=\"hub\"}")" - if [ "$LOG_AFTER" != "$LOG_BEFORE" ]; then - record FAIL "peer mount" "the hub was asked (${LOG_BEFORE} → ${LOG_AFTER}) though the identity is Expected" - elif [ "$PEER_AFTER" -gt "$PEER_BEFORE" ] && [ "$HUB_AFTER" -le "$HUB_BEFORE" ]; then - record PASS "peer mount" "the second node took ${PEER_AFTER} peer bytes and ${HUB_AFTER} hub bytes" - elif [ "$PEER_AFTER" -le "$PEER_BEFORE" ] && [ "$HUB_AFTER" -le "$HUB_BEFORE" ]; then - record PASS "peer mount" "the mount is a hit on pre-warmed content (bytes flat); the hub log stayed flat" - else - record FAIL "peer mount" "the second node pulled from the hub (${HUB_BEFORE} → ${HUB_AFTER})" - fi - else - record FAIL "peer mount" "not Running within ${POD_BOUND}s: $(pod_mount_events "$NS" "${P}-peer" | head -1)" - fi -fi - -print_rows "${ROWS[@]}" -exit "$FAILS" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-116.sh b/.agents/skills/gpustack-operator-e2e/cases/case-116.sh index 15f80b89a..93d993d06 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/case-116.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-116.sh @@ -1,20 +1,38 @@ #!/usr/bin/env bash # -# CASE 116 — Every encoder-role admission refusal fires, each naming what its rule refuses -# (MUTATING, self-recovering: one settings key and one injected node label, both restored by -# the trap; every verdict below is a server-side dry run that persists nothing) -# AUTO-SKIPS its matrix rows without an InstanceType that reports an admitted accelerator -# platform, which this case builds by injecting a CPU vendor label. +# CASE 116 — Every encoder-role admission refusal fires, each naming what its rule refuses, and +# the admitted matrix is read on the platforms it admits and on the one it refuses +# (MUTATING, self-recovering: one settings key and one injected node label, both restored +# by the trap; every verdict below is a server-side dry run that persists nothing) +# AUTO-SKIPS its matrix rows without an InstanceType that reports an admitted +# accelerator platform, which this case builds by injecting a CPU vendor label # # case-116.sh # # Goal: A deployment that declares an Encode role is refused by a set of rules that a # deployment without one never meets, and each refusal names its own rule. This case -# pins every rule on the encoder path and, like case 51 before it, pins WHICH rule +# pins every rule on the encoder path and, like case 45 before it, pins WHICH rule # answered: a row that only checked "it was rejected" passes identically whether the # rule exists or not, so every row asserts a fragment of the operator's own wording # read from the code, and row 0 is the same shape ACCEPTED. # +# THE MATRIX IS THE OPERATOR'S ADMITTED SET, and this case reads it three ways, in +# this order, and the order is the point: +# +# 1. ON THE CLUSTER AS IT STANDS, before anything here touches it. The platforms the +# cluster's own InstanceTypes report are read, and an unadmitted one is refused +# naming the platform beside the router and the engine it was checked against, and +# the admitted rows a user can compare against. This phase runs FIRST because the +# injected CPU vendor label of the next phase adds a manufacturer the cluster does +# not have, and this phase must read what the cluster really holds. +# 2. ON THE PLATFORM THE CLUSTER DOES NOT HAVE. A CPU vendor label is injected, so +# the CPU-only pool reports it, and the whole refusal matrix becomes reachable +# without a single accelerator: the wrong router, the wrong engine, the +# shared-claim rules and the mixed-platform rule. The rows read what that platform +# admits (SGLang over llm-d-router and over sglang-gateway) and what it refuses. +# 3. ON THE NETWORK LEG. The same injected label carries the NVIDIA vendor instead, +# so the rows only the NVIDIA and AMD platform admits are reachable too. +# # THE SHAPE RULES RUN BEFORE ANYTHING READS THE CLUSTER, so a role set that misses a # sibling kind, names a router-less shape, carries a deployment-wide cache, or # replaces a role's command is refused from the submitted object alone. THE MATRIX @@ -30,7 +48,8 @@ # Environment: Any cluster with a materialized scheduling chain (run case-1 first) and at least one # Ready, schedulable worker node that is not a control-plane node. NO accelerator and # no engine image: nothing here schedules or runs a replica. The gpustack-settings -# Secret must exist in the system namespace (E2E_SYSTEM_NS, default gpustack-system). +# Secret must exist in the system namespace (E2E_SYSTEM_NS, default gpustack-system), +# and jq must be on PATH. # # AUTO-SKIP: the matrix rows need a derived InstanceType reporting the injected # platform. The label rides a NodeFeature object, and NFD owns the vendor label this @@ -45,10 +64,14 @@ # # Inputs: MOCKED: one node label (the CPU vendor) injected through a NodeFeature object and # one settings key (CPU-manufacturer awareness) flipped on — both restored by the -# trap. REAL: every manifest goes through `kubectl apply --dry-run=server`, so the -# schema and the webhook answer exactly as they would for a real create. +# trap. The label carries the second vendor in the last phase, so the same object +# serves both. REAL: every manifest goes through `kubectl apply --dry-run=server`, so +# the schema and the webhook answer exactly as they would for a real create. # -# Expected: - the full encoder/prefill/decode shape on the injected platform, with the two +# Expected: - the platform the cluster holds is read as it stands, and an unadmitted platform it +# holds is refused naming the platform, the router and the engine, beside the +# admitted rows; +# - the full encoder/prefill/decode shape on the injected platform, with the two # shared-claim mounts, is ACCEPTED; # - each rule below refuses with its own wording: a missing sibling kind (each of # prefill and decode), a Server role beside the encoder, no router, a deployment- @@ -57,201 +80,74 @@ # disagreeing mount paths, an encoder or a prefiller without its claim, a router # that does not admit the injected platform for an encoder role, an engine a router # does not pair with, and a platform the matrix does not admit; -# - two roles on two different platforms are refused as a mixed deployment. +# - two roles on two different platforms are refused as a mixed deployment; +# - what the injected platform admits is ACCEPTED too: llm-d-router and sglang-gateway +# over SGLang, mounting no claim, and the pairing the matrix still refuses is +# refused naming BOTH halves and the admitted rows; +# - the network leg is read on the NVIDIA platform: vllm-router with vLLM and with an +# encoder role is ACCEPTED, vllm-router with SGLang is refused naming the pairing, +# a user-supplied --epd-config is refused as operator-owned, and vllm-router with +# no encoder role is admitted. # # Cleanup: The trap deletes the NodeFeature object, restores the settings key to its exact # original value (removing it if it was absent) and waits for the chain to settle back: -# the collapsed CPU pool returns only once the setting cache flips and the pool +# the derived CPU pool returns only once the setting cache flips and the pool # re-derives. Idempotent, runs on pass AND fail, safe to re-run. No ModelDeployment is # ever persisted: every apply is a server-side dry run. # set -uo pipefail -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: case-116.sh }" SYSTEM_NS="${E2E_SYSTEM_NS:-gpustack-system}" -SETTING=instance-type-aware-cpu-manufacturer # The vendor string becomes the platform after lowercasing; "Ascend" selects the shared-directory # row, which is the one whose claim rules this case refuses against. VENDOR="${E2E_C116_VENDOR:-Ascend}" PLATFORM="$(printf '%s' "$VENDOR" | tr '[:upper:]' '[:lower:]')" -NF="case116-cpu-vendor" +# The last phase reads the matrix on the network leg, which the NVIDIA rows are admitted on. +VENDOR_NVIDIA="${E2E_C123_VENDOR:-NVIDIA}" +NF=case116-cpu-vendor MD=case116-probe IMAGE="${E2E_MD_IMAGE:-registry.k8s.io/pause:3.10}" SETTLE_ROUNDS="${E2E_C116_SETTLE:-60}" # 60 x 5s = 5 min for the settings cache and re-derivation +ROUTER_NAME=llm-d-router +ENGINE=vLLM +ROUTER_EXTRA_ARGS="" +KVCACHE_BLOCK="" -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } # shellcheck source=/dev/null -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" - -# ---------------------------------------------------------------- the fixture, and its restore - -# One Ready, schedulable worker node (not a control-plane one): the vendor label lands there and -# that node's CPU becomes the pool whose manufacturer the case controls. -pick_worker_node() { - kubectl get nodes -o json 2>/dev/null | python3 -c ' -import json,sys -for n in json.load(sys.stdin).get("items",[]): - cs=n.get("status",{}).get("conditions",[]) - if not any(c.get("type")=="Ready" and c.get("status")=="True" for c in cs): - continue - if n.get("spec",{}).get("unschedulable"): - continue - # A CPU-vendor probe only reads back on a node that feeds the CPU-only catalog. An - # acceleratable node feeds the GPU InstanceType instead, whose manufacturer is the - # accelerator vendor, so an injected cpu-model.vendor_id never surfaces here and the - # platform-type poll below times out. Skip those nodes, exactly as case-18 reads the - # CPU pool (acceleratable=false) rather than the accelerator one. - if n.get("metadata",{}).get("labels",{}).get("feature.gpustack.ai/acceleratable") == "true": - continue - ts=n.get("spec",{}).get("taints",[]) - if any(str(t.get("key","")).startswith("node-role.kubernetes.io/") for t in ts): - continue - print(n["metadata"]["name"]) - break -' -} - -# The InstanceType that reports the injected platform: derived from the pool the labeled node -# feeds, which is the whole fixture. Not terminating and not Inactive, for the reasons case-51 -# gives: a departing type is refused at admission for a reason no row here is about. -epd_platform_type() { - kubectl get instancetypes.worker.gpustack.ai \ - -o jsonpath='{range .items[*]}{.metadata.name}|{.metadata.deletionTimestamp}|{.spec.inactive}|{.status.detail.manufacturer}{"\n"}{end}' \ - 2>/dev/null \ - | while IFS='|' read -r name deleting inactive manufacturer; do - [ -n "$name" ] || continue - [ -z "$deleting" ] || continue - [ "$inactive" = true ] && continue - if [ "$manufacturer" = "$PLATFORM" ]; then - echo "$name" - break - fi - done -} - -# Any InstanceType a deployment can name, for the shape rows: they fire before the platform is -# read, so the type's own platform does not matter to them. Same exclusions as case-51. -usable_instance_type() { - kubectl get instancetypes.worker.gpustack.ai \ - -o jsonpath='{range .items[*]}{.metadata.name}|{.metadata.deletionTimestamp}|{.spec.inactive}|{.spec.acceleratable}|{.status.accelerator.capacity}{"\n"}{end}' \ - 2>/dev/null \ - | while IFS='|' read -r name deleting inactive acceleratable capacity; do - [ -n "$name" ] || continue - [ -z "$deleting" ] || continue - [ "$inactive" = true ] && continue - if [ "$acceleratable" = true ]; then - case "$capacity" in "" | 0) continue ;; esac - fi - echo "$name" - break - done -} +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_md-lib.sh" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_epd-lib.sh" -# The Setting is read through the worker's thirty-second cache, so the flip and the restore are -# both absorbed by the type poll below rather than by a blind sleep. The value rides the Secret -# base64-encoded; the original is kept encoded and patched back verbatim (case-106's mechanic). -ORIG_SETTING="$(kubectl -n "$SYSTEM_NS" get secret gpustack-settings -o jsonpath="{.data.${SETTING}}" 2>/dev/null)" -set_awareness() { # true|false - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"${SETTING}\":\"$(printf '%s' "$1" | base64)\"}}" >/dev/null 2>&1 -} -restore_setting() { - local v=null - [ -n "$ORIG_SETTING" ] && v="\"${ORIG_SETTING}\"" - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"${SETTING}\":${v}}}" >/dev/null 2>&1 -} +command -v jq >/dev/null 2>&1 || { echo "[case-116] jq is required." >&2; exit 2; } -# Restoring the value is not the end of the restore: the worker reads it through a thirty-second -# cache, and the type derived from it outlives the patch by that cache plus one reconcile. Wait for -# the derived type to go, so the next case in a serial run does not list an aware type this run left -# behind (case-18's wait_generic, on this same fixture). A cluster that started with the setting -# already on keeps its derived type, so there is nothing to wait for and the wait is skipped. -settle_back() { - [ "$(printf '%s' "$ORIG_SETTING" | base64 -d 2>/dev/null)" = "true" ] && return 0 - for _ in $(seq 1 "$SETTLE_ROUNDS"); do - [ -z "$(epd_platform_type)" ] && return 0 - sleep 5 - done - return 0 -} +# ---------------------------------------------------------------- the fixture, and its restore -INJECTED=false +# The label rides a NodeFeature object (case-4's mechanism): the object stays and keeps re-asserting +# the label, where a bare `kubectl label` is a cpuid-owned value NFD reverts on its next pass. The +# same object carries the second vendor in the last phase, so one delete restores both. The label, +# the awareness setting, the type it waits for and the undo of all three are the library's. cleanup() { - if [ "$INJECTED" = true ]; then - kubectl -n "$NS" delete nodefeatures.nfd.k8s-sigs.io "$NF" --ignore-not-found >/dev/null 2>&1 - restore_setting - settle_back - fi + epd_teardown_cpu_vendor_fixture "$NS" "$NF" "$SETTLE_ROUNDS" "$SYSTEM_NS" } trap cleanup EXIT -PLATFORM_TYPE="$(epd_platform_type)" -if [ -z "$PLATFORM_TYPE" ]; then - NODE="$(pick_worker_node)" - if [ -z "$NODE" ]; then - echo "[case-116] no Ready schedulable worker node; this case cannot place its vendor label" >&2 - exit 2 - fi - - echo "[case-116] injecting CPU vendor '${VENDOR}' on node ${NODE} and turning CPU-manufacturer awareness on" - INJECTED=true - - # The label rides a NodeFeature object (case-4's mechanism): the object stays and keeps re-asserting - # the label, where a bare `kubectl label` is a cpuid-owned value NFD reverts on its next pass. - kubectl apply -f - >/dev/null 2>&1 <&2 +# The rows the earlier phases recorded are printed before an environment failure ends the case, so +# a skip recorded before the failure is not lost with it. +abort() { # message + echo "[case-116] $1" >&2 + print_rows exit 2 -fi - -# The shape rows embed whichever type exists: their rules fire before the platform is read. The -# matrix rows below name the platform type directly and are skipped without it. -IT="${PLATFORM_TYPE:-$FALLBACK_TYPE}" - -if [ -n "$PLATFORM_TYPE" ]; then - record PASS "a CPU-only pool reports an admitted platform" \ - "instancetype ${PLATFORM_TYPE} reports manufacturer '${PLATFORM}' from the node labels" -else - record SKIP "a CPU-only pool reports an admitted platform" \ - "no InstanceType reported manufacturer '${PLATFORM}' within $((SETTLE_ROUNDS * 5))s, so every matrix row below skips: NFD did not keep the injected cpu-model.vendor_id, or the pool never derived. The shape rows still run: they are refused before the platform is read" -fi +} # ---------------------------------------------------------------- the manifests and the rows -# ENGINE and ROUTER_NAME are the one-field dials the matrix rows turn; each is reset -# right after the row that turns it, because an assignment prefixed to a function call persists in -# bash after the call returns and would silently move every later row off the baseline. +# ENGINE, ROUTER_NAME, ROUTER_EXTRA_ARGS and KVCACHE_BLOCK are the dials the rows turn; each is +# reset right after the row that turns it, because an assignment prefixed to a function call +# persists in bash after the call returns and would silently move every later row off the baseline. manifest() { # roles-block cat <&1 | tr '\n' ' ')" - if [ -z "$out" ]; then - record FAIL "$check" "the apply produced no output at all, so nothing was refused" - return 0 + obj="$(md_dryrun_refuses "$want" "$@" < <(manifest "$roles"))" || status=1 + if [ "$status" -eq 0 ]; then + record PASS "$check" "$obj" + else + record FAIL "$check" "$obj" fi - for w in "$want" "$@"; do - case "$out" in - *"$w"*) ;; - *) record FAIL "$check" "refused with the wrong message, or accepted (missing '${w}'): ${out:0:220}"; return 0 ;; - esac - done - record PASS "$check" "refused naming: ${want}${*:+ + $*}" return 0 } @@ -337,20 +297,197 @@ refuses() { # at all is the shape this whole rule set exists to guard, so the baseline row proves the refusals # below are each their own rule rather than one blanket rejection. accepts() { - local check="$1" roles="$2" out - out="$(manifest "$roles" | kubectl apply --dry-run=server -f - 2>&1 | tr '\n' ' ')" - case "$out" in - *"created (server dry run)"* | *"configured (server dry run)"*) - record PASS "$check" "accepted by both the schema and the webhook" ;; - *) record FAIL "$check" "refused: ${out:0:220}" ;; - esac + local check="$1" roles="$2" obj status=0 + obj="$(md_dryrun_accepts < <(manifest "$roles"))" || status=1 + if [ "$status" -eq 0 ]; then + record PASS "$check" "$obj" + else + record FAIL "$check" "$obj" + fi return 0 } +# ---------------------------------------------------------------- phase 1: the cluster as it stands + +# This phase runs BEFORE anything here mutates the cluster. The unadmitted-platform rows read the +# manufacturers the cluster's own InstanceTypes report, and the injected CPU vendor label of the +# next phase adds a manufacturer the cluster does not have, so the reading has to happen first. +# +# TWO TYPES ARE SOUGHT, and neither is invented. The platform row needs a type reporting a +# manufacturer the matrix does NOT cover, and the baseline needs one that reports one it DOES. +# On a cluster holding any unadmitted accelerator the first is found and the rows run; on a +# cluster whose only accelerated types are the three admitted platforms the first is empty, the +# rows skip naming what was seen, and the baseline still runs against the second. +# +# THE UNADMITTED MANUFACTURERS ARE THE OPERATOR'S OWN LIST MINUS THE THREE IT ADMITS, so every +# accelerator this project supports is covered without the case having to know which of them is +# common. Three of the nine declared are admitted here and are therefore excluded; the general CPU +# fallback is not a manufacturer of an accelerator and is not in the list either. +UNADMITTED_PLATFORMS="cambricon hygon iluvatar metax mthreads thead" + +OTHER_IT="" +OTHER_MANUFACTURER="" +for m in $UNADMITTED_PLATFORMS; do + OTHER_IT="$(epd_platform_type "$m")" + if [ -n "$OTHER_IT" ]; then + OTHER_MANUFACTURER="$m" + break + fi +done + +# The baseline is ANY admitted platform, tried in the order the matrix admitted them, and the +# manufacturer is kept beside the type because the shape differs by row: ascend carries the +# shared claim, nvidia and amd carry the network leg and forbid the mount. +ADMITTED_IT="" +ADMITTED_MFG="" +for m in ascend nvidia amd; do + ADMITTED_IT="$(epd_platform_type "$m")" + if [ -n "$ADMITTED_IT" ]; then + ADMITTED_MFG="$m" + break + fi +done + +SEEN="$(kubectl get instancetypes.worker.gpustack.ai \ + -o jsonpath='{range .items[*]}{.metadata.name}={.status.detail.manufacturer}{" "}{end}' 2>/dev/null)" + +if [ -z "$OTHER_IT" ] && [ -z "$ADMITTED_IT" ]; then + record SKIP "the platform could be read" \ + "no usable InstanceType reported any manufacturer (the cluster holds: ${SEEN:-}), so there is no platform to submit and no baseline to compare against" +else + # Three roles being accepted at all is the shape this whole rule set exists to guard, so the + # baseline proves the refusals below are each their own rule rather than one blanket rejection. + # The shape is whichever admitted platform this cluster has: the ascend row carries the two + # shared-claim mounts, and the nvidia or amd rows carry none, because the network connector + # forbids the shared mount on its row. + if [ -n "$ADMITTED_IT" ]; then + accepts "an admitted combination is ACCEPTED by the same instrument" \ + "$(platform_baseline_roles "$ADMITTED_IT" "$ADMITTED_MFG")" + else + record SKIP "an admitted combination is ACCEPTED by the same instrument" \ + "no usable InstanceType reports an admitted platform (the cluster holds: ${SEEN:-}), so there is no accepted shape to compare the refusals against" + fi + + if [ -n "$OTHER_IT" ]; then + # THE HEADLINE OF THIS PHASE. The matrix gap is read router-first, then engine, then platform, + # and it returns ONE message: llm-d-router and vLLM are both admitted here, so the gap that + # fires is the platform one, which names the platform AND quotes the router and engine it + # ACCEPTED beside it. The word "and" before the engine is the fragment that proves this is the + # platform arm and not the engine arm above it — the two messages read alike and only that word + # tells them apart. The admitted rows come from the same sentence, appended by the rule that + # fires it. + refuses "a platform the matrix does not admit is refused, all three named" \ + "is not admitted for encoder roles under router" \ + "$(platform_baseline_roles "$OTHER_IT" "$OTHER_MANUFACTURER")" \ + 'and engine' \ + "the admitted rows are" + + # THE SAME PLATFORM WITH A DIFFERENT ENGINE, which reaches a different arm and stops earlier: + # the engine is checked before the platform, so this refusal is the engine message and never + # reaches the platform row above. The pair is what tells a user which of the two to change, and + # either alone leaves the other looking fine. + ENGINE=SGLang refuses "an unadmitted platform is refused naming the engine too" \ + 'is not admitted with engine "SGLang"' \ + "$(platform_baseline_roles "$OTHER_IT" "$OTHER_MANUFACTURER")" + ENGINE="" + + # TWO PLATFORMS IN ONE DEPLOYMENT answer a DIFFERENT rule, and the difference is the point: the + # matrix lookup is per platform, so the mixed-platform rule runs first and names both + # manufacturers instead of the matrix. A case that only submitted one unadmitted platform would + # never tell the two refusals apart. + # + # THE MINIMUM THAT REACHES IT IS ONE ROLE OF EACH KIND ON TWO TYPES. Six roles would reach it + # too, but a second role of a kind trips the at-most-one-role-of-each-kind rule alongside it, and + # this case states elsewhere that a manifest which trips a rule its row is not about is a row + # that has stopped being about its rule. + if [ -n "$ADMITTED_IT" ]; then + refuses "two platforms in one encoder deployment are refused by the mixing rule" \ + "more than one accelerator platform" \ + "$(platform_baseline_roles "$ADMITTED_IT" "$ADMITTED_MFG") + - name: decode + kind: Decode + instanceType: ${OTHER_IT} + image: ${IMAGE}" + else + record SKIP "two platforms in one encoder deployment are refused by the mixing rule" \ + "an unadmitted platform exists (${OTHER_MANUFACTURER}) but no usable InstanceType reports an admitted one, so there is nothing to mix it with" + fi + else + record SKIP "a platform the matrix does not admit is refused, all three named" \ + "no usable InstanceType reports any of the unadmitted manufacturers this operator declares (${UNADMITTED_PLATFORMS// /, }; the cluster holds: ${SEEN:-}), so there is no unadmitted platform to name" + record SKIP "an unadmitted platform is refused naming the engine too" \ + "no usable InstanceType reports any of the unadmitted manufacturers this operator declares (${UNADMITTED_PLATFORMS// /, }; the cluster holds: ${SEEN:-}), so there is no unadmitted platform to name" + record SKIP "two platforms in one encoder deployment are refused by the mixing rule" \ + "no usable InstanceType reports any of the unadmitted manufacturers this operator declares (${UNADMITTED_PLATFORMS// /, }; the cluster holds: ${SEEN:-}), so there is no second manufacturer to mix with" + fi + + # ------------------------------------------------- why the functional half lives elsewhere + # + # Stated as a row rather than left to the header, because a case that reads as a refusal-only + # case with no functional rows is otherwise indistinguishable from one that forgot to write them. + # The admitted platforms' functional readings are their own cases, and no unadmitted platform has + # an object to run until a live proof of concept admits its row. + record SKIP "an unadmitted platform serves a multimodal request through the router" \ + "the matrix admits ascend, nvidia and amd only, so any other platform has no object to run. The admitted platforms' functional readings are case 120 (the NVIDIA and AMD network leg) and case 121 (the Ascend shared-directory leg); this case asserts the refusal for every platform that has no row, and nothing else" +fi + +# ---------------------------------------------------------------- phase 2: the shared-directory leg + +# The pool needs the settings cache to expire, the node flavor to re-derive and the type to be +# authored, so this poll IS the wait. A timeout is a skip, not a failure: NFD may decline to let an +# injected object override a label its cpuid source reports, and that is an environment answer. +NODE="" +PLATFORM_TYPE="$(epd_platform_type "$PLATFORM")" +if [ -z "$PLATFORM_TYPE" ]; then + NODE="$(epd_pick_worker_node)" + if [ -z "$NODE" ]; then + abort "no Ready schedulable worker node; this case cannot place its vendor label" + fi + + echo "[case-116] injecting CPU vendor '${VENDOR}' on node ${NODE} and turning CPU-manufacturer awareness on" + epd_setup_cpu_vendor_fixture "$VENDOR" "$NS" "$NF" "$SETTLE_ROUNDS" "$SYSTEM_NS" "$NODE" + PLATFORM_TYPE="$EPD_FIXTURE_TYPE" +fi + +# Any InstanceType a deployment can name, for the shape rows: they fire before the platform is +# read, so the type's own platform does not matter to them. The exclusions and the order live in +# _md-lib.sh, and the first name it prints is the one the case used to pick. +FALLBACK_TYPE="$(md_usable_instance_types | sed -n '1p')" +if [ -z "$PLATFORM_TYPE" ] && [ -z "$FALLBACK_TYPE" ]; then + abort "no InstanceType in the cluster; run case-1 first" +fi + +# The shape rows embed whichever type exists: their rules fire before the platform is read. The +# matrix rows below name the platform type directly and are skipped without it. +IT="${PLATFORM_TYPE:-$FALLBACK_TYPE}" + +if [ -n "$PLATFORM_TYPE" ]; then + record PASS "a CPU-only pool reports an admitted platform" \ + "instancetype ${PLATFORM_TYPE} reports manufacturer '${PLATFORM}' from the node labels" +else + record SKIP "a CPU-only pool reports an admitted platform" \ + "no InstanceType reported manufacturer '${PLATFORM}' within $((SETTLE_ROUNDS * 5))s, so every matrix row below skips: NFD did not keep the injected cpu-model.vendor_id, or the pool never derived. The shape rows still run: they are refused before the platform is read" +fi + +# The same platform, read the way the rows below that hang on it read it: the pool reports the +# target platform, whether the cluster already had that platform or the injected label built it. +if [ "$EPD_FIXTURE_INJECTED" = false ] && [ -n "$PLATFORM_TYPE" ]; then + record PASS "an active pool reports the target platform" \ + "instancetype ${PLATFORM_TYPE} reports manufacturer '${PLATFORM}'" +else + if [ -z "$PLATFORM_TYPE" ]; then + record SKIP "the platform rows could be read" \ + "no InstanceType reported manufacturer '${PLATFORM}' within $((SETTLE_ROUNDS * 5))s: NFD did not keep the injected cpu-model.vendor_id, or the pool never derived" + else + record PASS "an active pool reports the target platform" \ + "instancetype ${PLATFORM_TYPE} reports manufacturer '${PLATFORM}' from the injected vendor label" + fi +fi + # ---------------------------------------------------------------- row 0: the accepted baseline if [ -n "$PLATFORM_TYPE" ]; then - accepts "the full encoder shape on the admitted platform is ACCEPTED" "$(baseline_roles)" + accepts "the full encoder shape on the admitted platform is ACCEPTED" "$(shared_claim_roles)" else record SKIP "the full encoder shape on the admitted platform is ACCEPTED" \ "no platform InstanceType exists, so there is no shape to accept and every refusal below would pass for the wrong reason" @@ -393,7 +530,7 @@ spec: model: name: Qwen/Qwen2.5-0.5B-Instruct roles: -$(baseline_roles) +$(shared_claim_roles) YAML )" case "$out" in @@ -408,7 +545,7 @@ esac KVCACHE_BLOCK="$(printf ' kvCache:\n poolRef:\n name: case116-no-such-binding\n')" \ refuses "a deployment-wide kvCache is refused on the encoder shape" \ "spec.kvCache is refused" \ - "$(baseline_roles)" + "$(shared_claim_roles)" KVCACHE_BLOCK="" refuses "a role command is refused on the encoder shape" \ @@ -475,7 +612,7 @@ if [ -n "$PLATFORM_TYPE" ]; then # router and engine. ROUTER_NAME=vllm-router refuses "a router that does not admit this platform for an encoder role is refused" \ 'platform "ascend" is not admitted for encoder roles under router "vllm-router" and engine "vLLM"' \ - "$(baseline_roles)" + "$(shared_claim_roles)" ROUTER_NAME="" # sglang-gateway drives the encoder leg on SGLang alone, so pairing it with vLLM is the pair the @@ -483,7 +620,7 @@ if [ -n "$PLATFORM_TYPE" ]; then # vLLM and the refusal has to name the pairing rather than either half on its own. ROUTER_NAME=sglang-gateway ENGINE=vLLM refuses "an engine this router does not pair with the encoder is refused" \ 'router "sglang-gateway" is not admitted with engine "vLLM"' \ - "$(baseline_roles)" + "$(shared_claim_roles)" ROUTER_NAME="" ENGINE="" @@ -531,7 +668,7 @@ fi # one, so an NVIDIA cluster (whose own types report nvidia) still runs it against its real types. # The unadmitted row needs a manufacturer NO admitted row covers -- neither the injected one, nor # amd, nor nvidia -- so that same cluster skips it cleanly instead of reading one of its own -# admitted platforms as a refusal case-124 already covers directly. With awareness on, the +# admitted platforms as a refusal the first phase already covers directly. With awareness on, the # cluster's own CPU types report their real vendor, so any such type serves; an all-AMD, # all-NVIDIA or all-unknown cluster may have none, and the row says so rather than inventing one. if [ -n "$PLATFORM_TYPE" ]; then @@ -588,7 +725,7 @@ if [ -n "$PLATFORM_TYPE" ]; then image: ${IMAGE}" else record SKIP "a platform the matrix does not admit is refused, all three named" \ - "every usable InstanceType reports an admitted platform (ascend, amd, nvidia) or none, so there is no unadmitted platform to name; case-124 covers every unadmitted platform" + "every usable InstanceType reports an admitted platform (ascend, amd, nvidia) or none, so there is no unadmitted platform to name; the first phase of this case covers every unadmitted platform" fi # Two platforms in one deployment, both claim mounts well formed so only the mixing answers. @@ -616,6 +753,123 @@ else "needs the platform InstanceType this run did not build; see the first row" fi +# ---------------------------------------------------------------- the rows the injected platform admits + +if [ -n "$PLATFORM_TYPE" ]; then + # Three roles being accepted at all is the shape this whole rule set exists to guard, so the + # baseline proves the refusals below are each their own rule rather than one blanket rejection. + accepts "the admitted combination is ACCEPTED by the same instrument" \ + "$(platform_baseline_roles "$IT" "$PLATFORM")" +else + record SKIP "the admitted combination is ACCEPTED by the same instrument" \ + "no platform InstanceType exists, so there is no shape to accept and every refusal below would pass for the wrong reason" +fi + +if [ -n "$PLATFORM_TYPE" ]; then + # The admitted SGLang rows, read through the same instrument as the vLLM baseline above. Both + # routers are accepted on SGLang with no claim mounted, because an address leg is what carries the + # encoder's output and a connector there is not a thing a user attaches. + ENGINE=SGLang accepts "llm-d-router with SGLang is ACCEPTED" \ + "$(network_roles "$IT")" + ENGINE=SGLang ROUTER_NAME=sglang-gateway \ + accepts "sglang-gateway with SGLang is ACCEPTED" \ + "$(network_roles "$IT")" + ROUTER_NAME=llm-d-router + + # The pairing the matrix still refuses. sglang-gateway drives the encoder leg on SGLang alone, so + # vLLM under it names both halves in one message along with the admitted rows. + ROUTER_NAME=sglang-gateway ENGINE=vLLM refuses "sglang-gateway with vLLM is refused naming the pairing" \ + 'router "sglang-gateway" is not admitted with engine "vLLM"' \ + "$(platform_baseline_roles "$IT" "$PLATFORM")" \ + 'the admitted rows are' + ROUTER_NAME=llm-d-router + ENGINE=vLLM +else + for label in \ + "llm-d-router with SGLang is ACCEPTED" \ + "sglang-gateway with SGLang is ACCEPTED" \ + "sglang-gateway with vLLM is refused naming the pairing"; do + record SKIP "$label" "needs the platform InstanceType this run did not build; see the baseline row" + done +fi + +# ---------------------------------------------------------------- phase 3: the network leg + +# The NVIDIA rows need a platform the matrix admits on the network leg. A cluster that already +# carries one is read directly and mutates nothing; a CPU-only cluster gets one by carrying the +# NVIDIA vendor on the injected label instead of the Ascend one, so the same object serves both +# phases and one delete restores both. +PHASE3_IT="" +PHASE3_PLATFORM="" +for m in nvidia amd; do + PHASE3_IT="$(epd_platform_type "$m")" + if [ -n "$PHASE3_IT" ]; then + PHASE3_PLATFORM="$m" + break + fi +done + +if [ -n "$PHASE3_IT" ]; then + record PASS "an admitted platform InstanceType is available" \ + "instancetype ${PHASE3_IT} reports manufacturer '${PHASE3_PLATFORM}'" +else + if [ -z "$NODE" ]; then + NODE="$(epd_pick_worker_node)" + fi + if [ -n "$NODE" ]; then + echo "[case-116] carrying CPU vendor '${VENDOR_NVIDIA}' on node ${NODE} for the network-leg rows" + epd_setup_cpu_vendor_fixture "$VENDOR_NVIDIA" "$NS" "$NF" "$SETTLE_ROUNDS" "$SYSTEM_NS" "$NODE" nvidia + PHASE3_IT="$EPD_FIXTURE_TYPE" + fi + if [ -n "$PHASE3_IT" ]; then + PHASE3_PLATFORM=nvidia + record PASS "a CPU-only pool reports the target platform" \ + "instancetype ${PHASE3_IT} reports manufacturer '${PHASE3_PLATFORM}' from the injected vendor label" + else + record SKIP "the platform rows could be read" \ + "no InstanceType reported manufacturer 'nvidia' or 'amd' and no CPU-vendor fixture could be derived; every row below names that type in its manifest, so none of them ran" + fi +fi + +if [ -n "$PHASE3_IT" ]; then + # The platform the rows below name is the one this phase read. + PLATFORM="$PHASE3_PLATFORM" + + # Row 0: the admitted vllm-router + vLLM EPD shape on nvidia/amd is accepted. + ROUTER_NAME=vllm-router accepts \ + "vllm-router + vLLM with an encoder role is ACCEPTED on ${PLATFORM}" \ + "$(network_roles "$PHASE3_IT")" + + # Row 2: vllm-router paired with SGLang is refused naming the router and engine. + ROUTER_NAME=vllm-router ENGINE=SGLang refuses \ + "vllm-router paired with SGLang is refused" \ + 'router "vllm-router" is not admitted with engine "SGLang"' \ + "$(network_roles "$PHASE3_IT")" \ + 'the admitted rows are' + ENGINE=vLLM + + # Row 3: --epd-config is owned by the operator for vllm-router and refused in extraArgs. + ROUTER_EXTRA_ARGS="$(printf ' extraArgs:\n - --epd-config={"encoder_urls":[]}\n')" \ + ROUTER_NAME=vllm-router refuses \ + "user-supplied --epd-config on vllm-router is refused as operator-owned" \ + '"--epd-config" is set by the operator for router "vllm-router"' \ + "$(network_roles "$PHASE3_IT")" + ROUTER_EXTRA_ARGS="" + + # Row 4: vllm-router with no Encode role (plain prefill/decode) remains accepted. + ROUTER_NAME=vllm-router accepts \ + "vllm-router with NO encoder role is admitted" \ + "$(pair_roles "$PHASE3_IT")" +else + for label in \ + "vllm-router + vLLM with an encoder role is ACCEPTED" \ + "vllm-router paired with SGLang is refused" \ + "user-supplied --epd-config on vllm-router is refused as operator-owned" \ + "vllm-router with NO encoder role is admitted"; do + record SKIP "$label" "needs an nvidia or amd InstanceType this run did not have; see the first row" + done +fi + # Results. print_rows [ "$FAILS" -eq 0 ] || { echo "[case-116] ${FAILS} check(s) FAILED"; exit 1; } diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-117.sh b/.agents/skills/gpustack-operator-e2e/cases/case-117.sh index 581b18571..18f4caada 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/case-117.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-117.sh @@ -1,180 +1,163 @@ #!/usr/bin/env bash # -# CASE 117 — The encoder's own Service exists, and the front door never points at it +# CASE 117 — An encoder deployment renders one Service per role, a router picker document that +# carries the encoder stage, and one shared claim under the encoder and the prefiller # (MUTATING, self-recovering) # AUTO-SKIPS without an InstanceType that reports an admitted accelerator platform, which this -# case builds by injecting a CPU vendor label. +# case builds by injecting a CPU vendor label; its witness row additionally skips, naming the +# image, when the witness image cannot be pulled. # # case-117.sh # -# Goal: A deployment with an encoder role gets one Service per role, and the address a -# caller dials never resolves to the encoder. The encoder answers no request of its -# own: it turns media into embeddings for a prefiller, so a front door pointed at one -# sends traffic to a process that serves none of it — with endpoints that resolve and -# answer health checks, which is what makes that failure quiet. So the case asserts -# the three per-role Services by NAME, each selecting only its own role's Pods (the -# encoder's may select none once the operator withdraws a placeholder that never -# becomes Ready), and -# the deployment-wide Service selecting the prefiller while the encoder sits FIRST in -# the roles list — the position where a "front the first role" implementation would -# pick it. +# Goal: THREE READINGS OF ONE RENDER, all on the same three-role deployment. The operator +# answers three different questions about one encoder deployment, and each answer has +# a failure that is quiet, so the case reads all three. +# +# THE ADDRESSES. The encoder answers no request of its own: it turns media into +# embeddings for a prefiller, so a front door pointed at one sends traffic to a +# process that serves none of it — with endpoints that resolve and answer health +# checks, which is what makes that failure quiet. So the case asserts the three +# per-role Services by NAME, each selecting only its own role's Pods (the encoder's +# may select none once the operator withdraws a placeholder that never becomes +# Ready), and the deployment-wide Service selecting the prefiller while the encoder +# sits FIRST in the roles list — the position where a "front the first role" +# implementation would pick it. # # THE SELECTOR IS THE OBSERVABLE, not the Service's existence alone. A Service whose # selector matched every role of the deployment would exist under the same name and # pass an existence check, so each row resolves the Service's own selector against the # live Pods and reads back which role's Pods it fronts. # +# THE PICKER DOCUMENT. llm-d-router is the only router the admitted rows name, so +# what its picker document says about an encoder is the part of the router that +# carries this whole feature. The document must carry an ENCODE FILTER (a +# label-selector-filter selecting role-kind=encode), an ENCODE PROFILE that names that +# filter and NOTHING ELSE — no queue scorer, no KV-cache scorer, no prefix scorer, +# because an encoder keeps no KV cache: the scorers read gauges it never exports and +# the prefix scorer asks about token overlap that does not exist before a request has +# tokens — and the MULTIMODAL DECIDER that the disagg-profile-handler routes BOTH +# "prefill" and "encode" at. A control deployment with no encoder role renders none of +# the three, which is what makes them the encoder's facts and not the router's +# defaults. +# +# THE SHARED CLAIM. On the platform whose encoder leg runs through a mounted +# directory rather than the network, the encoder writes its embeddings into a claim +# and the prefiller reads them out of the same one. Reading only the rendered specs +# proves the operator points both roles at one claim name; it cannot tell a claim +# that really shares bytes from two claims sharing a name. So the rows assert the +# volumes and mounts the two role Pods render, and then a pair of case-owned witness +# Pods — pinned to one node, mounting the same claim — writes through one mount and +# reads back through the other. +# # Environment: Any cluster with a materialized scheduling chain (run case-1 first) and at least one # Ready, schedulable worker node that is not a control-plane node. NO accelerator: the -# encoder shape reaches admission through the platform the injected CPU vendor builds. -# The gpustack-settings Secret must exist in the system namespace (E2E_SYSTEM_NS, -# default gpustack-system). The replicas are placeholders that never serve (the -# explicit pause image carries no engine), and the router image may be unpullable: -# nothing here reads the router beyond the endpoint string. +# encoder shape reaches admission through the platform an injected CPU vendor label +# builds ("Ascend" — the admitted row whose connector is a mounted directory rather +# than a network leg). The gpustack-settings Secret must exist in the system namespace +# (E2E_SYSTEM_NS, default gpustack-system). The case creates its own PV and PVC (a +# hostPath volume under /tmp on the labeled node, manually bound), so no StorageClass +# is needed. The runner needs jq and pyyaml to read the router document. # -# AUTO-SKIP: the whole subject needs the encoder shape ADMITTED, which needs a -# platform InstanceType. If the vendor label does not take (NFD owns cpu-model.vendor_id -# and may decline an injected override) or the pool never derives, the case prints one -# SKIP row naming that and exits 0. +# AUTO-SKIP: the subject needs the encoder shape ADMITTED, which needs a platform +# InstanceType. If the vendor label does not take (NFD owns cpu-model.vendor_id and +# may decline an injected override) or the pool never derives, the case prints SKIP +# rows naming that and exits 0. It also skips, naming the queue, when the pool's +# entrance LocalQueue is missing from the namespace: nothing is admitted and no +# document is ever rendered. The witness row additionally skips, naming the image, +# when the witness image cannot be pulled — a container nobody could start is a +# reading nobody took, not a verdict about the operator. # -# Existing platform: if an InstanceType already reports the manufacturer named by E2E_C117_VENDOR -# (an accelerator cluster, for example E2E_C117_VENDOR=Ascend), the case uses it and -# injects nothing, so the label, the setting and the settle wait are skipped. +# Existing platform: if an InstanceType already reports the manufacturer named by +# E2E_C117_VENDOR (an accelerator cluster, for example E2E_C117_VENDOR=Ascend), the +# case uses it and injects nothing, so the label, the setting and the settle wait +# are skipped. # -# Inputs: MOCKED: one node label (the CPU vendor, "AMD" — the platform whose admitted row -# needs no shared claim) injected through a NodeFeature object, and one settings key -# (CPU-manufacturer awareness) flipped on; both restored by the trap. REAL: one -# ModelDeployment with the encode role listed first, engine vLLM, router -# llm-d-router, one replica per role on the platform type. +# Inputs: MOCKED: one node label (the CPU vendor) injected through a NodeFeature object and +# one settings key (CPU-manufacturer awareness) flipped on — both restored by the +# trap. REAL: one PV, one PVC, one three-role ModelDeployment (encode first, engine +# vLLM, router llm-d-router, one replica per role on the platform type, the encode and +# prefill roles mounting the claim at one path), the control deployment the same +# router with no encoder role, and two one-shot witness Pods mounting the same claim. # -# Expected: - the deployment is created and its three role Pods exist (gated counts, they never -# need to run); +# Expected: - the deployment is admitted and created, and its three role Pods exist (gated +# counts, they never need to run); # - Services -encode, -prefill and -decode exist, # each selecting exactly its own role's Pod and nobody else's; # - the deployment-wide Service exists and selects the prefill Pod, NOT the encode # Pod, with the encode role first in the list; -# - status.roleSummary carries the ready counts in the fixed order E, P, D; -# - the RoleKindsReady condition accounts the roles (KindsNotReady while the pause -# placeholders fail the engine probes, AllKindsReady if something serves) rather -# than reporting NoRoleStatuses; -# - status.endpoint, when published, does not name the encoder; -# - the v1 ModelDeployment kubectl table carries the Endpoint column alongside Engine, -# Roles and Phase. +# - status.roleSummary carries the ready counts in the fixed order E, P, D, and the +# RoleKindsReady condition accounts the roles rather than reporting NoRoleStatuses; +# - status.endpoint, when published, does not name the encoder, and the v1 table +# carries the Endpoint column; +# - the router's ConfigMap carries an EndpointPickerConfig whose encode filter selects +# role-kind=encode, whose encode profile names that filter alone, whose multimodal +# decider the handler routes both prefill and encode at, and whose prefix-cache +# producer excludes the kinds that publish no events — while a deployment with no +# encoder role renders none of that; +# - the encoder Pod renders a volume naming the claim at the shared path and NOT +# read-only, the prefiller Pod renders the same claim at the same path, and the +# decoder Pod renders no volume naming the claim; +# - the PVC reaches Bound against the case's own PV, and a file written through the +# claim by one witness Pod is read back byte-for-byte by the other, proving one +# filesystem behind both mounts. # -# Cleanup: The trap deletes the deployment, releases any Workload still holding its replicas -# (Kueue keeps a finalizer on every Pod of a serving group), deletes the NodeFeature -# object, restores the settings key to its exact original value and waits for the chain -# to settle back: the derived type goes only once the setting cache flips and the pool -# re-derives. Idempotent, runs on pass AND fail, safe to re-run. +# Cleanup: The trap deletes the witness Pods and both deployments, releases any Workload still +# holding the replicas (Kueue keeps a finalizer on every Pod of a serving group), +# deletes the PVC and the PV, deletes the NodeFeature object, restores the settings +# key to its exact original value and waits for the chain to settle back: the +# derived type goes only once the setting cache flips and the pool re-derives. +# Idempotent, runs on pass AND fail, safe to re-run. # set -uo pipefail -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: case-117.sh }" SYSTEM_NS="${E2E_SYSTEM_NS:-gpustack-system}" -SETTING=instance-type-aware-cpu-manufacturer -VENDOR="${E2E_C117_VENDOR:-AMD}" +VENDOR="${E2E_C117_VENDOR:-Ascend}" PLATFORM="$(printf '%s' "$VENDOR" | tr '[:upper:]' '[:lower:]')" - -# A platform whose encoder-cache connector carries the output through a directory (Ascend) is -# admitted only when the encode and prefill roles mount one shared claim. Name an existing -# ReadWriteMany claim in E2E_C117_EC_CLAIM; it stays empty for platforms that need none. -EC_CLAIM="${E2E_C117_EC_CLAIM:-}" -EC_VOLUMES="" -if [ -n "$EC_CLAIM" ]; then - EC_VOLUMES="$(printf ' additionalVolumes:\n - mountPath: /ec-cache\n persistentVolumeClaim:\n claimName: %s' "$EC_CLAIM")" -fi -NF="case117-cpu-vendor" +NF=case117-cpu-vendor MD=case117-epd +PD=case117-pd +CLAIM=case117-claim +PV=case117-pv IMAGE="${E2E_MD_IMAGE:-registry.k8s.io/pause:3.10}" -SETTLE_ROUNDS="${E2E_C117_SETTLE:-60}" # 60 x 5s = 5 min for the settings cache and re-derivation -POD_ROUNDS="${E2E_C117_PODS:-40}" # 40 x 5s for the group to form and its Pods to exist +WITNESS_IMAGE="${E2E_C117_WITNESS_IMAGE:-busybox:1.36}" +MOUNT=/ec-cache +SETTLE_ROUNDS="${E2E_C117_SETTLE:-60}" # 60 x 5s for the settings cache and re-derivation +POD_ROUNDS="${E2E_C117_PODS:-40}" # 40 x 5s for the group to render its Pods STATUS_ROUNDS="${E2E_C117_STATUS:-12}" # 12 x 5s for the observed roles to reach the status -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } # shellcheck source=/dev/null -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" - -# One Ready, schedulable worker node (not a control-plane one): the vendor label lands there and -# that node's CPU becomes the pool whose manufacturer the case controls. -pick_worker_node() { - kubectl get nodes -o json 2>/dev/null | python3 -c ' -import json,sys -for n in json.load(sys.stdin).get("items",[]): - cs=n.get("status",{}).get("conditions",[]) - if not any(c.get("type")=="Ready" and c.get("status")=="True" for c in cs): - continue - if n.get("spec",{}).get("unschedulable"): - continue - # A CPU-vendor probe only reads back on a node that feeds the CPU-only catalog. An - # acceleratable node feeds the GPU InstanceType instead, whose manufacturer is the - # accelerator vendor, so an injected cpu-model.vendor_id never surfaces here and the - # platform-type poll below times out. Skip those nodes, exactly as case-18 reads the - # CPU pool (acceleratable=false) rather than the accelerator one. - if n.get("metadata",{}).get("labels",{}).get("feature.gpustack.ai/acceleratable") == "true": - continue - ts=n.get("spec",{}).get("taints",[]) - if any(str(t.get("key","")).startswith("node-role.kubernetes.io/") for t in ts): - continue - print(n["metadata"]["name"]) - break -' -} +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_epd-lib.sh" -# The InstanceType reporting the injected platform, skipping types on their way out and Inactive -# ones for the reasons case-51 records: naming either is refused at admission for a reason no row -# here is about. -epd_platform_type() { - kubectl get instancetypes.worker.gpustack.ai \ - -o jsonpath='{range .items[*]}{.metadata.name}|{.metadata.deletionTimestamp}|{.spec.inactive}|{.status.detail.manufacturer}{"\n"}{end}' \ - 2>/dev/null \ - | while IFS='|' read -r name deleting inactive manufacturer; do - [ -n "$name" ] || continue - [ -z "$deleting" ] || continue - [ "$inactive" = true ] && continue - if [ "$manufacturer" = "$PLATFORM" ]; then - echo "$name" - break - fi - done +# The picker document is YAML, so the assertions on the converted document need jq and a YAML +# reader. jq is a runner requirement the whole suite shares. A missing YAML reader instead +# degrades only the picker-document section: the platform, deployment, Service, status and mount +# rows need no YAML parse, so they still run - the section SKIPs its rows loudly instead of +# taking the whole case down (the base case ran its full surface without this dependency; the +# dependency arrived with the absorbed picker checks and must not widen past them). +command -v jq >/dev/null 2>&1 || { echo "[case-117] jq is required to read the router document." >&2; exit 2; } +PICKER_YAML_OK=yes +python3 -c 'import yaml' >/dev/null 2>&1 || { + echo "[case-117] pyyaml is absent: the picker-document rows will SKIP." >&2 + PICKER_YAML_OK=no } -ORIG_SETTING="$(kubectl -n "$SYSTEM_NS" get secret gpustack-settings -o jsonpath="{.data.${SETTING}}" 2>/dev/null)" -set_awareness() { # true|false - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"${SETTING}\":\"$(printf '%s' "$1" | base64)\"}}" >/dev/null 2>&1 -} -restore_setting() { - local v=null - [ -n "$ORIG_SETTING" ] && v="\"${ORIG_SETTING}\"" - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"${SETTING}\":${v}}}" >/dev/null 2>&1 -} - -# Restoring the value is not the end of the restore: the worker reads it through a thirty-second -# cache, and the type derived from it outlives the patch by that cache plus one reconcile. Wait for -# the derived type to go, so the next case in a serial run does not list an aware type this run left -# behind (case-18's wait_generic, on this same fixture). A cluster that started with the setting -# already on keeps its derived type, so there is nothing to wait for and the wait is skipped. -settle_back() { - [ "$(printf '%s' "$ORIG_SETTING" | base64 -d 2>/dev/null)" = "true" ] && return 0 - for _ in $(seq 1 "$SETTLE_ROUNDS"); do - [ -z "$(epd_platform_type)" ] && return 0 - sleep 5 - done - return 0 -} +# One Ready, schedulable worker node (not a control-plane one): the vendor label lands there and +# that node's CPU becomes the pool whose manufacturer the case controls. +# +# The InstanceType reporting the injected platform, the label that builds it, the setting it turns +# on, the type it waits for and the undo of all three are the library's: epd_pick_worker_node, +# epd_platform_type and epd_setup_cpu_vendor_fixture. # One list call, because this runs from the EXIT trap: Kueue holds a finalizer on every Pod of a # serving group and releases it only when the Workload is deleted, so a cleanup that blocks on the # deployment would block for as long as the caller allows (case-50's mechanic). -force_release() { - local row wl uids owners u - uids="$(kubectl -n "$NS" get pods -l "app.kubernetes.io/instance=${MD}" \ +force_release() { # + local md="$1" row wl uids owners u + uids="$(kubectl -n "$NS" get pods -l "app.kubernetes.io/instance=${md}" \ -o jsonpath='{range .items[*]}{.metadata.uid}{"\n"}{end}' 2>/dev/null)" [ -n "$uids" ] || return 0 while IFS= read -r row; do @@ -197,88 +180,156 @@ EOF return 0 } -INJECTED=false cleanup() { echo echo "[case-117] cleanup" - kubectl -n "$NS" delete modeldeployments.worker.gpustack.ai "$MD" \ - --ignore-not-found --wait=false >/dev/null 2>&1 + epd_teardown_cpu_vendor_fixture "$NS" "$NF" "$SETTLE_ROUNDS" "$SYSTEM_NS" cleanup_deployments +} +trap cleanup EXIT + +# The deployments and the claim this run made, removed while the platform they were admitted on +# still exists. The fixture teardown runs after this, so the type is still there while the +# Workloads are released. +cleanup_deployments() { + kubectl -n "$NS" delete pod case117-witness-write case117-witness-read \ + --ignore-not-found --force --grace-period=0 >/dev/null 2>&1 + for md in "$MD" "$PD"; do + kubectl -n "$NS" delete modeldeployments.worker.gpustack.ai "$md" \ + --ignore-not-found --wait=false >/dev/null 2>&1 + done for _ in $(seq 1 12); do sleep 5 - force_release - [ -z "$(kubectl -n "$NS" get pods -l "app.kubernetes.io/instance=${MD}" --no-headers 2>/dev/null)" ] && break + force_release "$MD" + force_release "$PD" + [ -z "$(kubectl -n "$NS" get pods -l "app.kubernetes.io/instance=${MD}" --no-headers 2>/dev/null)" ] && + [ -z "$(kubectl -n "$NS" get pods -l "app.kubernetes.io/instance=${PD}" --no-headers 2>/dev/null)" ] && break done - if [ "$INJECTED" = true ]; then - kubectl -n "$NS" delete nodefeatures.nfd.k8s-sigs.io "$NF" --ignore-not-found >/dev/null 2>&1 - restore_setting - settle_back - fi + kubectl -n "$NS" delete pvc "$CLAIM" --ignore-not-found --wait=false >/dev/null 2>&1 + kubectl delete pv "$PV" --ignore-not-found --wait=false >/dev/null 2>&1 } -trap cleanup EXIT -# ---------------------------------------------------------------- the fixture +# ---------------------------------------------------------------- the shared fixture -IT="$(epd_platform_type)" +# The platform this deployment is admitted on: an InstanceType reporting the injected vendor, or +# one the cluster already carried. It is the ONE platform all three readings below share. +IT="$(epd_platform_type "$PLATFORM")" if [ -z "$IT" ]; then - NODE="$(pick_worker_node)" + NODE="$(epd_pick_worker_node)" if [ -z "$NODE" ]; then echo "[case-117] no Ready schedulable worker node; this case cannot place its vendor label" >&2 exit 2 fi echo "[case-117] injecting CPU vendor '${VENDOR}' on node ${NODE} and turning CPU-manufacturer awareness on" - INJECTED=true - - kubectl apply -f - >/dev/null 2>&1 </dev/null | head -n 1)" +fi + # The type's entrance LocalQueue in this namespace is what routes the group into its queue; the # operator materializes it in every non-reserved namespace once the queue exists, so this poll is # part of the fixture rather than a subject. ENTRANCE="$(kubectl get instancetypes.worker.gpustack.ai "$IT" -o jsonpath='{.status.entrance}' 2>/dev/null)" +ENTRANCE_LQ="" for _ in $(seq 1 "$SETTLE_ROUNDS"); do - [ -n "$(kubectl -n "$NS" get localqueue "$ENTRANCE" -o name 2>/dev/null)" ] && break + ENTRANCE_LQ="$(kubectl -n "$NS" get localqueue "${ENTRANCE:-}" -o name 2>/dev/null)" + [ -n "$ENTRANCE_LQ" ] && break sleep 5 done record PASS "a CPU-only pool reports an admitted platform" \ - "instancetype ${IT} reports manufacturer '${PLATFORM}' from the node labels; entrance ${ENTRANCE}" + "instancetype ${IT} reports manufacturer '${PLATFORM}' from ${ORIGIN}; entrance ${ENTRANCE}" -# ---------------------------------------------------------------- the subject +if [ -z "$ENTRANCE_LQ" ]; then + # Without the entrance queue nothing is admitted, no role Pod is created and no router document + # is ever rendered, so every reading below would report the same absence three times. + record SKIP "the pool's entrance LocalQueue exists" \ + "instancetype ${IT} names entrance '${ENTRANCE:-}' and no localqueue/${ENTRANCE:-} answered in ${NS} within $((SETTLE_ROUNDS * 5))s, so nothing is admitted, no role Pod is created and no router document is ever rendered" + print_rows + echo "[case-117] skipped: no entrance LocalQueue" + exit 0 +fi + +record PASS "the pool's entrance LocalQueue exists" "${ENTRANCE_LQ##*/} in ${NS}" + +# The claim the two encoder-side roles mount: a hostPath volume under /tmp on the labeled node, +# manually bound. The pool the labeled node feeds is the only pool the deployment names, so every +# Pod that mounts this claim lands on that node and ReadWriteOnce is enough; DirectoryOrCreate +# spares the case any need to reach the node's shell. A storage class is deliberately not used: +# binding by volumeName needs none, and the default class of an arbitrary cluster is not this +# case's to depend on. +kubectl apply -f - >/dev/null 2>&1 </dev/null 2>&1; then + DEPLOYED=yes +fi + +# The three Pods need to EXIST, not to run: every selector row reads labels the Pod carries from +# render time, and the group renders its Pods gated while Kueue has not admitted them. A refused +# apply is not waited on — there is nothing that would arrive. +PODS=0 +if [ "$DEPLOYED" = yes ]; then + for _ in $(seq 1 "$POD_ROUNDS"); do + PODS="$(kubectl -n "$NS" get pods -l "app.kubernetes.io/instance=${MD}" --no-headers 2>/dev/null | grep -c . || true)" + [ "${PODS:-0}" -ge 3 ] && break + sleep 5 + done +fi + +# ---------------------------------------------------------------- the addresses + +# The role Services, the front door and the status view: what a caller's connection lands on. + +record PASS "a CPU-only pool reports an admitted platform" \ + "instancetype ${IT} reports manufacturer '${PLATFORM}' from the node labels; entrance ${ENTRANCE}" + +if [ "$DEPLOYED" = yes ]; then record PASS "the encoder deployment is admitted and created" \ "one encode, one prefill and one decode role on ${IT}, llm-d-router, image ${IMAGE}" else @@ -326,14 +407,6 @@ else exit 1 fi -# The three Pods need to EXIST, not to run: every selector row reads labels the Pod carries from -# render time, and the group renders its Pods gated while Kueue has not admitted them. -PODS="" -for _ in $(seq 1 "$POD_ROUNDS"); do - PODS="$(kubectl -n "$NS" get pods -l "app.kubernetes.io/instance=${MD}" --no-headers 2>/dev/null | grep -c . || true)" - [ "${PODS:-0}" -ge 3 ] && break - sleep 5 -done if [ "${PODS:-0}" -ge 3 ]; then record PASS "all three role Pods exist" "${PODS} Pods rendered for the three roles" else @@ -457,8 +530,8 @@ fi # it. A check that accepted it would accept any broken selector as withdrawal. # # A selector that names the encoder cannot also reach another role: label terms are a conjunction, -# so component=encode already excludes every other component. That is why the control is built on -# the empty set and not on a foreign Pod. +# so component=encode already excludes every other component. That is why the control is built on the +# empty set and not on a foreign Pod. DANGLING="app.kubernetes.io/name=model-deployment,app.kubernetes.io/instance=${MD},app.kubernetes.io/component=encode,case117.gpustack.ai/no-such-label=true" if selector_fronts_only "$DANGLING" encode; then record FAIL "the per-role Service check refuses an empty set that nothing explains" \ @@ -550,7 +623,7 @@ esac # The published endpoint is the front door as a string. For a routed deployment it is derived from # the router's objects, and on a cluster that cannot pull the router image it may never publish — # which is a NO-READ rather than a FAIL: the router image is deliberately not this case's subject -# (case 71 isolates it). An endpoint that names the encoder role, on any cluster, is a FAIL. +# (case 70 isolates it). An endpoint that names the encoder role, on any cluster, is a FAIL. EP="$(kubectl -n "$NS" get modeldeployments.worker.gpustack.ai "$MD" -o jsonpath='{.status.endpoint}' 2>/dev/null)" if [ -z "$EP" ]; then record NO-READ "status.endpoint never names the encoder" \ @@ -578,6 +651,448 @@ case "$HDR" in ;; esac +# ---------------------------------------------------------------- the router picker document + +# What the router is TOLD about the encoder. The rows read the document the operator renders into +# the router Pod, so an unpullable router image does not skip them: the ConfigMap exists before +# any image is pulled. + +record PASS "a CPU-only pool reports an admitted platform" \ + "instancetype ${IT} reports manufacturer '${PLATFORM}' from ${ORIGIN}" + +# The refusal itself is reported by the addresses section above, which stops the run on it; this +# row states the same admission in the words of this subject. +if [ "$DEPLOYED" = yes ]; then + record PASS "the encoder deployment is admitted and created" \ + "one encode, one prefill and one decode role on ${IT}, llm-d-router" +else + record FAIL "the encoder deployment is admitted and created" \ + "no modeldeployment/${MD} after the apply: $(printf '%s' "${APPLY_OUT:-}" | tr '\n' ' ' | cut -c1-220)" +fi + +# The CONTROL deployment is the same router with no encoder at all. Reading the same document twice +# is what makes the encoder's facts encoder facts rather than prefill/decode facts that happen to +# be there: every term the control lacks is a term the encoder added. +# +# Its apply output is KEPT rather than discarded. A refusal here would otherwise surface as the +# pod-count row below, which names a missing Pod and not the refusal that caused it — the same +# wrong-cause report the LocalQueue poll was fixed for. +PD_APPLY_OUT="$(cat <&1 +apiVersion: worker.gpustack.ai/v1alpha1 +kind: ModelDeployment +metadata: + name: ${PD} + namespace: ${NS} +spec: + engine: + name: vLLM + image: ${IMAGE} + model: + name: Qwen/Qwen2.5-0.5B-Instruct + router: + name: llm-d-router + roles: + - name: prefill + kind: Prefill + instanceType: ${IT} + replicas: 1 + image: ${IMAGE} + - name: decode + kind: Decode + instanceType: ${IT} + replicas: 1 + image: ${IMAGE} +YAML +)" + +# An empty output is not an accepted apply, so it counts as refused here too. +PD_REFUSED=no +case "$PD_APPLY_OUT" in + *created* | *configured*) ;; + *) PD_REFUSED=yes ;; +esac + +# THE ENCODER DEPLOYMENT RENDERS THREE ROLE PODS AND THE CONTROL TWO, and the count each row +# waits for is that deployment's own role count rather than a flat number: a two-of-three gate +# would pass a renderer that dropped the encoder, which is the failure this case exists to catch. +for md in "$MD" "$PD"; do + if [ "$md" = "$MD" ]; then want=3; else want=2; fi + if [ "$md" = "$MD" ]; then + # The shared fixture already waited for this deployment's three role Pods. + count="$PODS" + else + count="" + for _ in $(seq 1 "$POD_ROUNDS"); do + count="$(kubectl -n "$NS" get pods -l "app.kubernetes.io/instance=${md}" --no-headers 2>/dev/null | grep -c . || true)" + [ "${count:-0}" -ge "$want" ] && break + sleep 5 + done + fi + if [ "${count:-0}" -ge "$want" ]; then + record PASS "${md} rendered its role Pods" "${count} Pods for ${want} roles; the rows below read the router document, not the containers" + else + # A refused control apply is the cause of a missing control Pod, so it is named here instead of + # leaving the row to be read as a renderer that dropped a role. + cause="" + if [ "$md" = "$PD" ] && [ "$PD_REFUSED" = yes ]; then + cause="; its apply was refused: $(printf '%s' "$PD_APPLY_OUT" | tr '\n' ' ' | cut -c1-220)" + fi + record FAIL "${md} rendered its role Pods" \ + "only ${count:-0} of ${want} role Pods appeared within $((POD_ROUNDS * 5))s, so the router ConfigMap may never be rendered${cause}" + fi +done + +# The document as YAML, from the ConfigMap the operator renders into the router Pod. A router +# configured by argv renders none, so its ABSENCE is a distinct answer from an empty one and both +# are read rather than assumed. +picker_yaml() { # + kubectl -n "$NS" get configmap "$1-router" -o jsonpath='{.data.epp\.yaml}' 2>/dev/null +} + +# Each check runs the document through jq as YAML (jq reads YAML only with -y on some builds), so +# the document is converted once with python3, which every case in this suite already requires. +json_of() { # yaml-on-stdin + python3 -c ' +import json,sys +try: + import yaml +except ImportError: + sys.stderr.write("pyyaml is required to read the router document") + raise SystemExit(3) +d = yaml.safe_load(sys.stdin.read()) +print(json.dumps(d) if d is not None else "{}") +' +} + +if [ "$PICKER_YAML_OK" != yes ]; then + record SKIP "the router ConfigMap carries an EndpointPickerConfig" \ + "pyyaml is absent in the runner, so the rendered document cannot be read here" +else +DOC="$(picker_yaml "$MD")" +if [ -z "$DOC" ]; then + record FAIL "the router ConfigMap carries an EndpointPickerConfig" \ + "no configmap/${MD}-router carries a key epp.yaml, so there is no document to read" +else + DOC_JSON="$(printf '%s' "$DOC" | json_of)" || { + echo "[case-117] cannot read the router document: pyyaml is required in the runner." >&2 + exit 2 + } + PD_JSON="$(printf '%s' "$(picker_yaml "$PD")" | json_of)" || PD_JSON="" + + if printf '%s' "$DOC_JSON" | jq -e '.kind == "EndpointPickerConfig" and (.plugins | type == "array")' \ + >/dev/null 2>&1; then + record PASS "the router ConfigMap carries an EndpointPickerConfig" \ + "$(printf '%s' "$DOC_JSON" | jq -r '.plugins | length') plugins and $(printf '%s' "$DOC_JSON" | jq -r '.schedulingProfiles | length') scheduling profiles" + else + record FAIL "the router ConfigMap carries an EndpointPickerConfig" \ + "kind=$(printf '%s' "$DOC_JSON" | jq -r '.kind // ""'), and the document has no plugins array to read" + fi + + # The encode filter: one label-selector-filter whose matchLabels select role-kind=encode. The + # label key is the role-kind label the operator stamps on every Pod, so this is the selector the + # router will actually match against, read off the document rather than assumed from the render. + FILTER_OK="$(printf '%s' "$DOC_JSON" | jq -r ' + [ .plugins[]? + | select(.type == "label-selector-filter") + | select(.parameters.matchLabels["modeldeployment.gpustack.ai/role-kind"] == "encode") + ] | length')" + if [ "${FILTER_OK:-0}" -ge 1 ]; then + record PASS "an encode filter selects the encoder's Pods" \ + "a label-selector-filter matches modeldeployment.gpustack.ai/role-kind=encode" + else + record FAIL "an encode filter selects the encoder's Pods" \ + "no label-selector-filter matches role-kind=encode; the picker would never route to an encoder" + fi + + # THE ENCODE PROFILE'S PLUGIN LIST, asserted exactly. Upstream completes any profile naming none + # with its own max-score-picker at load time, so [encode-pods] is a valid, runnable profile — and + # any additional plugin is a scorer that cannot separate encoder candidates. + ENCODE_PLUGINS="$(printf '%s' "$DOC_JSON" | jq -c \ + '[.schedulingProfiles[]? | select(.name == "encode") | .plugins[]? | .pluginRef] | unique')" + if [ "$ENCODE_PLUGINS" = '["encode-pods"]' ]; then + record PASS "the encode profile scores on nothing but its own filter" \ + "plugins=${ENCODE_PLUGINS}: upstream appends its max-score-picker to a profile naming none" + else + record FAIL "the encode profile scores on nothing but its own filter" \ + "encode profile plugins=${ENCODE_PLUGINS}, expected [\"encode-pods\"]: an encoder keeps no KV cache, so the queue and KV scorers read gauges it never exports and the prefix scorer asks about tokens that do not exist yet" + fi + + # The decider, and the handler that routes BOTH halves at one. Without the encode term in the + # handler the decider is present but never consulted, and the encoder stage is silently unused. + DECIDER="$(printf '%s' "$DOC_JSON" | jq -r \ + '[.plugins[]? | select(.type == "always-disagg-multimodal-decider")] | length')" + HANDLER="$(printf '%s' "$DOC_JSON" | jq -c \ + '[.plugins[]? | select(.type == "disagg-profile-handler") | .parameters] | first // {}')" + if [ "${DECIDER:-0}" -ge 1 ]; then + record PASS "the multimodal decider is present" "always-disagg-multimodal-decider is in the document" + else + record FAIL "the multimodal decider is present" \ + "no always-disagg-multimodal-decider: without it the encoder stage is never selected and a media request is answered undivided" + fi + + HANDLER_ENCODE="$(printf '%s' "$HANDLER" | jq -r '.profiles.encode // ""')" + HANDLER_PREFILL="$(printf '%s' "$HANDLER" | jq -r '.profiles.prefill // ""')" + HANDLER_DECIDER_ENCODE="$(printf '%s' "$HANDLER" | jq -r '.deciders.encode // ""')" + if [ "$HANDLER_ENCODE" = encode ] && [ "$HANDLER_PREFILL" = prefill ] \ + && [ "$HANDLER_DECIDER_ENCODE" = always-disagg-multimodal-decider ]; then + record PASS "the handler routes the encoder stage at the multimodal decider" \ + "profiles.encode=encode, profiles.prefill=prefill, deciders.encode=always-disagg-multimodal-decider" + else + record FAIL "the handler routes the encoder stage at the multimodal decider" \ + "profiles.encode='${HANDLER_ENCODE}' profiles.prefill='${HANDLER_PREFILL}' deciders.encode='${HANDLER_DECIDER_ENCODE}'" + fi + + # The precise prefix-cache producer's Pod discovery EXCLUDES the two kinds that publish no events. + # The exclusion is what makes the terms exist at all, so the row reads the selector string rather + # than the plugin's presence. + # + # THE KEY IS podLabelSelector, which is what the renderer writes. A producer configured by a + # different field would render a document this case could not read, and reading one that does not + # exist would answer "no producer" on a cluster that has one — so the key is named here rather than + # guessed, and the failure message quotes it. + DISCOVERY="$(printf '%s' "$DOC_JSON" | jq -r ' + [ .plugins[]? + | select(.type == "precise-prefix-cache-producer") + | .parameters.kvEventsConfig.podDiscoveryConfig.podLabelSelector // "" + ] | first // ""')" + case "$DISCOVERY" in + *role-kind!=decode*role-kind!=encode* | *role-kind!=encode*role-kind!=decode*) + record PASS "the prefix-cache producer excludes the kinds that publish no events" \ + "podLabelSelector: ${DISCOVERY}" + ;; + "") + # The renderer picks the PRECISE producer whenever the deployment opens a KV event port, and an + # encoder leg does — so this branch is the alternative a renderer can still emit, not the + # expected shape: the approximate producer discovers Pods by service and carries no role + # selector, so there is nothing to exclude and nothing wrong. It is a PASS, because a + # deployment that renders no events has no eventless kind to exclude. + APPROX="$(printf '%s' "$DOC_JSON" | jq -r \ + '[.plugins[]? | select(.type == "approx-prefix-cache-producer")] | length')" + if [ "${APPROX:-0}" -ge 1 ]; then + record PASS "the prefix-cache producer excludes the kinds that publish no events" \ + "the approximate producer renders instead, which discovers Pods by service and has no role selector to exclude" + else + record FAIL "the prefix-cache producer excludes the kinds that publish no events" \ + "neither producer is in the document, so neither kind is excluded and nothing is subscribed" + fi + ;; + *) + record FAIL "the prefix-cache producer excludes the kinds that publish no events" \ + "podLabelSelector '${DISCOVERY}' lacks role-kind!=decode or role-kind!=encode: both kinds publish no KV events and would be subscribers dialing a socket the engine never opens" + ;; + esac + + # ---------------------------------------------------------------- the control deployment + + # The same router, the same engine, one role fewer. Every term the control lacks is a term the + # encoder added, which is what makes the rows above encoder facts rather than router defaults. + if [ -z "$PD_JSON" ]; then + record SKIP "a deployment with no encoder role renders no encoder stage" \ + "no configmap/${PD}-router carries a key epp.yaml, so the control document could not be read; the encoder rows above stand on their own" + else + CTRL_DECIDER="$(printf '%s' "$PD_JSON" | jq -r \ + '[.plugins[]? | select(.type == "always-disagg-multimodal-decider")] | length')" + CTRL_PROFILE="$(printf '%s' "$PD_JSON" | jq -r \ + '[.schedulingProfiles[]? | select(.name == "encode")] | length')" + CTRL_FILTER="$(printf '%s' "$PD_JSON" | jq -r \ + '[.plugins[]? | select(.type == "label-selector-filter") + | select(.parameters.matchLabels["modeldeployment.gpustack.ai/role-kind"] == "encode")] | length')" + if [ "$CTRL_DECIDER" -eq 0 ] && [ "$CTRL_PROFILE" -eq 0 ] && [ "$CTRL_FILTER" -eq 0 ]; then + record PASS "a deployment with no encoder role renders no encoder stage" \ + "the control's picker carries no multimodal decider, no encode profile and no encode filter, so all three are the encoder's and not the router's defaults" + else + record FAIL "a deployment with no encoder role renders no encoder stage" \ + "the control still carries decider=${CTRL_DECIDER} profile=${CTRL_PROFILE} filter=${CTRL_FILTER}, which would mean the encoder rows above prove nothing about the encoder" + fi + fi +fi +fi # PICKER_YAML_OK: the picker section is closed here; its SKIP shape sits above + +# ---------------------------------------------------------------- the shared claim mounts + +# The claim one role's FIRST Pod mounts, as "claimName|mountPath|readOnly", or "" when it mounts no +# claim. Pending Pods count: the render is what these rows pin, and a gated Pod carries its full +# spec (case-89's reading posture). The reading itself is the library's: epd_role_pod_claim. + +record PASS "a CPU-only pool reports an admitted platform" \ + "instancetype ${IT} reports manufacturer '${PLATFORM}'; entrance ${ENTRANCE}" + +# The refusal itself is reported by the addresses section above, which stops the run on it; this +# row states the same admission in the words of this subject. +if [ "$DEPLOYED" = yes ]; then + record PASS "the shared-claim encoder deployment is admitted and created" \ + "encode and prefill mount ${CLAIM} at ${MOUNT} on ${IT}" +else + record FAIL "the shared-claim encoder deployment is admitted and created" \ + "no modeldeployment/${MD} after the apply: $(printf '%s' "${APPLY_OUT:-}" | tr '\n' ' ' | cut -c1-220)" +fi + +if [ "${PODS:-0}" -ge 3 ]; then + record PASS "all three role Pods exist" "${PODS} Pods rendered; the mount rows below read specs, not running containers" +else + record FAIL "all three role Pods exist" \ + "only ${PODS:-0} Pod(s) of the group appeared within $((POD_ROUNDS * 5))s, so the mount rows below have nothing to read" +fi + +ENCODE_CLAIM="$(epd_role_pod_claim "$NS" "$MD" encode)" +PREFILL_CLAIM="$(epd_role_pod_claim "$NS" "$MD" prefill)" +DECODE_CLAIM="$(epd_role_pod_claim "$NS" "$MD" decode)" + +case "$ENCODE_CLAIM" in + "${CLAIM}|${MOUNT}|false") + record PASS "the encoder Pod mounts the claim at the shared path, writable" \ + "rendered ${ENCODE_CLAIM}" + ;; + *) + record FAIL "the encoder Pod mounts the claim at the shared path, writable" \ + "rendered '${ENCODE_CLAIM:-no claim mount}', expected '${CLAIM}|${MOUNT}|false'" + ;; +esac + +case "$PREFILL_CLAIM" in + "${CLAIM}|${MOUNT}|"*) + + # Read-only is the PREFILLER's own choice — it only reads — so the row pins the claim name and + # the path and leaves the flag open, unlike the encoder's row above. + record PASS "the prefiller Pod mounts the same claim at the same path" \ + "rendered ${PREFILL_CLAIM}, matching the encoder's claim and path" + ;; + *) + record FAIL "the prefiller Pod mounts the same claim at the same path" \ + "rendered '${PREFILL_CLAIM:-no claim mount}', expected '${CLAIM}|${MOUNT}|'" + ;; +esac + +if [ -z "$DECODE_CLAIM" ]; then + record PASS "the decoder Pod mounts no claim" \ + "the mounted-directory leg ends at the prefiller, and the decode role declares no volume" +else + record FAIL "the decoder Pod mounts no claim" "rendered ${DECODE_CLAIM}" +fi + +# ---------------------------------------------------------------- the claim is one filesystem + +PVC_PHASE="$(kubectl -n "$NS" get pvc "$CLAIM" -o jsonpath='{.status.phase}' 2>/dev/null)" +if [ "$PVC_PHASE" = Bound ]; then + record PASS "the claim is Bound" "pvc/${CLAIM} bound to pv/${PV}" +else + record FAIL "the claim is Bound" "pvc/${CLAIM} phase='${PVC_PHASE:-}', so no witness can mount it" +fi + +# Two one-shot witnesses pinned to the labeled node: the pool's only node, which is also where the +# hostPath lives. The writer records a token through its mount; the reader cats the same path. That +# round-trip is the reading that separates one shared filesystem from two claims sharing a name. +# +# THE WRITER IS CREATED FIRST AND THE READER ONLY AFTER IT HAS RUN, because the reader cats a file +# that does not exist until the writer has written it: created together, whichever the kubelet starts +# first decides the verdict, and on a cluster that already holds the image the reader usually loses, +# failing a claim that shares perfectly well. Nothing in the manifest, the node or the loop orders +# them, so the case orders them. +# +# This is the LAST reading of the case, so an unpullable witness image skips this row alone and +# leaves every row above it standing. +if [ "$PVC_PHASE" = Bound ]; then + TOKEN="case117-$(date +%s)" + kubectl apply -f - >/dev/null 2>&1 < /shared/written-by-the-other-mount"] + volumeMounts: + - name: shared + mountPath: /shared + volumes: + - name: shared + persistentVolumeClaim: + claimName: ${CLAIM} +YAML + + # An image that cannot be pulled is a SKIP naming the image (case-81's rule): a container nobody + # could start is a reading nobody took, not a verdict about the claim. The reason a witness Pod + # is waiting on is read by name, because it carries none of the deployment's own labels. + W_ROUNDS=$((POD_ROUNDS)) + W_OK=no + for _ in $(seq 1 "$W_ROUNDS"); do + case "$(kubectl -n "$NS" get pod case117-witness-write -o jsonpath='{.status.phase}' 2>/dev/null)" in + Succeeded) W_OK=yes; break ;; + Failed) break ;; + esac + case "$(epd_pod_pull_reason "$NS" case117-witness-write)" in ImagePullBackOff | ErrImagePull) break ;; esac + sleep 5 + done + + if [ "$W_OK" != yes ]; then + case "$(epd_pod_pull_reason "$NS" case117-witness-write)" in + ImagePullBackOff | ErrImagePull) + record SKIP "a file written through one mount is read back through the other" \ + "the witness image '${WITNESS_IMAGE}' could not be pulled ($(epd_pod_pull_reason "$NS" case117-witness-write)), so no reading was taken; the render rows above stand on their own" + ;; + *) + record FAIL "a file written through one mount is read back through the other" \ + "the writer witness never completed (phase '$(kubectl -n "$NS" get pod case117-witness-write -o jsonpath='{.status.phase}' 2>/dev/null)')" + ;; + esac + else + # The file exists now, so the reader is safe to start. + kubectl apply -f - >/dev/null 2>&1 </dev/null)" in + Succeeded | Failed) R_OK=yes; break ;; + esac + case "$(epd_pod_pull_reason "$NS" case117-witness-read)" in ImagePullBackOff | ErrImagePull) break ;; esac + sleep 5 + done + + READ_BACK="" + if [ "$R_OK" = yes ]; then + READ_BACK="$(kubectl -n "$NS" logs case117-witness-read -c read 2>/dev/null)" + fi + if [ "$READ_BACK" = "$TOKEN" ]; then + record PASS "a file written through one mount is read back through the other" \ + "witness pair exchanged '${READ_BACK}' through pvc/${CLAIM}: one filesystem behind both mounts" + else + record FAIL "a file written through one mount is read back through the other" \ + "reader witness returned '${READ_BACK:-}', expected '${TOKEN}'" + fi + fi +fi + # Results. print_rows [ "$FAILS" -eq 0 ] || { echo "[case-117] ${FAILS} check(s) FAILED"; exit 1; } diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-118.sh b/.agents/skills/gpustack-operator-e2e/cases/case-118.sh index 959346df1..193b8dd2f 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/case-118.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-118.sh @@ -67,12 +67,11 @@ # set -uo pipefail -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: case-118.sh }" SYSTEM_NS="${E2E_SYSTEM_NS:-gpustack-system}" -SETTING=instance-type-aware-cpu-manufacturer VENDOR="${E2E_C118_VENDOR:-AMD}" PLATFORM="$(printf '%s' "$VENDOR" | tr '[:upper:]' '[:lower:]')" @@ -95,164 +94,56 @@ PROBE=case118-probe FILLER=case118-filler SUBJECT=case118-subject -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } -# shellcheck source=/dev/null -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" # shellcheck source=/dev/null . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_quota-lib.sh" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_md-lib.sh" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_epd-lib.sh" # ---------------------------------------------------------------- the fixture -pick_worker_node() { - kubectl get nodes -o json 2>/dev/null | python3 -c ' -import json,sys -for n in json.load(sys.stdin).get("items",[]): - cs=n.get("status",{}).get("conditions",[]) - if not any(c.get("type")=="Ready" and c.get("status")=="True" for c in cs): - continue - if n.get("spec",{}).get("unschedulable"): - continue - # A CPU-vendor probe only reads back on a node that feeds the CPU-only catalog. An - # acceleratable node feeds the GPU InstanceType instead, whose manufacturer is the - # accelerator vendor, so an injected cpu-model.vendor_id never surfaces here and the - # platform-type poll below times out. Skip those nodes, exactly as case-18 reads the - # CPU pool (acceleratable=false) rather than the accelerator one. - if n.get("metadata",{}).get("labels",{}).get("feature.gpustack.ai/acceleratable") == "true": - continue - ts=n.get("spec",{}).get("taints",[]) - if any(str(t.get("key","")).startswith("node-role.kubernetes.io/") for t in ts): - continue - print(n["metadata"]["name"]) - break -' -} - -epd_platform_type() { - kubectl get instancetypes.worker.gpustack.ai \ - -o jsonpath='{range .items[*]}{.metadata.name}|{.metadata.deletionTimestamp}|{.spec.inactive}|{.status.detail.manufacturer}{"\n"}{end}' \ - 2>/dev/null \ - | while IFS='|' read -r name deleting inactive manufacturer; do - [ -n "$name" ] || continue - [ -z "$deleting" ] || continue - [ "$inactive" = true ] && continue - if [ "$manufacturer" = "$PLATFORM" ]; then - echo "$name" - break - fi - done -} - -ORIG_SETTING="$(kubectl -n "$SYSTEM_NS" get secret gpustack-settings -o jsonpath="{.data.${SETTING}}" 2>/dev/null)" -set_awareness() { # true|false - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"${SETTING}\":\"$(printf '%s' "$1" | base64)\"}}" >/dev/null 2>&1 -} -restore_setting() { - local v=null - [ -n "$ORIG_SETTING" ] && v="\"${ORIG_SETTING}\"" - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"${SETTING}\":${v}}}" >/dev/null 2>&1 -} - -# Restoring the value is not the end of the restore: the worker reads it through a thirty-second -# cache, and the type derived from it outlives the patch by that cache plus one reconcile. Wait for -# the derived type to go, so the next case in a serial run does not list an aware type this run left -# behind (case-18's wait_generic, on this same fixture). A cluster that started with the setting -# already on keeps its derived type, so there is nothing to wait for and the wait is skipped. -settle_back() { - [ "$(printf '%s' "$ORIG_SETTING" | base64 -d 2>/dev/null)" = "true" ] && return 0 - for _ in $(seq 1 "$SETTLE_ROUNDS"); do - [ -z "$(epd_platform_type)" ] && return 0 - sleep 5 - done - return 0 -} - -# One list call, because this runs from the EXIT trap on every invocation, pass or fail: Kueue holds -# a finalizer on every Pod of a serving group and releases it only when the Workload is deleted. -force_release() { - local md="$1" row wl uids owners u - uids="$(kubectl -n "$NS" get pods -l "app.kubernetes.io/instance=${md}" \ - -o jsonpath='{range .items[*]}{.metadata.uid}{"\n"}{end}' 2>/dev/null)" - [ -n "$uids" ] || return 0 - while IFS= read -r row; do - [ -n "$row" ] || continue - wl="${row%%=*}" - owners="${row#*=}" - for u in $uids; do - case " $owners " in *" $u "*) - kubectl -n "$NS" delete workloads.kueue.x-k8s.io "$wl" \ - --ignore-not-found --wait=false >/dev/null 2>&1 - break - ;; - esac - done - done </dev/null) -EOF +# The InstanceType reporting the injected platform, the label that builds it, the setting it turns +# on, the type it waits for and the undo of all three are the library's: epd_platform_type, +# epd_pick_worker_node and epd_setup_cpu_vendor_fixture. - return 0 +cleanup() { + epd_teardown_cpu_vendor_fixture "$NS" "$NF" "$SETTLE_ROUNDS" "$SYSTEM_NS" cleanup_deployments } +trap cleanup EXIT -INJECTED=false -cleanup() { +# The deployments this run made, removed while the platform they were admitted on still exists. +# The fixture teardown runs after this, so the type is still there while the Workloads are released. +cleanup_deployments() { kubectl -n "$NS" delete modeldeployments.worker.gpustack.ai \ "$PROBE" "$FILLER" "$SUBJECT" --ignore-not-found --wait=false >/dev/null 2>&1 # The release is a loop, not one pass: the probe holds up to MAX_REPLICAS Pods, a deletion takes - # longer than a single round trip, and force_release deletes only the Workloads whose owners it can - # still see. One pass finds nothing while the Pods are still going and leaves them holding the quota + # longer than a single round trip, and md_force_release deletes only the Workloads whose owners it + # can still see. One pass finds nothing while the Pods are still going and leaves them holding the quota # the next case needs. for _ in $(seq 1 12); do sleep 5 - force_release "$PROBE" - force_release "$FILLER" - force_release "$SUBJECT" + md_force_release "$NS" "$PROBE" + md_force_release "$NS" "$FILLER" + md_force_release "$NS" "$SUBJECT" [ -z "$(kubectl -n "$NS" get pods \ -l "app.kubernetes.io/instance in (${PROBE},${FILLER},${SUBJECT})" \ --no-headers 2>/dev/null)" ] && break done - if [ "$INJECTED" = true ]; then - kubectl -n "$NS" delete nodefeatures.nfd.k8s-sigs.io "$NF" --ignore-not-found >/dev/null 2>&1 - restore_setting - settle_back - fi } -trap cleanup EXIT -IT="$(epd_platform_type)" +IT="$(epd_platform_type "$PLATFORM")" if [ -z "$IT" ]; then - NODE="$(pick_worker_node)" + NODE="$(epd_pick_worker_node)" if [ -z "$NODE" ]; then echo "[case-118] no Ready schedulable worker node; this case cannot place its vendor label" >&2 exit 2 fi echo "[case-118] injecting CPU vendor '${VENDOR}' on node ${NODE} and turning CPU-manufacturer awareness on" - INJECTED=true - kubectl apply -f - >/dev/null 2>&1 </dev/null)" +CQ="$(md_cq_of_it "$NS" "$IT")" if [ -z "$CQ" ]; then echo "[case-118] the platform InstanceType names no reachable ClusterQueue through ${NS}/${ENTRANCE}" >&2 exit 2 @@ -357,41 +248,13 @@ YAML # ---------------------------------------------------------------- the reading instruments (case 50's) -# Every Workload owning one of this deployment's Pods, one "name=status=role" line each. Two API -# calls per invocation, not one per Workload: this is the hot path of every settle loop below. -workload_verdicts() { - local md="$1" pods row wl rest status owners p - pods="$(kubectl -n "$NS" get pods -l "app.kubernetes.io/instance=${md}" \ - -o jsonpath='{range .items[*]}{.metadata.uid}={.metadata.labels.app\.kubernetes\.io/component}{"\n"}{end}' \ - 2>/dev/null)" - [ -n "$pods" ] || return 0 - while IFS= read -r row; do - [ -n "$row" ] || continue - wl="${row%%=*}" - rest="${row#*=}" - status="${rest%%=*}" - owners="${rest#*=}" - for p in $pods; do - # Padded on both sides so a uid cannot match a longer one it is a prefix of. - case " $owners " in *" ${p%%=*} "*) - echo "${wl}=${status}=${p#*=}" - break - ;; - esac - done - done </dev/null) -EOF -} - # "RESERVED WAITING" once every replica has a Workload and the counts stop moving. Waiting is what # did NOT reserve, not what Kueue judged short: once one replica is turned away, the identical ones # behind it carry no condition at all. settle_verdicts() { local md="$1" want="$2" prev="" cur n r=0 w=0 for _ in $(seq 1 60); do - cur="$(workload_verdicts "$md")" + cur="$(md_workload_verdicts "$NS" "$md")" n="$(printf '%s' "$cur" | grep -c . || true)" r="$(printf '%s\n' "$cur" | grep -c '^[^=]*=True=' || true)" w="$(printf '%s\n' "$cur" | grep -c '^[^=]*=False=' || true)" @@ -412,29 +275,11 @@ settle_verdicts() { return 1 } -deployment_workloads() { - local md="$1" uids row wl owners u - uids="$(kubectl -n "$NS" get pods -l "app.kubernetes.io/instance=${md}" \ - -o jsonpath='{range .items[*]}{.metadata.uid}{"\n"}{end}' 2>/dev/null)" - [ -n "$uids" ] || return 0 - while IFS= read -r row; do - [ -n "$row" ] || continue - wl="${row%%=*}" - owners="${row#*=}" - for u in $uids; do - case " $owners " in *" $u "*) echo "$wl"; break ;; esac - done - done </dev/null) -EOF -} - # Every PodSet this deployment holds an assignment for, across ALL of its Workloads: three words is # the whole set, and reading a single Workload yields one and looks exactly like a starved role. assigned_sets() { local wl - for wl in $(deployment_workloads "$1"); do + for wl in $(md_workloads "$NS" "$1"); do kubectl -n "$NS" get workloads.kueue.x-k8s.io "$wl" \ -o jsonpath='{range .status.admission.podSetAssignments[*]}{.name}{" "}{end}' 2>/dev/null done @@ -453,10 +298,10 @@ wait_admitted() { } # Deletes a deployment and waits for its Pods to go, then deletes any Workload that outlived them. -# The Workloads are named BEFORE the delete: force_release finds them through the Pods. +# The Workloads are named BEFORE the delete: md_force_release finds them through the Pods. release_deployment() { local md="$1" wls wl - wls="$(workload_verdicts "$md" | cut -d= -f1)" + wls="$(md_workload_verdicts "$NS" "$md" | cut -d= -f1)" kubectl -n "$NS" delete modeldeployments.worker.gpustack.ai "$md" \ --ignore-not-found --wait=false >/dev/null 2>&1 for _ in $(seq 1 60); do @@ -464,7 +309,7 @@ release_deployment() { --no-headers 2>/dev/null | wc -l | tr -d ' ')" = 0 ] && break sleep 2 done - force_release "$md" + md_force_release "$NS" "$md" for wl in $wls; do kubectl -n "$NS" delete workloads.kueue.x-k8s.io "$wl" \ --ignore-not-found --wait=false >/dev/null 2>&1 @@ -488,7 +333,7 @@ fi # What Kueue charged ONE replica, read off a baseline Workload while it still holds its reservation: # the unit the pool's quota is written in, whatever the InstanceType's own numbers say. CHARGE="$(workload_charge "$NS" \ - "$(workload_verdicts "$SUBJECT" | sed -n 's/=True=.*$//p' | head -n 1)")" + "$(md_workload_verdicts "$NS" "$SUBJECT" | sed -n 's/=True=.*$//p' | head -n 1)")" release_deployment "$SUBJECT" @@ -585,7 +430,7 @@ if [ "$FILL" -ge 1 ]; then # stop the rows below rather than passing them vacuously. SUBJ="$(settle_verdicts "$SUBJECT" 3)" SUBJ_RC=$? - VERDICTS="$(workload_verdicts "$SUBJECT")" + VERDICTS="$(md_workload_verdicts "$NS" "$SUBJECT")" HOLDING="$(printf '%s\n' "$VERDICTS" | sed -n 's/^[^=]*=True=//p' | sort -u | tr '\n' ' ')" WAITING="$(printf '%s\n' "$VERDICTS" | sed -n 's/^[^=]*=False=//p' | sort -u | tr '\n' ' ')" WIDE=no @@ -708,7 +553,7 @@ if [ "$FILL" -ge 1 ]; then kubectl -n "$NS" delete modeldeployments.worker.gpustack.ai "$FILLER" \ --ignore-not-found --wait=false >/dev/null 2>&1 - force_release "$FILLER" + md_force_release "$NS" "$FILLER" AFTER="$(wait_admitted "$SUBJECT" 3)" if [ "$(echo "$AFTER" | wc -w | tr -d ' ')" = 3 ]; then diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-119.sh b/.agents/skills/gpustack-operator-e2e/cases/case-119.sh deleted file mode 100644 index a702f4932..000000000 --- a/.agents/skills/gpustack-operator-e2e/cases/case-119.sh +++ /dev/null @@ -1,570 +0,0 @@ -#!/usr/bin/env bash -# -# CASE 119 — The shared claim mounts the same files in the encoder's and the prefiller's Pods -# (MUTATING, self-recovering) -# AUTO-SKIPS without an InstanceType that reports an admitted accelerator platform, which this -# case builds by injecting a CPU vendor label; its runtime row additionally skips, naming the -# image, when the witness image cannot be pulled. -# -# case-119.sh -# -# Goal: On the platform whose encoder leg runs through a mounted directory rather than the -# network, the encoder writes its embeddings into a claim and the prefiller reads them -# out of the same one. Admission holds the contract on the spec side (one claim, one -# sub-path, one mount path, a writable encoder mount), and this case holds the render -# and the storage side: both role Pods mount THE SAME claim at the same path, and the -# claim itself is one filesystem — a file written through one mount is readable -# through the other. -# -# THE RENDERED MOUNTS AND THE CLAIM'S OWN SHARING ARE TWO HALVES. Reading only the -# rendered specs proves the operator points both roles at one claim name; it cannot -# tell a claim that really shares bytes from two claims sharing a name. Reading only a -# witness exchange proves the claim shares; it cannot tell whether the deployment -# mounts it. So the rows assert the volumes and mounts the two role Pods render, and -# then a pair of case-owned witness Pods — pinned to the one node the pool covers, -# mounting the same claim — writes through one mount and reads back through the other. -# -# THE WITNESSES STAND IN FOR THE ROLE PODS AT RUNTIME, on purpose. The role replicas -# are placeholders on an explicit pause image, which carries no shell: their mounts -# are the render's contract, and the witness pair answers the question a shell-less -# replica cannot — whether the claim the renders name is one filesystem. A -# bytes-through-the-engine-encoder measurement belongs to the accelerator cases. -# -# Environment: Any cluster with a materialized scheduling chain (run case-1 first) and at least one -# Ready, schedulable worker node that is not a control-plane node. NO accelerator: the -# encoder shape reaches admission through the platform an injected CPU vendor label -# builds ("Ascend" — the admitted row whose connector is a mounted directory rather -# than a network leg). The gpustack-settings Secret -# must exist in the system namespace (E2E_SYSTEM_NS, default gpustack-system). The case -# creates its own PV and PVC (a hostPath volume under /tmp on the labeled node, -# manually bound), so no StorageClass is needed. -# -# AUTO-SKIP: the subject needs the platform InstanceType, and the vendor label may not -# take (NFD owns cpu-model.vendor_id and may decline an injected override). The -# witness half additionally needs E2E_C119_WITNESS_IMAGE (default busybox:1.36) to be -# pullable; an image-pull failure is a SKIP naming the image, never a FAIL — a verdict -# about the operator cannot ride on a container nobody could start. -# -# Inputs: MOCKED: one node label (the CPU vendor) injected through a NodeFeature object and -# one settings key (CPU-manufacturer awareness) flipped on — both restored by the -# trap. REAL: one PV, one PVC and one ModelDeployment whose encode and prefill roles -# mount the claim at one path and whose decode role mounts nothing; two one-shot -# witness Pods mounting the same claim, the writer started first and the reader created -# only once the writer has finished. -# -# Expected: - the deployment is admitted and created, and its three role Pods exist (gated -# counts, they never need to run); -# - the encoder Pod renders a volume naming the claim, mounted at /ec-cache and NOT -# read-only; -# - the prefiller Pod renders the same claim name at the same mount path; -# - the decoder Pod renders no volume naming the claim; -# - the PVC reaches Bound against the case's own PV; -# - a file written through the claim by one witness Pod is read back byte-for-byte by -# the other, proving one filesystem behind both mounts. -# -# Cleanup: The trap deletes the deployment, releases any Workload still holding its replicas, -# deletes the witness Pods, the PVC and the PV, the NodeFeature object, and restores -# the settings key to its exact original value, then waits for the chain to settle back: -# the derived type goes only once the setting cache flips and the pool re-derives. -# Idempotent, runs on pass AND fail, safe to re-run. -# -set -uo pipefail - -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" - -NS="${1:?usage: case-119.sh }" -SYSTEM_NS="${E2E_SYSTEM_NS:-gpustack-system}" -SETTING=instance-type-aware-cpu-manufacturer -VENDOR="${E2E_C119_VENDOR:-Ascend}" -PLATFORM="$(printf '%s' "$VENDOR" | tr '[:upper:]' '[:lower:]')" -NF="case119-cpu-vendor" -MD=case119-epd -CLAIM=case119-claim -PV=case119-pv -IMAGE="${E2E_MD_IMAGE:-registry.k8s.io/pause:3.10}" -WITNESS_IMAGE="${E2E_C119_WITNESS_IMAGE:-busybox:1.36}" -MOUNT=/ec-cache -SETTLE_ROUNDS="${E2E_C119_SETTLE:-60}" # 60 x 5s for the settings cache and re-derivation -POD_ROUNDS="${E2E_C119_PODS:-40}" # 40 x 5s for the group to render its Pods - -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } -# shellcheck source=/dev/null -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" - -pick_worker_node() { - kubectl get nodes -o json 2>/dev/null | python3 -c ' -import json,sys -for n in json.load(sys.stdin).get("items",[]): - cs=n.get("status",{}).get("conditions",[]) - if not any(c.get("type")=="Ready" and c.get("status")=="True" for c in cs): - continue - if n.get("spec",{}).get("unschedulable"): - continue - # A CPU-vendor probe only reads back on a node that feeds the CPU-only catalog. An - # acceleratable node feeds the GPU InstanceType instead, whose manufacturer is the - # accelerator vendor, so an injected cpu-model.vendor_id never surfaces here and the - # platform-type poll below times out. Skip those nodes, exactly as case-18 reads the - # CPU pool (acceleratable=false) rather than the accelerator one. - if n.get("metadata",{}).get("labels",{}).get("feature.gpustack.ai/acceleratable") == "true": - continue - ts=n.get("spec",{}).get("taints",[]) - if any(str(t.get("key","")).startswith("node-role.kubernetes.io/") for t in ts): - continue - print(n["metadata"]["name"]) - break -' -} - -epd_platform_type() { - kubectl get instancetypes.worker.gpustack.ai \ - -o jsonpath='{range .items[*]}{.metadata.name}|{.metadata.deletionTimestamp}|{.spec.inactive}|{.status.detail.manufacturer}{"\n"}{end}' \ - 2>/dev/null \ - | while IFS='|' read -r name deleting inactive manufacturer; do - [ -n "$name" ] || continue - [ -z "$deleting" ] || continue - [ "$inactive" = true ] && continue - if [ "$manufacturer" = "$PLATFORM" ]; then - echo "$name" - break - fi - done -} - -ORIG_SETTING="$(kubectl -n "$SYSTEM_NS" get secret gpustack-settings -o jsonpath="{.data.${SETTING}}" 2>/dev/null)" -set_awareness() { # true|false - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"${SETTING}\":\"$(printf '%s' "$1" | base64)\"}}" >/dev/null 2>&1 -} -restore_setting() { - local v=null - [ -n "$ORIG_SETTING" ] && v="\"${ORIG_SETTING}\"" - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"${SETTING}\":${v}}}" >/dev/null 2>&1 -} - -# Restoring the value is not the end of the restore: the worker reads it through a thirty-second -# cache, and the type derived from it outlives the patch by that cache plus one reconcile. Wait for -# the derived type to go, so the next case in a serial run does not list an aware type this run left -# behind (case-18's wait_generic, on this same fixture). A cluster that started with the setting -# already on keeps its derived type, so there is nothing to wait for and the wait is skipped. -settle_back() { - [ "$(printf '%s' "$ORIG_SETTING" | base64 -d 2>/dev/null)" = "true" ] && return 0 - for _ in $(seq 1 "$SETTLE_ROUNDS"); do - [ -z "$(epd_platform_type)" ] && return 0 - sleep 5 - done - return 0 -} - -# One list call, because this runs from the EXIT trap: Kueue holds a finalizer on every Pod of a -# serving group and releases it only when the Workload is deleted (case-50's mechanic). -force_release() { - local row wl uids owners u - uids="$(kubectl -n "$NS" get pods -l "app.kubernetes.io/instance=${MD}" \ - -o jsonpath='{range .items[*]}{.metadata.uid}{"\n"}{end}' 2>/dev/null)" - [ -n "$uids" ] || return 0 - while IFS= read -r row; do - [ -n "$row" ] || continue - wl="${row%%=*}" - owners="${row#*=}" - for u in $uids; do - case " $owners " in *" $u "*) - kubectl -n "$NS" delete workloads.kueue.x-k8s.io "$wl" \ - --ignore-not-found --wait=false >/dev/null 2>&1 - break - ;; - esac - done - done </dev/null) -EOF - - return 0 -} - -INJECTED=false -cleanup() { - echo - echo "[case-119] cleanup" - kubectl -n "$NS" delete pod case119-witness-write case119-witness-read \ - --ignore-not-found --force --grace-period=0 >/dev/null 2>&1 - kubectl -n "$NS" delete modeldeployments.worker.gpustack.ai "$MD" \ - --ignore-not-found --wait=false >/dev/null 2>&1 - for _ in $(seq 1 12); do - sleep 5 - force_release - [ -z "$(kubectl -n "$NS" get pods -l "app.kubernetes.io/instance=${MD}" --no-headers 2>/dev/null)" ] && break - done - kubectl -n "$NS" delete pvc "$CLAIM" --ignore-not-found --wait=false >/dev/null 2>&1 - kubectl delete pv "$PV" --ignore-not-found --wait=false >/dev/null 2>&1 - if [ "$INJECTED" = true ]; then - kubectl -n "$NS" delete nodefeatures.nfd.k8s-sigs.io "$NF" --ignore-not-found >/dev/null 2>&1 - restore_setting - settle_back - fi -} -trap cleanup EXIT - -# ---------------------------------------------------------------- the fixture - -IT="$(epd_platform_type)" -if [ -n "$IT" ]; then - NODE="$(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null | head -n 1)" -else - NODE="$(pick_worker_node)" - if [ -z "$NODE" ]; then - echo "[case-119] no Ready schedulable worker node; this case cannot place its vendor label" >&2 - exit 2 - fi - - echo "[case-119] injecting CPU vendor '${VENDOR}' on node ${NODE} and turning CPU-manufacturer awareness on" - INJECTED=true - - kubectl apply -f - >/dev/null 2>&1 </dev/null)" -for _ in $(seq 1 "$SETTLE_ROUNDS"); do - [ -n "$(kubectl -n "$NS" get localqueue "$ENTRANCE" -o name 2>/dev/null)" ] && break - sleep 5 -done - -record PASS "a CPU-only pool reports an admitted platform" \ - "instancetype ${IT} reports manufacturer '${PLATFORM}'; entrance ${ENTRANCE}" - -# The claim the two roles mount: a hostPath volume under /tmp on the labeled node, manually bound. -# The pool the labeled node feeds is the only pool the deployment names, so every Pod that mounts -# this claim lands on that node and ReadWriteOnce is enough; DirectoryOrCreate spares the case any -# need to reach the node's shell. A storage class is deliberately not used: binding by volumeName -# needs none, and the default class of an arbitrary cluster is not this case's to depend on. -kubectl apply -f - >/dev/null 2>&1 </dev/null 2>&1; then - record PASS "the shared-claim encoder deployment is admitted and created" \ - "encode and prefill mount ${CLAIM} at ${MOUNT} on ${IT}" -else - record FAIL "the shared-claim encoder deployment is admitted and created" \ - "no modeldeployment/${MD} after the apply: $(printf '%s' "${APPLY_OUT:-}" | tr '\n' ' ' | cut -c1-220)" - print_rows - exit 1 -fi - -PODS="" -for _ in $(seq 1 "$POD_ROUNDS"); do - PODS="$(kubectl -n "$NS" get pods -l "app.kubernetes.io/instance=${MD}" --no-headers 2>/dev/null | grep -c . || true)" - [ "${PODS:-0}" -ge 3 ] && break - sleep 5 -done -if [ "${PODS:-0}" -ge 3 ]; then - record PASS "all three role Pods exist" "${PODS} Pods rendered; the mount rows below read specs, not running containers" -else - record FAIL "all three role Pods exist" \ - "only ${PODS:-0} Pod(s) of the group appeared within $((POD_ROUNDS * 5))s, so the mount rows below have nothing to read" -fi - -# ---------------------------------------------------------------- the rendered mounts - -# The claim one role's FIRST Pod mounts, as "claimName|mountPath|readOnly", or "" when it mounts no -# claim. Pending Pods count: the render is what these rows pin, and a gated Pod carries its full -# spec (case-89's reading posture). -# -# EVERY CONTAINER IS SEARCHED, not just the first. A direct decode renders the routing sidecar as -# the FIRST regular container, and reading that one would let the decoder's row pass whatever the -# engine mounts: the sidecar carries no mounts of its own, so a claim the decode role grew would -# still read as "no claim" there. Looking through all of them keeps every row a question about the -# whole Pod. -role_pod_claim() { # - kubectl -n "$NS" get pods \ - -l "app.kubernetes.io/instance=${MD},app.kubernetes.io/component=$1" \ - -o json 2>/dev/null | python3 -c ' -import json,sys -items=json.load(sys.stdin).get("items",[]) -if not items: - print("") - raise SystemExit -pod=items[0] -claims={} -for v in pod["spec"].get("volumes",[]): - p=v.get("persistentVolumeClaim") - if p and p.get("claimName"): - claims[v["name"]]=p["claimName"] -for ctr in pod["spec"].get("containers",[]): - for m in ctr.get("volumeMounts",[]): - claim=claims.get(m.get("name","")) - if claim: - print(claim+"|"+m.get("mountPath","")+"|"+str(m.get("readOnly",False)).lower()) - raise SystemExit -print("")' -} - -ENCODE_CLAIM="$(role_pod_claim encode)" -PREFILL_CLAIM="$(role_pod_claim prefill)" -DECODE_CLAIM="$(role_pod_claim decode)" - -case "$ENCODE_CLAIM" in - "${CLAIM}|${MOUNT}|false") - record PASS "the encoder Pod mounts the claim at the shared path, writable" \ - "rendered ${ENCODE_CLAIM}" - ;; - *) - record FAIL "the encoder Pod mounts the claim at the shared path, writable" \ - "rendered '${ENCODE_CLAIM:-no claim mount}', expected '${CLAIM}|${MOUNT}|false'" - ;; -esac - -case "$PREFILL_CLAIM" in - "${CLAIM}|${MOUNT}|"*) - - # Read-only is the PREFILLER's own choice — it only reads — so the row pins the claim name and - # the path and leaves the flag open, unlike the encoder's row above. - record PASS "the prefiller Pod mounts the same claim at the same path" \ - "rendered ${PREFILL_CLAIM}, matching the encoder's claim and path" - ;; - *) - record FAIL "the prefiller Pod mounts the same claim at the same path" \ - "rendered '${PREFILL_CLAIM:-no claim mount}', expected '${CLAIM}|${MOUNT}|'" - ;; -esac - -if [ -z "$DECODE_CLAIM" ]; then - record PASS "the decoder Pod mounts no claim" \ - "the mounted-directory leg ends at the prefiller, and the decode role declares no volume" -else - record FAIL "the decoder Pod mounts no claim" "rendered ${DECODE_CLAIM}" -fi - -# ---------------------------------------------------------------- the claim is one filesystem - -PVC_PHASE="$(kubectl -n "$NS" get pvc "$CLAIM" -o jsonpath='{.status.phase}' 2>/dev/null)" -if [ "$PVC_PHASE" = Bound ]; then - record PASS "the claim is Bound" "pvc/${CLAIM} bound to pv/${PV}" -else - record FAIL "the claim is Bound" "pvc/${CLAIM} phase='${PVC_PHASE:-}', so no witness can mount it" -fi - -# Two one-shot witnesses pinned to the labeled node: the pool's only node, which is also where the -# hostPath lives. The writer records a token through its mount; the reader cats the same path. That -# round-trip is the reading that separates one shared filesystem from two claims sharing a name. -# -# THE WRITER IS CREATED FIRST AND THE READER ONLY AFTER IT HAS RUN, because the reader cats a file -# that does not exist until the writer has written it: created together, whichever the kubelet starts -# first decides the verdict, and on a cluster that already holds the image the reader usually loses, -# failing a claim that shares perfectly well. Nothing in the manifest, the node or the loop orders -# them, so the case orders them. -if [ "$PVC_PHASE" = Bound ]; then - TOKEN="case119-$(date +%s)" - kubectl apply -f - >/dev/null 2>&1 < /shared/written-by-the-other-mount"] - volumeMounts: - - name: shared - mountPath: /shared - volumes: - - name: shared - persistentVolumeClaim: - claimName: ${CLAIM} -YAML - - # An image that cannot be pulled is a SKIP naming the image (case-81's rule): a container nobody - # could start is a reading nobody took, not a verdict about the claim. - pull_reason() { # - kubectl -n "$NS" get pod "$1" \ - -o jsonpath='{.status.containerStatuses[0].state.waiting.reason}' 2>/dev/null - } - - W_ROUNDS=$((POD_ROUNDS)) - W_OK=no - for _ in $(seq 1 "$W_ROUNDS"); do - case "$(kubectl -n "$NS" get pod case119-witness-write -o jsonpath='{.status.phase}' 2>/dev/null)" in - Succeeded) W_OK=yes; break ;; - Failed) break ;; - esac - case "$(pull_reason case119-witness-write)" in ImagePullBackOff | ErrImagePull) break ;; esac - sleep 5 - done - - if [ "$W_OK" != yes ]; then - case "$(pull_reason case119-witness-write)" in - ImagePullBackOff | ErrImagePull) - record SKIP "a file written through one mount is read back through the other" \ - "the witness image '${WITNESS_IMAGE}' could not be pulled ($(pull_reason case119-witness-write)), so no reading was taken; the render rows above stand on their own" - ;; - *) - record FAIL "a file written through one mount is read back through the other" \ - "the writer witness never completed (phase '$(kubectl -n "$NS" get pod case119-witness-write -o jsonpath='{.status.phase}' 2>/dev/null)')" - ;; - esac - else - # The file exists now, so the reader is safe to start. - kubectl apply -f - >/dev/null 2>&1 </dev/null)" in - Succeeded | Failed) R_OK=yes; break ;; - esac - case "$(pull_reason case119-witness-read)" in ImagePullBackOff | ErrImagePull) break ;; esac - sleep 5 - done - - READ_BACK="" - if [ "$R_OK" = yes ]; then - READ_BACK="$(kubectl -n "$NS" logs case119-witness-read -c read 2>/dev/null)" - fi - if [ "$READ_BACK" = "$TOKEN" ]; then - record PASS "a file written through one mount is read back through the other" \ - "witness pair exchanged '${READ_BACK}' through pvc/${CLAIM}: one filesystem behind both mounts" - else - record FAIL "a file written through one mount is read back through the other" \ - "reader witness returned '${READ_BACK:-}', expected '${TOKEN}'" - fi - fi -fi - -# Results. -print_rows -[ "$FAILS" -eq 0 ] || { echo "[case-119] ${FAILS} check(s) FAILED"; exit 1; } -echo "[case-119] all checks passed" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-12.sh b/.agents/skills/gpustack-operator-e2e/cases/case-12.sh deleted file mode 100755 index babb19d2f..000000000 --- a/.agents/skills/gpustack-operator-e2e/cases/case-12.sh +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env bash -# -# CASE 12 — Logically sliceable Instance webhook: slice-% sizes CPU/RAM, accelerator pinned to 1 (MUTATING, self-recovering; AUTO-SKIPS without a real logically sliceable accelerated pool) -# -# case-12.sh -# -# Goal: On a SLICEABLE accelerated InstanceType, assert the Instance admission webhooks — -# Q1: Default scales the per-card unit CPU/RAM to the slice percentages (compute % sizes -# CPU, memory % sizes RAM), floors fractions, never below 1, and pins the accelerator to 1; -# Q2: a lone memory percentage is mirrored to the compute percentage, so CPU is sized too; -# Q3: Validate rejects a sliceable request whose accelerator count is not 1 (the slice is -# expressed through the percentages, not the card count). -# Environment: Needs a REAL LOGICALLY sliceable accelerated pool (an InstanceType whose observed -# Status.Detail reports a logical-slice count). A partition-only pool does NOT qualify: -# a slice percentage against a pool that offers no logical slicing is now rejected at -# admission, where it used to be served silently as a whole card. AUTO-SKIPS (exit 0) -# when none is present. Only the admission result is asserted (probes are deleted at once). -# Inputs: - the real sliceable accelerated InstanceType, READ-ONLY (its immutable per-card unit spec -# drives the expected slice math — the unit is not pinned/mutated); -# - real probes: INST_OK (mem%=25, cores%=25), INST_MEMONLY (mem%=50, cores unset), -# INST_BAD (mem%=25, cores%=25, accelerator=2). -# Expected: - Q1 — INST_OK persists accelerator=1, cpu=unitCPU×25% (floor, min 1), ram=unitRAM×25%; -# - Q2 — INST_MEMONLY mirrors cores%=50 and sizes cpu=unitCPU×50%, ram=unitRAM×50%; -# - Q3 — INST_BAD is REJECTED with the webhook's message "accelerator request must be exactly 1 -# for a sliced request"; a denial with any other text fails. -# Cleanup: Trap deletes the three test Instances (the InstanceType is never mutated). -set -uo pipefail - -# Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail -# on transport alone, and a check that takes such a failure for an answer reports a verdict -# about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" - -NS="${1:?usage: case-12.sh }" -INST_OK=gpustack-e2e-slice-ok -INST_MEMONLY=gpustack-e2e-slice-memonly -INST_BAD=gpustack-e2e-slice-badacc -IT=""; UNIT_CPU=""; UNIT_RAM="" # the real sliceable IT (read-only) + its per-card unit spec - -restore() { - echo - echo "[case-12] cleanup: deleting test Instances" - kubectl -n default delete instance "$INST_OK" "$INST_MEMONLY" "$INST_BAD" --ignore-not-found 2>/dev/null || true - sleep 5 -} -trap restore EXIT - -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } - -# 1. Find a real LOGICALLY sliceable accelerated InstanceType (observed Status.Detail carries a -# logical-slice count) and read its per-card unit spec. AUTO-SKIP when none is present. -read -r IT UNIT_CPU UNIT_RAM <<<"$(kubectl get instancetypes.worker.gpustack.ai -o json 2>/dev/null | python3 -c " -import json,sys -for it in json.load(sys.stdin).get('items',[]): - s=it.get('spec',{}); d=it.get('status',{}).get('detail',{}); sd=d.get('slicedDetail',{}); u=s.get('unitResources',{}) - # LOGICALLY sliceable only: a hardware-partitioned card serves no logical slice. - sliceable=(sd.get('logical',{}).get('count',0) or 0)>0 - if s.get('acceleratable') and sliceable: - print(it['metadata']['name'], u.get('cpu',''), u.get('ram','')); break -")" -if [ -z "$IT" ]; then - echo "== CASE 12 — SKIPPED ==" - echo "No real logically sliceable accelerated InstanceType (observed Status.Detail reports a logical-slice" - echo "count) — this case needs real accelerator hardware whose cards are logically sliceable, i.e. NOT in a" - echo "hardware partitioning mode. Run it on such a cluster." - exit 0 -fi -echo "[case-12] logically sliceable InstanceType: ${IT} (per-card unit cpu=${UNIT_CPU} ram=${UNIT_RAM})" - -# 2. Derive the expected slice sizing from the pool's per-card unit spec (immutable — read, don't pin). -# CPU floors to an integer core, never below 1; RAM scales linearly. Requires an integer unit CPU -# and a Gi unit RAM (what a real accelerated pool derives) — SKIP on an unexpected unit shape so a -# slice check never fails for the wrong reason. -case "$UNIT_CPU" in ''|*[!0-9]*) echo "== CASE 12 — SKIPPED =="; echo "unit CPU '${UNIT_CPU}' is not an integer core count — cannot compute deterministic slice math"; exit 0;; esac -case "$UNIT_RAM" in *Gi) ;; *) echo "== CASE 12 — SKIPPED =="; echo "unit RAM '${UNIT_RAM}' is not in Gi — cannot compute deterministic slice math"; exit 0;; esac -RAM_N="${UNIT_RAM%Gi}" -cpu_pct() { local v=$(( UNIT_CPU * $1 / 100 )); [ "$v" -lt 1 ] && v=1; echo "$v"; } -ram_pct() { echo "$(( RAM_N * $1 / 100 ))Gi"; } -EXP_CPU_OK=$(cpu_pct 25); EXP_RAM_OK=$(ram_pct 25) -EXP_CPU_MEM=$(cpu_pct 50); EXP_RAM_MEM=$(ram_pct 50) -echo "[case-12] expected: 25% slice → cpu=${EXP_CPU_OK} ram=${EXP_RAM_OK}; 50% slice → cpu=${EXP_CPU_MEM} ram=${EXP_RAM_MEM}" - -# mk_slice NAME MEM_PCT CORES_PCT [ACCELERATOR] — apply a sliced Instance (accelerator omitted -# when the 4th arg is empty, so Default fills it). Prints kubectl's combined output; exits non-zero -# when admission rejects. A 0 cores-percentage means "unset" (exercises the mirror-from-memory path). -mk_slice() { - { - cat <&1 -} - -get_res() { kubectl -n default get instance "$1" -o jsonpath="{.spec.resources.$2}" 2>/dev/null; } - -# Q1 — a 25% slice: accelerator pinned to 1, CPU = unitCPU × 25%, RAM = unitRAM × 25%. -echo "[case-12] Q1 apply: $(mk_slice "$INST_OK" 25 25)" -acc=""; for _ in $(seq 1 10); do acc=$(get_res "$INST_OK" accelerator); [ -n "$acc" ] && break; sleep 1; done -cpu=$(get_res "$INST_OK" cpu); ram=$(get_res "$INST_OK" ram) -if [ "$acc" = "1" ] && [ "$cpu" = "$EXP_CPU_OK" ] && [ "$ram" = "$EXP_RAM_OK" ]; then - record PASS "Default scales slice to unit CPU/RAM, accelerator=1" "acc=${acc} cpu=${cpu} ram=${ram} (unit ${UNIT_CPU}/${UNIT_RAM} × 25%)" -else - record FAIL "Default scales slice to unit CPU/RAM, accelerator=1" "acc='${acc:-?}' cpu='${cpu:-?}' ram='${ram:-?}', want 1/${EXP_CPU_OK}/${EXP_RAM_OK}" -fi - -# Q2 — a lone memory percentage (50%) is mirrored to compute, so CPU is sized too. -echo "[case-12] Q2 apply: $(mk_slice "$INST_MEMONLY" 50 0)" -cores2=""; for _ in $(seq 1 10); do cores2=$(get_res "$INST_MEMONLY" acceleratorSlicedCoresPercentage); [ -n "$cores2" ] && break; sleep 1; done -cpu2=$(get_res "$INST_MEMONLY" cpu); ram2=$(get_res "$INST_MEMONLY" ram) -if [ "$cores2" = "50" ] && [ "$cpu2" = "$EXP_CPU_MEM" ] && [ "$ram2" = "$EXP_RAM_MEM" ]; then - record PASS "lone memory % mirrored to compute, sizes CPU" "cores%=${cores2} cpu=${cpu2} ram=${ram2}" -else - record FAIL "lone memory % mirrored to compute, sizes CPU" "cores%='${cores2:-?}' cpu='${cpu2:-?}' ram='${ram2:-?}', want 50/${EXP_CPU_MEM}/${EXP_RAM_MEM}" -fi - -# Q3 — Validate rejects a sliceable request whose accelerator count is not 1. -# Match the webhook's own message: any admission denial says "denied the request", so a looser -# pattern would pass on a rejection for an unrelated reason. -Q3_MSG='accelerator request must be exactly 1 for a sliced request' -err=$(mk_slice "$INST_BAD" 25 25 2); rc=$? -if [ "$rc" -ne 0 ] && echo "$err" | grep -qF "$Q3_MSG"; then - record PASS "Validate rejects accelerator != 1 on sliceable" "rejected: $(echo "$err" | grep -oF "$Q3_MSG" | head -1)" -elif [ "$rc" -ne 0 ]; then - record FAIL "Validate rejects accelerator != 1 on sliceable" "rejected, but not with '${Q3_MSG}': $(echo "$err" | head -1)" -else - kubectl -n default delete instance "$INST_BAD" --ignore-not-found >/dev/null 2>&1 || true - record FAIL "Validate rejects accelerator != 1 on sliceable" "accepted (rc=${rc}) — sliceable accelerator must be 1" -fi - -echo -echo "== CASE 12 — Logically sliceable Instance webhook: slice-% sizes CPU/RAM, accelerator pinned to 1 ==" -{ - echo "STATUS|CHECK|OBJECT" - printf '%s\n' "${ROWS[@]}" -} | column -t -s '|' - -if [ "$FAILS" -ne 0 ]; then - echo - echo "FAILED ${FAILS} check(s). On a sliceable InstanceType the Default webhook must size CPU/RAM by the" - echo "slice percentages of ONE card's unit (compute%→CPU, memory%→RAM, floor, min 1) and Validate must" - echo "pin the accelerator count to 1. Diagnose: kubectl -n ${NS} logs deploy/gpustack-operator-worker --tail=200" - exit 1 -fi -echo "CASE 12 PASS" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-120.sh b/.agents/skills/gpustack-operator-e2e/cases/case-120.sh index 46db2477b..7cccdf1cc 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-120.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-120.sh @@ -10,10 +10,10 @@ # # Goal: This is the functional reading of the admitted matrix rows # llm-d-router + vLLM + NVIDIA CUDA (or AMD ROCm) + ECCPUConnector. Case 116 pins the -# ADMISSION of the shape, case 119 pins the shared-directory render of the OTHER row; +# ADMISSION of the shape, case 117 pins the shared-directory render of the OTHER row; # nothing yet proves these run. The encoder's output reaches the prefiller over NIXL # here — -# the sidecar on the decode Pod pulls it — so the rendered mount rows of case 119 +# the sidecar on the decode Pod pulls it — so the rendered mount rows of case 117 # say nothing about this leg, and the only reading that says anything is a real # engine moving real encodings: three roles Ready on real NVIDIA or AMD cards, the # rendered leg naming the network connector on both halves that carry it, and one @@ -112,8 +112,8 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:-}" if [ -z "$NS" ]; then @@ -121,6 +121,12 @@ if [ -z "$NS" ]; then exit 2 fi +# The image override and the live serving group are read through the library: the registry the +# operator prefixes onto every image, the pin and the restore of the router and the sidecar, the +# role Pods, the router and the connector document. +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_epd-lib.sh" + MD=case120-epd IT="${E2E_C120_INSTANCE_TYPE:-}" # The engine image default depends on the platform the named InstanceType reports, so the value is @@ -147,29 +153,15 @@ SETTLE_BOUND="${E2E_C120_SETTLE_BOUND:-180}" # The operator prefixes `container-registry` onto every image it chooses, and the two settings below # take a reference of the form `namespace/name:tag`. A value that already names a host would render as # `//...` on a cluster that sets a registry, so drop the host there. -registry_relative() { - case "${1%%/*}" in - *.*|*:*|localhost) printf '%s' "${1#*/}" ;; - *) printf '%s' "$1" ;; - esac -} -CLUSTER_REGISTRY="$(kubectl -n "$SYSTEM_NS" get secret gpustack-settings \ - -o jsonpath='{.data.container-registry}' 2>/dev/null | base64 -d 2>/dev/null)" -if [ -n "$CLUSTER_REGISTRY" ]; then - ROUTER_IMAGE="$(registry_relative "$ROUTER_IMAGE")" - SIDECAR_IMAGE="$(registry_relative "$SIDECAR_IMAGE")" +if [ -n "$(epd_cluster_registry "$SYSTEM_NS")" ]; then + ROUTER_IMAGE="$(epd_registry_relative "$ROUTER_IMAGE")" + SIDECAR_IMAGE="$(epd_registry_relative "$SIDECAR_IMAGE")" fi # 30 minutes is not slack: this row loads a vision-language model on three roles, and the encoder # compiles a model it then never answers with. A smaller bound would report a slow cluster as a # broken leg. PROBE_NAME=case120-probe -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } -# shellcheck source=/dev/null -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" - skip() { echo "[case-120] SKIP: $1"; echo "NOTHING WAS VERIFIED"; exit 0; } # --- input gates (exit 2) --------------------------------------------------- @@ -232,54 +224,8 @@ echo "[case-120] pool '${IT}': ${FREE_EX} free exclusive cards, need ${NEED_CARD # set, and deleting the former would leave this run's baseline changed for every case after it. ROUTER_SETTING=model-deployment-router-image SIDECAR_SETTING=model-deployment-routing-sidecar-image -ORIG_ROUTER="$(kubectl -n "$SYSTEM_NS" get secret gpustack-settings \ - -o jsonpath="{.data.${ROUTER_SETTING}}" 2>/dev/null)" -ORIG_SIDECAR="$(kubectl -n "$SYSTEM_NS" get secret gpustack-settings \ - -o jsonpath="{.data.${SIDECAR_SETTING}}" 2>/dev/null)" - -set_image_setting() { # key value - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"$1\":\"$(printf '%s' "$2" | base64)\"}}" >/dev/null 2>&1 -} - -restore_image_setting() { # key original - local v=null - [ -n "$2" ] && v="\"$2\"" - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"$1\":${v}}}" >/dev/null 2>&1 -} - -# Restoring the bytes is not the end of the restore: the worker reads settings through a cache and -# re-renders the router Deployment by the value, so a case that returned while the router still ran -# this case's image would hand the next one a router it did not choose. Wait for the restore to be -# READ BACK, which is what the Deployment's own image shows. -router_image() { - kubectl -n "$NS" get deploy "${MD}-router" \ - -o jsonpath='{.spec.template.spec.containers[*].image}' 2>/dev/null -} - -settle_back() { - local deadline=$((SECONDS + SETTLE_BOUND)) image - while [ "$SECONDS" -lt "$deadline" ]; do - image="$(router_image)" - # No router was ever rendered — an apply that was refused, or a run that never reached it — so - # nothing carries this case's image and there is nothing to wait for. - [ -n "$image" ] || return 0 - # The restored value is not always comparable: a key the cluster shipped absent is restored by - # deleting it, and what the Deployment then carries is the operator's own default, which this - # case never knew. "No longer this case's image" is the condition both spellings satisfy. - case "$image" in - *"$ROUTER_IMAGE"*) ;; - *) return 0 ;; - esac - sleep 5 - done - # The bound is spent and the router still runs this case's image: the restore was written but the - # worker has not read it back. Cleanup cannot record a row from here, and returning silently would - # hand the next case a router it did not choose without saying so, so say it on stderr. - echo "[case-120] settle bound ${SETTLE_BOUND}s spent with the router still on '${ROUTER_IMAGE}'; the settings restore has not been read back" >&2 - return 0 -} +# The pin, the restore and the wait for the router to be re-rendered are the library's: +# epd_pin_router_sidecar_images, epd_restore_router_sidecar_images and epd_settle_router_image. force_release() { local md="$1" row wl uids u @@ -304,17 +250,19 @@ $(kubectl -n "$NS" get workloads.kueue.x-k8s.io \ EOF return 0 } - -# THE SETTINGS GO BACK BEFORE THE DEPLOYMENT IS DELETED, which is the order that lets settle_back +# THE SETTINGS GO BACK BEFORE THE DEPLOYMENT IS DELETED, which is the order that lets the settle # prove the restore: while the ModelDeployment stands, its router Deployment stands with it, and # the image on that Deployment is the one thing in the cluster that shows what the worker has read # since the patch. Deleting the deployment first would remove the only witness. cleanup() { echo echo "[case-120] cleanup" - restore_image_setting "$ROUTER_SETTING" "$ORIG_ROUTER" - restore_image_setting "$SIDECAR_SETTING" "$ORIG_SIDECAR" - settle_back + epd_restore_router_sidecar_images "$SYSTEM_NS" "$ROUTER_SETTING" "$SIDECAR_SETTING" + # The bound is spent and the router still runs this case's image: the restore was written but the + # worker has not read it back. Cleanup cannot record a row from here, and returning silently would + # hand the next case a router it did not choose without saying so, so say it on stderr. + epd_settle_router_image "$NS" "$MD" "$SETTLE_BOUND" "$ROUTER_IMAGE" || + echo "[case-120] settle bound ${SETTLE_BOUND}s spent with the router still on '${ROUTER_IMAGE}'; the settings restore has not been read back" >&2 # By name only. kubectl refuses a name combined with a selector ("name cannot be provided when a # selector is specified") and stderr is silenced below, so a stale probe would survive every run # and take the name the next run's kubectl run needs. The e2e-probe label is still on the Pod; it @@ -331,15 +279,15 @@ cleanup() { # cannot leave the cluster running a router image this run chose. trap cleanup EXIT -set_image_setting "$ROUTER_SETTING" "$ROUTER_IMAGE" -set_image_setting "$SIDECAR_SETTING" "$SIDECAR_IMAGE" +epd_pin_router_sidecar_images "$SYSTEM_NS" "$ROUTER_SETTING" "$SIDECAR_SETTING" \ + "$ROUTER_IMAGE" "$SIDECAR_IMAGE" # --- fixture ---------------------------------------------------------------- # The decode role carries the sidecar that pulls encodings over NIXL, so the deployment needs the # router: an encoder role without one is refused at admission. No claim is mounted anywhere: this # row's connector moves embeddings over the network, and a claim here would be the OTHER row's -# subject (case 119). +# subject (case 117). APPLY_OUT="$(cat <&1 apiVersion: worker.gpustack.ai/v1alpha1 kind: ModelDeployment @@ -415,52 +363,13 @@ fi # A pull failure on any of the three images is a SKIP naming the image (case-81's rule): a # container nobody could start is not a verdict about the leg. The bound is generous because a -# vision-language model is loaded three times over. -pull_reason() { # - kubectl -n "$NS" get pods -l "app.kubernetes.io/instance=${MD},app.kubernetes.io/component=$1" \ - -o jsonpath='{range .items[*]}{.status.containerStatuses[*].state.waiting.reason}{" "}{end}' 2>/dev/null -} - -wait_pods() { # -> pod name on stdout once Ready, rc!=0 on timeout - local comp="$1" deadline=$((SECONDS + READY_BOUND)) pods ready p - while [ "$SECONDS" -lt "$deadline" ]; do - pods="$(kubectl -n "$NS" get pods \ - -l "app.kubernetes.io/instance=${MD},app.kubernetes.io/component=${comp}" \ - -o json 2>/dev/null)" - ready="$(printf '%s' "$pods" | jq -r \ - '[.items[] | select([.status.conditions // [] | .[] | select(.type=="Ready" and .status=="True")] | length > 0)] | length')" - if [ "${ready:-0}" -ge 1 ]; then - printf '%s' "$pods" | jq -r '.items[0].metadata.name' - return 0 - fi - for p in ImagePullBackOff ErrImagePull; do - case "$(pull_reason "$comp")" in - *"$p"*) return 2 ;; - esac - done - sleep 15 - done - return 1 -} - -# Each role is polled separately and the exit code is read IMMEDIATELY, because a pull failure on -# the first role means the other two polls would sit out the whole bound before anything reported -# it. 2 is "an image would not pull", which is an environment answer; 1 is a timeout. -poll_role() { # -> sets POD_ and returns the wait_pods code - local comp="$1" pod rc - pod="$(wait_pods "$comp")"; rc=$? - case "$comp" in - encode) EN_POD="$pod" ;; - prefill) PF_POD="$pod" ;; - decode) DC_POD="$pod" ;; - esac - return "$rc" -} +# vision-language model is loaded three times over. The reading of a role's Pods is the library's: +# epd_pull_reason, epd_wait_pods and epd_poll_role. EN_POD=""; PF_POD=""; DC_POD="" UNPULLABLE_COMPONENT="" for comp in encode prefill decode; do - poll_role "$comp"; rc=$? + epd_poll_role "$NS" "$MD" "$READY_BOUND" "$comp"; rc=$? if [ "$rc" = 2 ]; then UNPULLABLE_COMPONENT="$comp" break @@ -469,7 +378,7 @@ done if [ -n "$UNPULLABLE_COMPONENT" ]; then record SKIP "the three role Pods reach Ready" \ - "the ${UNPULLABLE_COMPONENT} Pod waits on $(pull_reason "$UNPULLABLE_COMPONENT" | tr -s ' '): no container started, so the leg was never exercised. Render the images and re-run" + "the ${UNPULLABLE_COMPONENT} Pod waits on $(epd_pull_reason "$NS" "$MD" "$UNPULLABLE_COMPONENT" | tr -s ' '): no container started, so the leg was never exercised. Render the images and re-run" print_rows exit 0 fi @@ -488,62 +397,14 @@ done # role poll can see. It is the environment answer the role images already get, and reading it as an # operator defect would cost the whole bound and then be wrong. # -# The router's Pods are selected by the router Deployment's OWN selector rather than by a label -# spelled here. The router carries the same instance label the roles carry, plus a router key of its -# own, so an instance-only selector would return the role Pods too and learn nothing about the -# router; the Deployment's selector is the one that cannot drift from what was rendered. -router_pods() { - local sel - # shellcheck disable=SC2016 # the dollar names are kubectl jsonpath map variables, and must not expand - sel="$(kubectl -n "$NS" get deploy "${MD}-router" \ - -o go-template='{{range $k,$v := .spec.selector.matchLabels}}{{$k}}={{$v}},{{end}}' 2>/dev/null | sed 's/,$//')" - [ -n "$sel" ] || return 0 - kubectl -n "$NS" get pods -l "$sel" -o json 2>/dev/null -} - -# What any router container is WAITING on, joined; empty when none is waiting on an image. Read for -# the same two reasons wait_pods reads them on the roles. -router_pull_reason() { - router_pods | jq -r \ - '[.items[]? | .status.containerStatuses[]? | .state.waiting?.reason // empty] | join(" ")' 2>/dev/null -} - -# The Ready router Pod's name, or empty when the router is not up. Read once when the verdict is -# formed, because by then the bound is spent and the router's state is settled. -router_pod() { - router_pods | jq -r \ - '[.items[]? | select([.status.conditions // [] | .[] | select(.type=="Ready" and .status=="True")] | length > 0) | .metadata.name] | if length > 0 then .[0] else "" end' 2>/dev/null -} - -# The endpoint is written once the ROUTER's OWN Deployment reports a ready replica — it is the -# router's status the operator copies, not a fact about the role Pods — so a single read here -# races that writer by design, and the wait is bounded because the router is a fourth workload no -# role poll above covers. Prints the endpoint and returns 0 on arrival, 2 when the router image -# would not pull (the same code wait_pods returns for a role image), 1 once the bound is spent. -wait_endpoint() { - local deadline=$((SECONDS + READY_BOUND)) ep p - while [ "$SECONDS" -lt "$deadline" ]; do - ep="$(kubectl -n "$NS" get modeldeployments.worker.gpustack.ai "$MD" \ - -o jsonpath='{.status.endpoint}' 2>/dev/null)" - if [ -n "$ep" ]; then - printf '%s' "$ep" - return 0 - fi - for p in ImagePullBackOff ErrImagePull; do - case "$(router_pull_reason)" in - *"$p"*) return 2 ;; - esac - done - sleep 5 - done - return 1 -} +# The reading of the router is the library's: epd_router_pods, epd_router_pull_reason, +# epd_router_pod and epd_wait_endpoint. ALL_READY=no [ -n "$EN_POD" ] && [ -n "$PF_POD" ] && [ -n "$DC_POD" ] && ALL_READY=yes -ENDPOINT="$(wait_endpoint)"; ENDPOINT_RC=$? -ROUTER_POD="$(router_pod)" +ENDPOINT="$(epd_wait_endpoint "$NS" "$MD" "$READY_BOUND")"; ENDPOINT_RC=$? +ROUTER_POD="$(epd_router_pod "$NS" "$MD")" # The FAIL verdict needs the WHOLE chain up, the router included. A router that never came up # publishes no front door either, and calling that a defect would blame the operator for an image # it was told to run. @@ -556,7 +417,7 @@ elif [ "$ENDPOINT_RC" = 2 ]; then # An image that would not pull is not a verdict about the leg: no container started, so there was # never a front door to publish. Same shape as the role-image SKIP above, naming the image. record SKIP "endpoint" \ - "the router Pod waits on $(router_pull_reason | tr -s ' '), so router image '${ROUTER_IMAGE}' would not pull and no front door could be published. Render the image and re-run" + "the router Pod waits on $(epd_router_pull_reason "$NS" "$MD" | tr -s ' '), so router image '${ROUTER_IMAGE}' would not pull and no front door could be published. Render the image and re-run" elif [ "$CHAIN_UP" = yes ]; then # Every Pod came up, the router's included, and the controller still publishes no front door. # Nothing else explains that, so it is a defect in what the operator publishes rather than an @@ -584,28 +445,11 @@ fi # The document is the argument AFTER --ec-transfer-config, which is how the operator renders it: # the flag has no environment equivalent, so it is always two argv entries. Reading the pair off -# the live argv rather than pattern-matching the JSON keeps the row a question about the flag. -ec_document() { # - kubectl -n "$NS" get pod "$1" -o json 2>/dev/null | jq -r \ - --arg f "--ec-transfer-config" \ - '[(.spec.containers[] | select(.name=="main") | (.command // []) + (.args // []))][0] // [] - | (index($f) // -1) as $i | if $i < 0 then "" else .[$i + 1] end' 2>/dev/null -} - -ec_env() { # - kubectl -n "$NS" get pod "$1" -o json 2>/dev/null | jq -r \ - --arg n "$2" '[(.spec.containers[] | select(.name=="main") | .env // [])[] | select(.name==$n)][0].value // ""' 2>/dev/null -} - -# The downward-API half of the same read: the field selector the kubelet resolves, not the value it -# resolved. Rendered by valueFrom.fieldRef.fieldPath, so the variable has no .value to read. -ec_env_field_path() { # - kubectl -n "$NS" get pod "$1" -o json 2>/dev/null | jq -r \ - --arg n "$2" '[(.spec.containers[] | select(.name=="main") | .env // [])[] | select(.name==$n)][0].valueFrom.fieldRef.fieldPath // ""' 2>/dev/null -} +# the live argv rather than pattern-matching the JSON keeps the row a question about the flag. The +# reading itself is the library's: epd_ec_document, epd_ec_env and epd_ec_env_field_path. if [ -n "$EN_POD" ]; then - DOC="$(ec_document "$EN_POD")" + DOC="$(epd_ec_document "$NS" "$EN_POD")" if [ -n "$DOC" ]; then if printf '%s' "$DOC" | jq -e '.ec_connector == "ECCPUConnector" and .ec_role == "ec_producer"' \ >/dev/null 2>&1; then @@ -622,9 +466,9 @@ if [ -n "$EN_POD" ]; then # # The port half is the opposite: a plain string the renderer owns outright, so .value is the # right key and the only one there is. - SIDE_HOST_FIELD="$(ec_env_field_path "$EN_POD" VLLM_EC_SIDE_CHANNEL_HOST)" - SIDE_PORT="$(ec_env "$EN_POD" VLLM_EC_SIDE_CHANNEL_PORT)" - RUNNER="$(ec_env "$EN_POD" VLLM_USE_V2_MODEL_RUNNER)" + SIDE_HOST_FIELD="$(epd_ec_env_field_path "$NS" "$EN_POD" VLLM_EC_SIDE_CHANNEL_HOST)" + SIDE_PORT="$(epd_ec_env "$NS" "$EN_POD" VLLM_EC_SIDE_CHANNEL_PORT)" + RUNNER="$(epd_ec_env "$NS" "$EN_POD" VLLM_USE_V2_MODEL_RUNNER)" POD_IP="$(kubectl -n "$NS" get pod "$EN_POD" -o jsonpath='{.status.podIP}' 2>/dev/null)" if [ "$SIDE_HOST_FIELD" = status.podIP ] && [ -n "$POD_IP" ] && [ -n "$SIDE_PORT" ]; then record PASS "encoder side channel" \ @@ -645,7 +489,7 @@ if [ -n "$EN_POD" ]; then fi if [ -n "$PF_POD" ]; then - DOC="$(ec_document "$PF_POD")" + DOC="$(epd_ec_document "$NS" "$PF_POD")" if [ -n "$DOC" ]; then if printf '%s' "$DOC" | jq -e '.ec_connector == "ECCPUConnector" and .ec_role == "ec_consumer"' \ >/dev/null 2>&1; then @@ -654,7 +498,7 @@ if [ -n "$PF_POD" ]; then record FAIL "prefiller document" "got: ${DOC}" fi # A consumer that rendered the side channel would advertise an address it never binds. - SIDE_PORT="$(ec_env "$PF_POD" VLLM_EC_SIDE_CHANNEL_PORT)" + SIDE_PORT="$(epd_ec_env "$NS" "$PF_POD" VLLM_EC_SIDE_CHANNEL_PORT)" if [ -z "$SIDE_PORT" ]; then record PASS "prefiller binds no side channel" \ "VLLM_EC_SIDE_CHANNEL_PORT is absent on ${PF_POD}, which is right: a consumer does not bind" @@ -825,4 +669,4 @@ check_stage_log "the decoder pulled the encodings" "$STAGE_DC" routing-proxy 'pr # Results. print_rows [ "$FAILS" -eq 0 ] || { echo "[case-120] ${FAILS} check(s) FAILED"; exit 1; } -echo "[case-120] all checks passed" \ No newline at end of file +echo "[case-120] all checks passed" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-121.sh b/.agents/skills/gpustack-operator-e2e/cases/case-121.sh index f071f8dff..701f84308 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-121.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-121.sh @@ -106,8 +106,8 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:-}" if [ -z "$NS" ]; then @@ -115,6 +115,12 @@ if [ -z "$NS" ]; then exit 2 fi +# The image override and the live serving group are read through the library: the registry the +# operator prefixes onto every image, the pin and the restore of the router and the sidecar, the +# role Pods, the router and the connector document. +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_epd-lib.sh" + MD=case121-epd IT="${E2E_C121_INSTANCE_TYPE:-}" CLAIM="${E2E_C121_CLAIM:-}" @@ -136,26 +142,12 @@ SETTLE_BOUND="${E2E_C121_SETTLE_BOUND:-180}" # The operator prefixes `container-registry` onto every image it chooses, and the two settings below # take a reference of the form `namespace/name:tag`. A value that already names a host would render as # `//...` on a cluster that sets a registry, so drop the host there. -registry_relative() { - case "${1%%/*}" in - *.*|*:*|localhost) printf '%s' "${1#*/}" ;; - *) printf '%s' "$1" ;; - esac -} -CLUSTER_REGISTRY="$(kubectl -n "$SYSTEM_NS" get secret gpustack-settings \ - -o jsonpath='{.data.container-registry}' 2>/dev/null | base64 -d 2>/dev/null)" -if [ -n "$CLUSTER_REGISTRY" ]; then - ROUTER_IMAGE="$(registry_relative "$ROUTER_IMAGE")" - SIDECAR_IMAGE="$(registry_relative "$SIDECAR_IMAGE")" +if [ -n "$(epd_cluster_registry "$SYSTEM_NS")" ]; then + ROUTER_IMAGE="$(epd_registry_relative "$ROUTER_IMAGE")" + SIDECAR_IMAGE="$(epd_registry_relative "$SIDECAR_IMAGE")" fi PROBE_NAME=case121-probe -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } -# shellcheck source=/dev/null -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" - # A skip before anything was recorded is prose, and a skip after it is a row: the answer this run # reached about its own input belongs in the table beside the passes it already recorded, so the # rows are printed before the case leaves either way. `${#ROWS[@]}` is the same emptiness test @@ -241,54 +233,8 @@ PVC_PHASE="$(printf '%s' "$PVC_JSON" | jq -r '.status.phase // ""')" ROUTER_SETTING=model-deployment-router-image SIDECAR_SETTING=model-deployment-routing-sidecar-image -ORIG_ROUTER="$(kubectl -n "$SYSTEM_NS" get secret gpustack-settings \ - -o jsonpath="{.data.${ROUTER_SETTING}}" 2>/dev/null)" -ORIG_SIDECAR="$(kubectl -n "$SYSTEM_NS" get secret gpustack-settings \ - -o jsonpath="{.data.${SIDECAR_SETTING}}" 2>/dev/null)" - -set_image_setting() { # key value - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"$1\":\"$(printf '%s' "$2" | base64)\"}}" >/dev/null 2>&1 -} - -restore_image_setting() { # key original - local v=null - [ -n "$2" ] && v="\"$2\"" - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"$1\":${v}}}" >/dev/null 2>&1 -} - -# Restoring the bytes is not the end of the restore: the worker reads settings through a cache and -# re-renders the router Deployment by the value, so returning while the router still ran this -# case's image would hand the next case a router it did not choose. Wait for the restore to be READ -# BACK, which is what the Deployment's own image shows. -router_image() { - kubectl -n "$NS" get deploy "${MD}-router" \ - -o jsonpath='{.spec.template.spec.containers[*].image}' 2>/dev/null -} - -settle_back() { - local deadline=$((SECONDS + SETTLE_BOUND)) image - while [ "$SECONDS" -lt "$deadline" ]; do - image="$(router_image)" - # No router was ever rendered — an apply that was refused, or a run that never reached it — so - # nothing carries this case's image and there is nothing to wait for. - [ -n "$image" ] || return 0 - # The restored value is not always comparable: a key the cluster shipped absent is restored by - # deleting it, and what the Deployment then carries is the operator's own default, which this - # case never knew. "No longer this case's image" is the condition both spellings satisfy. - case "$image" in - *"$ROUTER_IMAGE"*) ;; - *) return 0 ;; - esac - sleep 5 - done - # The bound is spent and the router still runs this case's image: the restore was written but the - # worker has not read it back. Cleanup cannot record a row from here, and returning silently would - # hand the next case a router it did not choose without saying so, so say it on stderr. - echo "[case-121] settle bound ${SETTLE_BOUND}s spent with the router still on '${ROUTER_IMAGE}'; the settings restore has not been read back" >&2 - return 0 -} +# The pin, the restore and the wait for the router to be re-rendered are the library's: +# epd_pin_router_sidecar_images, epd_restore_router_sidecar_images and epd_settle_router_image. force_release() { local md="$1" row wl uids u @@ -313,17 +259,19 @@ $(kubectl -n "$NS" get workloads.kueue.x-k8s.io \ EOF return 0 } - -# THE SETTINGS GO BACK BEFORE THE DEPLOYMENT IS DELETED, which is the order that lets settle_back +# THE SETTINGS GO BACK BEFORE THE DEPLOYMENT IS DELETED, which is the order that lets the settle # prove the restore: while the ModelDeployment stands, its router Deployment stands with it, and # the image on that Deployment is the one thing in the cluster that shows what the worker has read # since the patch. Deleting the deployment first would remove the only witness. cleanup() { echo echo "[case-121] cleanup" - restore_image_setting "$ROUTER_SETTING" "$ORIG_ROUTER" - restore_image_setting "$SIDECAR_SETTING" "$ORIG_SIDECAR" - settle_back + epd_restore_router_sidecar_images "$SYSTEM_NS" "$ROUTER_SETTING" "$SIDECAR_SETTING" + # The bound is spent and the router still runs this case's image: the restore was written but the + # worker has not read it back. Cleanup cannot record a row from here, and returning silently would + # hand the next case a router it did not choose without saying so, so say it on stderr. + epd_settle_router_image "$NS" "$MD" "$SETTLE_BOUND" "$ROUTER_IMAGE" || + echo "[case-121] settle bound ${SETTLE_BOUND}s spent with the router still on '${ROUTER_IMAGE}'; the settings restore has not been read back" >&2 kubectl -n "$NS" delete pod "$PROBE_NAME" --ignore-not-found --wait=false >/dev/null 2>&1 kubectl -n "$NS" delete modeldeployments.worker.gpustack.ai "$MD" \ --ignore-not-found --wait=false >/dev/null 2>&1 @@ -335,8 +283,8 @@ cleanup() { # cannot leave the cluster running a router image this run chose. trap cleanup EXIT -set_image_setting "$ROUTER_SETTING" "$ROUTER_IMAGE" -set_image_setting "$SIDECAR_SETTING" "$SIDECAR_IMAGE" +epd_pin_router_sidecar_images "$SYSTEM_NS" "$ROUTER_SETTING" "$SIDECAR_SETTING" \ + "$ROUTER_IMAGE" "$SIDECAR_IMAGE" # --- fixture ---------------------------------------------------------------- @@ -425,51 +373,13 @@ fi # --- wait: pods Ready, no crash-loop, endpoint published -------------------- # A pull failure on any of the three images is a SKIP naming the image (case-81's rule): a -# container nobody could start is not a verdict about the leg. -pull_reason() { # - kubectl -n "$NS" get pods -l "app.kubernetes.io/instance=${MD},app.kubernetes.io/component=$1" \ - -o jsonpath='{range .items[*]}{.status.containerStatuses[*].state.waiting.reason}{" "}{end}' 2>/dev/null -} - -wait_pods() { # -> pod name on stdout once Ready, rc!=0 on timeout, rc=2 on a pull failure - local comp="$1" deadline=$((SECONDS + READY_BOUND)) pods ready p - while [ "$SECONDS" -lt "$deadline" ]; do - pods="$(kubectl -n "$NS" get pods \ - -l "app.kubernetes.io/instance=${MD},app.kubernetes.io/component=${comp}" \ - -o json 2>/dev/null)" - ready="$(printf '%s' "$pods" | jq -r \ - '[.items[] | select([.status.conditions // [] | .[] | select(.type=="Ready" and .status=="True")] | length > 0)] | length')" - if [ "${ready:-0}" -ge 1 ]; then - printf '%s' "$pods" | jq -r '.items[0].metadata.name' - return 0 - fi - for p in ImagePullBackOff ErrImagePull; do - case "$(pull_reason "$comp")" in - *"$p"*) return 2 ;; - esac - done - sleep 15 - done - return 1 -} - -# Each role is polled separately and the exit code read IMMEDIATELY, so a pull failure on the -# first role is reported instead of costing the whole bound twice more. -poll_role() { # - local comp="$1" pod rc - pod="$(wait_pods "$comp")"; rc=$? - case "$comp" in - encode) EN_POD="$pod" ;; - prefill) PF_POD="$pod" ;; - decode) DC_POD="$pod" ;; - esac - return "$rc" -} +# container nobody could start is not a verdict about the leg. The reading of a role's Pods is the +# library's: epd_pull_reason, epd_wait_pods and epd_poll_role. EN_POD=""; PF_POD=""; DC_POD="" UNPULLABLE_COMPONENT="" for comp in encode prefill decode; do - poll_role "$comp"; rc=$? + epd_poll_role "$NS" "$MD" "$READY_BOUND" "$comp"; rc=$? if [ "$rc" = 2 ]; then UNPULLABLE_COMPONENT="$comp" break @@ -478,7 +388,7 @@ done if [ -n "$UNPULLABLE_COMPONENT" ]; then record SKIP "the three role Pods reach Ready" \ - "the ${UNPULLABLE_COMPONENT} Pod waits on $(pull_reason "$UNPULLABLE_COMPONENT" | tr -s ' '): no container started, so the leg was never exercised. Render the images and re-run" + "the ${UNPULLABLE_COMPONENT} Pod waits on $(epd_pull_reason "$NS" "$MD" "$UNPULLABLE_COMPONENT" | tr -s ' '): no container started, so the leg was never exercised. Render the images and re-run" print_rows exit 0 fi @@ -497,62 +407,14 @@ done # role poll can see. It is the environment answer the role images already get, and reading it as an # operator defect would cost the whole bound and then be wrong. # -# The router's Pods are selected by the router Deployment's OWN selector rather than by a label -# spelled here. The router carries the same instance label the roles carry, plus a router key of its -# own, so an instance-only selector would return the role Pods too and learn nothing about the -# router; the Deployment's selector is the one that cannot drift from what was rendered. -router_pods() { - local sel - # shellcheck disable=SC2016 # the dollar names are kubectl jsonpath map variables, and must not expand - sel="$(kubectl -n "$NS" get deploy "${MD}-router" \ - -o go-template='{{range $k,$v := .spec.selector.matchLabels}}{{$k}}={{$v}},{{end}}' 2>/dev/null | sed 's/,$//')" - [ -n "$sel" ] || return 0 - kubectl -n "$NS" get pods -l "$sel" -o json 2>/dev/null -} - -# What any router container is WAITING on, joined; empty when none is waiting on an image. Read for -# the same two reasons wait_pods reads them on the roles. -router_pull_reason() { - router_pods | jq -r \ - '[.items[]? | .status.containerStatuses[]? | .state.waiting?.reason // empty] | join(" ")' 2>/dev/null -} - -# The Ready router Pod's name, or empty when the router is not up. Read once when the verdict is -# formed, because by then the bound is spent and the router's state is settled. -router_pod() { - router_pods | jq -r \ - '[.items[]? | select([.status.conditions // [] | .[] | select(.type=="Ready" and .status=="True")] | length > 0) | .metadata.name] | if length > 0 then .[0] else "" end' 2>/dev/null -} - -# The endpoint is written once the ROUTER's OWN Deployment reports a ready replica — it is the -# router's status the operator copies, not a fact about the role Pods — so a single read here -# races that writer by design, and the wait is bounded because the router is a fourth workload no -# role poll above covers. Prints the endpoint and returns 0 on arrival, 2 when the router image -# would not pull (the same code wait_pods returns for a role image), 1 once the bound is spent. -wait_endpoint() { - local deadline=$((SECONDS + READY_BOUND)) ep p - while [ "$SECONDS" -lt "$deadline" ]; do - ep="$(kubectl -n "$NS" get modeldeployments.worker.gpustack.ai "$MD" \ - -o jsonpath='{.status.endpoint}' 2>/dev/null)" - if [ -n "$ep" ]; then - printf '%s' "$ep" - return 0 - fi - for p in ImagePullBackOff ErrImagePull; do - case "$(router_pull_reason)" in - *"$p"*) return 2 ;; - esac - done - sleep 5 - done - return 1 -} +# The reading of the router is the library's: epd_router_pods, epd_router_pull_reason, +# epd_router_pod and epd_wait_endpoint. ALL_READY=no [ -n "$EN_POD" ] && [ -n "$PF_POD" ] && [ -n "$DC_POD" ] && ALL_READY=yes -ENDPOINT="$(wait_endpoint)"; ENDPOINT_RC=$? -ROUTER_POD="$(router_pod)" +ENDPOINT="$(epd_wait_endpoint "$NS" "$MD" "$READY_BOUND")"; ENDPOINT_RC=$? +ROUTER_POD="$(epd_router_pod "$NS" "$MD")" # The FAIL verdict needs the WHOLE chain up, the router included. A router that never came up # publishes no front door either, and calling that a defect would blame the operator for an image # it was told to run. @@ -565,7 +427,7 @@ elif [ "$ENDPOINT_RC" = 2 ]; then # An image that would not pull is not a verdict about the leg: no container started, so there was # never a front door to publish. Same shape as the role-image SKIP above, naming the image. record SKIP "endpoint" \ - "the router Pod waits on $(router_pull_reason | tr -s ' '), so router image '${ROUTER_IMAGE}' would not pull and no front door could be published. Render the image and re-run" + "the router Pod waits on $(epd_router_pull_reason "$NS" "$MD" | tr -s ' '), so router image '${ROUTER_IMAGE}' would not pull and no front door could be published. Render the image and re-run" elif [ "$CHAIN_UP" = yes ]; then # Every Pod came up, the router's included, and the controller still publishes no front door. # Nothing else explains that, so it is a defect in what the operator publishes rather than an @@ -591,47 +453,11 @@ fi # --- the rendered leg, and the mounts that carry it ------------------------- -ec_document() { # - kubectl -n "$NS" get pod "$1" -o json 2>/dev/null | jq -r \ - --arg f "--ec-transfer-config" \ - '[(.spec.containers[] | select(.name=="main") | (.command // []) + (.args // []))][0] // [] - | (index($f) // -1) as $i | if $i < 0 then "" else .[$i + 1] end' 2>/dev/null -} - -ec_env() { # - kubectl -n "$NS" get pod "$1" -o json 2>/dev/null | jq -r \ - --arg n "$2" '[(.spec.containers[] | select(.name=="main") | .env // [])[] | select(.name==$n)][0].value // ""' 2>/dev/null -} - -# The claim one role's FIRST Pod mounts, as "claimName|mountPath|readOnly". Every container is -# searched, not just the first: a direct decode renders the routing sidecar as the first regular -# container, which carries no mounts, so reading that one would answer every role with "no claim". -role_pod_claim() { # - kubectl -n "$NS" get pods \ - -l "app.kubernetes.io/instance=${MD},app.kubernetes.io/component=$1" \ - -o json 2>/dev/null | python3 -c ' -import json,sys -items=json.load(sys.stdin).get("items",[]) -if not items: - print("") - raise SystemExit -pod=items[0] -claims={} -for v in pod["spec"].get("volumes",[]): - p=v.get("persistentVolumeClaim") - if p and p.get("claimName"): - claims[v["name"]]=p["claimName"] -for ctr in pod["spec"].get("containers",[]): - for m in ctr.get("volumeMounts",[]): - claim=claims.get(m.get("name","")) - if claim: - print(claim+"|"+m.get("mountPath","")+"|"+str(m.get("readOnly",False)).lower()) - raise SystemExit -print("")' -} +# The document the engine was handed, the variable it was given and the claim a role mounts are +# read through the library: epd_ec_document, epd_ec_env and epd_role_pod_claim. if [ -n "$EN_POD" ]; then - DOC="$(ec_document "$EN_POD")" + DOC="$(epd_ec_document "$NS" "$EN_POD")" if [ -n "$DOC" ]; then if printf '%s' "$DOC" | jq -e \ --arg p "$MOUNT" \ @@ -646,8 +472,8 @@ if [ -n "$EN_POD" ]; then # and VLLM_USE_V2_MODEL_RUNNER is the other connector's precondition; both are asserted # absent so a render that reached for the wrong connector is caught here rather than as a # transfer failure the log never names. - SIDE_PORT="$(ec_env "$EN_POD" VLLM_EC_SIDE_CHANNEL_PORT)" - RUNNER="$(ec_env "$EN_POD" VLLM_USE_V2_MODEL_RUNNER)" + SIDE_PORT="$(epd_ec_env "$NS" "$EN_POD" VLLM_EC_SIDE_CHANNEL_PORT)" + RUNNER="$(epd_ec_env "$NS" "$EN_POD" VLLM_USE_V2_MODEL_RUNNER)" if [ -z "$SIDE_PORT" ] && [ -z "$RUNNER" ]; then record PASS "encoder carries no NIXL variable" \ "neither VLLM_EC_SIDE_CHANNEL_PORT nor VLLM_USE_V2_MODEL_RUNNER is set on ${EN_POD}, which is right for the shared-directory connector" @@ -661,7 +487,7 @@ if [ -n "$EN_POD" ]; then fi if [ -n "$PF_POD" ]; then - DOC="$(ec_document "$PF_POD")" + DOC="$(epd_ec_document "$NS" "$PF_POD")" if [ -n "$DOC" ]; then if printf '%s' "$DOC" | jq -e \ --arg p "$MOUNT" \ @@ -678,9 +504,9 @@ if [ -n "$PF_POD" ]; then fi if [ -n "$EN_POD" ] && [ -n "$PF_POD" ] && [ -n "$DC_POD" ]; then - EN_CLAIM="$(role_pod_claim encode)" - PF_CLAIM="$(role_pod_claim prefill)" - DC_CLAIM="$(role_pod_claim decode)" + EN_CLAIM="$(epd_role_pod_claim "$NS" "$MD" encode)" + PF_CLAIM="$(epd_role_pod_claim "$NS" "$MD" prefill)" + DC_CLAIM="$(epd_role_pod_claim "$NS" "$MD" decode)" case "$EN_CLAIM" in "${CLAIM}|${MOUNT}|false") @@ -695,7 +521,7 @@ if [ -n "$EN_POD" ] && [ -n "$PF_POD" ] && [ -n "$DC_POD" ]; then case "$PF_CLAIM" in "${CLAIM}|${MOUNT}|"*) # Read-only is the PREFILLER's own choice — it only reads — so this row pins the claim and - # the path and leaves the flag open, exactly as case 119 does. + # the path and leaves the flag open, exactly as case 117 does. record PASS "prefiller mounts the same claim at the same path" \ "rendered ${PF_CLAIM}, matching the encoder's claim and path" ;; @@ -779,4 +605,4 @@ fi # Results. print_rows [ "$FAILS" -eq 0 ] || { echo "[case-121] ${FAILS} check(s) FAILED"; exit 1; } -echo "[case-121] all checks passed" \ No newline at end of file +echo "[case-121] all checks passed" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-122.sh b/.agents/skills/gpustack-operator-e2e/cases/case-122.sh deleted file mode 100755 index 9a673ee98..000000000 --- a/.agents/skills/gpustack-operator-e2e/cases/case-122.sh +++ /dev/null @@ -1,344 +0,0 @@ -#!/usr/bin/env bash -# -# CASE 122 — SGLang's admitted encoder rows, and the pairings the matrix still refuses -# (NON-MUTATING: server-side dry runs only) -# AUTO-SKIPS the platform rows without an InstanceType reporting an admitted platform -# -# case-122.sh -# -# Goal: SGLang carries six admitted encoder rows: llm-d-router and sglang-gateway across -# nvidia, amd and ascend, naming the address leg -# (encoder-urls) and mounting no claim. This case reads that on a live API server rather -# than in a table a user cannot see: on the platform it injects, both routers with -# SGLang are ACCEPTED with no claim mounted, and the pairing the matrix still refuses -# is refused naming BOTH halves and the admitted rows. -# -# Environment: Any cluster with a materialized scheduling chain (run case-1 first). If an active -# InstanceType already reports the target manufacturer (E2E_C122_VENDOR, default Ascend; -# also supports nvidia or amd), it is used directly without mutating node labels or -# settings. Otherwise, on a CPU-only cluster, the case injects the vendor label on one -# Ready schedulable worker node and flips CPU-manufacturer awareness on in -# gpustack-settings (E2E_SYSTEM_NS, default gpustack-system). -# -# AUTO-SKIP: every admitted-row check below needs a usable InstanceType for the target -# platform. If neither an existing pool nor the injected CPU vendor label yields one, -# the rows record SKIP and the case exits 0. -# -# Inputs: MOCKED (only when no matching InstanceType already exists): one node label (the CPU -# vendor) injected through a NodeFeature object and one settings key (CPU-manufacturer -# awareness) flipped on — both restored by the trap. -# REAL: every assertion is a server-side dry run, so no ModelDeployment is stored. -# -# Expected: - the vLLM baseline shape for the target platform is ACCEPTED; -# - llm-d-router + SGLang, mounting no claim, is ACCEPTED on the target platform, and -# sglang-gateway + SGLang is ACCEPTED the same way; -# - sglang-gateway with vLLM is REFUSED naming both the router and the engine it is not -# paired with, and listing the admitted rows a user can compare against. -# -# Cleanup: If the fixture injected a NodeFeature object and flipped the settings key, the trap -# deletes the NodeFeature, restores the settings key to its exact original value, and -# waits for the derived type to settle back. Idempotent, runs on pass AND fail. -set -uo pipefail - -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" - -NS="${1:-}" -if [ -z "$NS" ]; then - echo "usage: case-122.sh " >&2 - exit 2 -fi - -SYSTEM_NS="${E2E_SYSTEM_NS:-gpustack-system}" -SETTING=instance-type-aware-cpu-manufacturer -VENDOR="${E2E_C122_VENDOR:-Ascend}" -PLATFORM="$(printf '%s' "$VENDOR" | tr '[:upper:]' '[:lower:]')" -NF=case122-cpu-vendor -MD=case122-epd -IMAGE="${E2E_MD_IMAGE:-registry.k8s.io/pause:3.10}" -SETTLE_ROUNDS="${E2E_C122_SETTLE:-60}" -ROUTER_NAME=llm-d-router -ENGINE=vLLM - -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } -# shellcheck source=/dev/null -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" - -command -v jq >/dev/null 2>&1 || { echo "[case-122] jq is required." >&2; exit 2; } - -pick_worker_node() { - kubectl get nodes -o json 2>/dev/null | python3 -c ' -import json,sys -for n in json.load(sys.stdin).get("items",[]): - cs=n.get("status",{}).get("conditions",[]) - if not any(c.get("type")=="Ready" and c.get("status")=="True" for c in cs): - continue - if n.get("spec",{}).get("unschedulable"): - continue - if n.get("metadata",{}).get("labels",{}).get("feature.gpustack.ai/acceleratable") == "true": - continue - ts=n.get("spec",{}).get("taints",[]) - if any(str(t.get("key","")).startswith("node-role.kubernetes.io/") for t in ts): - continue - print(n["metadata"]["name"]) - break -' -} - -epd_platform_type() { - kubectl get instancetypes.worker.gpustack.ai \ - -o jsonpath='{range .items[*]}{.metadata.name}|{.metadata.deletionTimestamp}|{.spec.inactive}|{.status.detail.manufacturer}{"\n"}{end}' \ - 2>/dev/null \ - | while IFS='|' read -r name deleting inactive manufacturer; do - [ -n "$name" ] || continue - [ -z "$deleting" ] || continue - [ "$inactive" = true ] && continue - if [ "$manufacturer" = "$PLATFORM" ]; then - echo "$name" - break - fi - done -} - -ORIG_SETTING="$(kubectl -n "$SYSTEM_NS" get secret gpustack-settings -o jsonpath="{.data.${SETTING}}" 2>/dev/null)" -set_awareness() { # true|false - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"${SETTING}\":\"$(printf '%s' "$1" | base64)\"}}" >/dev/null 2>&1 -} -restore_setting() { - local v=null - [ -n "$ORIG_SETTING" ] && v="\"${ORIG_SETTING}\"" - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"${SETTING}\":${v}}}" >/dev/null 2>&1 -} - -# Restoring the value is not the end of the restore: the worker reads it through a thirty-second -# cache, and the type derived from it outlives the patch by that cache plus one reconcile. Wait for -# the derived type to go, so the next case in a serial run does not list an aware type this run left -# behind. A cluster that started with the setting already on keeps its derived type, so there is -# nothing to wait for. -settle_back() { - [ "$(printf '%s' "$ORIG_SETTING" | base64 -d 2>/dev/null)" = "true" ] && return 0 - for _ in $(seq 1 "$SETTLE_ROUNDS"); do - [ -z "$(epd_platform_type)" ] && return 0 - sleep 5 - done - return 0 -} - -INJECTED=false -cleanup() { - echo - echo "[case-122] cleanup" - if [ "$INJECTED" = true ]; then - kubectl -n "$NS" delete nodefeatures.nfd.k8s-sigs.io "$NF" --ignore-not-found >/dev/null 2>&1 - restore_setting - settle_back - fi -} -trap cleanup EXIT - -# ---------------------------------------------------------------- the fixture - -IT="$(epd_platform_type)" -if [ -n "$IT" ]; then - record PASS "an active pool reports the target platform" \ - "instancetype ${IT} reports manufacturer '${PLATFORM}'" -else - NODE="$(pick_worker_node)" - if [ -z "$NODE" ]; then - echo "[case-122] no active '${PLATFORM}' InstanceType and no Ready schedulable CPU worker node" >&2 - exit 2 - fi - - echo "[case-122] injecting CPU vendor '${VENDOR}' on node ${NODE} and turning CPU-manufacturer awareness on" - INJECTED=true - - kubectl apply -f - >/dev/null 2>&1 <&1 | tr '\n' ' ')" - if [ -z "$out" ]; then - record FAIL "$check" "the apply produced no output at all, so nothing was refused" - return 0 - fi - for w in "$want" "$@"; do - case "$out" in - *"$w"*) ;; - *) record FAIL "$check" "refused with the wrong message, or accepted (missing '${w}'): ${out:0:220}"; return 0 ;; - esac - done - record PASS "$check" "refused naming: ${want}${*:+ and $*}" - return 0 -} - -accepts() { - local check="$1" roles="$2" out - out="$(manifest "$roles" | kubectl apply --dry-run=server -f - 2>&1 | tr '\n' ' ')" - case "$out" in - *"created (server dry run)"* | *"configured (server dry run)"*) - record PASS "$check" "accepted by both the schema and the webhook" ;; - *) record FAIL "$check" "refused: ${out:0:220}" ;; - esac - return 0 -} - -# ---------------------------------------------------------------- row 0: the accepted baseline - -# Three roles being accepted at all is the shape this whole rule set exists to guard, so the -# baseline proves the refusals below are each their own rule rather than one blanket rejection. -if [ -n "$IT" ]; then - accepts "the admitted combination is ACCEPTED by the same instrument" "$(baseline_roles)" -else - record SKIP "the admitted combination is ACCEPTED by the same instrument" \ - "no platform InstanceType exists, so there is no shape to accept and every refusal below would pass for the wrong reason" -fi - -# ---------------------------------------------------------------- the SGLang rows - -if [ -n "$IT" ]; then - # The admitted SGLang rows, read through the same instrument as the vLLM baseline above. Both - # routers are accepted on SGLang with no claim mounted, because an address leg is what carries the - # encoder's output and a connector there is not a thing a user attaches. - ENGINE=SGLang accepts "llm-d-router with SGLang is ACCEPTED" \ - "$(address_leg_roles)" - ENGINE=SGLang ROUTER_NAME=sglang-gateway \ - accepts "sglang-gateway with SGLang is ACCEPTED" \ - "$(address_leg_roles)" - ROUTER_NAME=llm-d-router - - # The pairing the matrix still refuses. sglang-gateway drives the encoder leg on SGLang alone, so - # vLLM under it names both halves in one message along with the admitted rows. - ROUTER_NAME=sglang-gateway ENGINE=vLLM refuses "sglang-gateway with vLLM is refused naming the pairing" \ - 'router "sglang-gateway" is not admitted with engine "vLLM"' \ - "$(baseline_roles)" \ - 'the admitted rows are' - ROUTER_NAME=llm-d-router - ENGINE=vLLM -else - for label in \ - "llm-d-router with SGLang is ACCEPTED" \ - "sglang-gateway with SGLang is ACCEPTED" \ - "sglang-gateway with vLLM is refused naming the pairing"; do - record SKIP "$label" "needs the platform InstanceType this run did not build; see the baseline row" - done -fi - -# Results. -print_rows -[ "$FAILS" -eq 0 ] || { echo "[case-122] ${FAILS} check(s) FAILED"; exit 1; } -echo "[case-122] all checks passed" \ No newline at end of file diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-123.sh b/.agents/skills/gpustack-operator-e2e/cases/case-123.sh deleted file mode 100755 index 2f59cce03..000000000 --- a/.agents/skills/gpustack-operator-e2e/cases/case-123.sh +++ /dev/null @@ -1,315 +0,0 @@ -#!/usr/bin/env bash -# -# CASE 123 — vllm-router admits vLLM encoder deployments on NVIDIA and AMD, refuses unadmitted -# engines and user-supplied --epd-config -# (MUTATING, self-recovering: the platform fixture on CPU-only clusters, restored by -# the trap — GPU clusters need no fixture, and every verdict is a server-side dry run) -# AUTO-SKIPS the platform rows without an InstanceType reporting nvidia or amd -# -# case-123.sh -# -# Goal: vllm-router drives three-stage EPD disaggregation for vLLM on NVIDIA and AMD via -# --epd-config={"encoder_urls":[...]} while the engine roles exchange vision embeddings -# over NIXL (ECCPUConnector) and KV cache over MooncakeConnector. -# This case pins the admission contract for vllm-router encoder deployments: -# - vllm-router + vLLM on nvidia or amd is ACCEPTED; -# - vllm-router + SGLang is REFUSED naming that the router and engine do not pair; -# - user-supplied --epd-config in spec.router.extraArgs is REFUSED as operator-owned; -# - vllm-router with NO Encode role (plain prefill/decode) is ACCEPTED. -# -# Environment: Any cluster with a materialized scheduling chain (run case-1 first). When the cluster -# already carries an active nvidia or amd InstanceType, the case reads that type -# directly and mutates nothing. On a CPU-only cluster it injects a CPU vendor label -# ("NVIDIA") on one non-acceleratable worker node and flips CPU-manufacturer awareness -# on, restoring both on exit. -# -# AUTO-SKIP: if no nvidia/amd InstanceType exists and NFD declines the injected CPU -# vendor label, the platform-bound rows record SKIP and the case exits 0. -# -# Inputs: Every ModelDeployment assertion is a server-side dry run (`kubectl apply --dry-run=server`), -# so no ModelDeployment is persisted. -# -# Expected: - vllm-router + vLLM 0.30.0 on nvidia/amd with Encode/Prefill/Decode is ACCEPTED; -# - vllm-router + SGLang with an Encode role is REFUSED naming the router/engine pair; -# - vllm-router passing --epd-config in spec.router.extraArgs is REFUSED as owned; -# - vllm-router with NO Encode role (plain prefill/decode) is ACCEPTED. -# -# Cleanup: If the CPU-vendor fixture was used, the trap deletes the NodeFeature object, restores -# the settings key, and waits for the derived InstanceType to settle back. -set -uo pipefail - -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" - -NS="${1:-}" -if [ -z "$NS" ]; then - echo "usage: case-123.sh " >&2 - exit 2 -fi - -SYSTEM_NS="${E2E_SYSTEM_NS:-gpustack-system}" -SETTING=instance-type-aware-cpu-manufacturer -VENDOR="${E2E_C123_VENDOR:-NVIDIA}" -PLATFORM="$(printf '%s' "$VENDOR" | tr '[:upper:]' '[:lower:]')" -NF=case123-cpu-vendor -MD=case123-epd -IMAGE="${E2E_MD_IMAGE:-registry.k8s.io/pause:3.10}" -SETTLE_ROUNDS="${E2E_C123_SETTLE:-60}" -ROUTER_NAME=vllm-router -ENGINE=vLLM -ROUTER_EXTRA_ARGS="" - -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } -# shellcheck source=/dev/null -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" - -command -v jq >/dev/null 2>&1 || { echo "[case-123] jq is required." >&2; exit 2; } - -usable_type_for() { # - kubectl get instancetypes.worker.gpustack.ai \ - -o jsonpath='{range .items[*]}{.metadata.name}|{.metadata.deletionTimestamp}|{.spec.inactive}|{.status.detail.manufacturer}{"\n"}{end}' \ - 2>/dev/null \ - | while IFS='|' read -r name deleting inactive manufacturer; do - [ -n "$name" ] || continue - [ -z "$deleting" ] || continue - [ "$inactive" = true ] && continue - if [ "$manufacturer" = "$1" ]; then - echo "$name" - break - fi - done -} - -pick_worker_node() { - kubectl get nodes -o json 2>/dev/null | python3 -c ' -import json,sys -for n in json.load(sys.stdin).get("items",[]): - cs=n.get("status",{}).get("conditions",[]) - if not any(c.get("type")=="Ready" and c.get("status")=="True" for c in cs): - continue - if n.get("spec",{}).get("unschedulable"): - continue - if n.get("metadata",{}).get("labels",{}).get("feature.gpustack.ai/acceleratable") == "true": - continue - ts=n.get("spec",{}).get("taints",[]) - if any(str(t.get("key","")).startswith("node-role.kubernetes.io/") for t in ts): - continue - print(n["metadata"]["name"]) - break -' -} - -epd_platform_type() { - usable_type_for "$PLATFORM" -} - -ORIG_SETTING="$(kubectl -n "$SYSTEM_NS" get secret gpustack-settings -o jsonpath="{.data.${SETTING}}" 2>/dev/null)" -INJECTED_FIXTURE=false - -set_awareness() { # true|false - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"${SETTING}\":\"$(printf '%s' "$1" | base64)\"}}" >/dev/null 2>&1 -} -restore_setting() { - local v=null - [ -n "$ORIG_SETTING" ] && v="\"${ORIG_SETTING}\"" - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"${SETTING}\":${v}}}" >/dev/null 2>&1 -} - -settle_back() { - [ "$(printf '%s' "$ORIG_SETTING" | base64 -d 2>/dev/null)" = "true" ] && return 0 - for _ in $(seq 1 "$SETTLE_ROUNDS"); do - [ -z "$(epd_platform_type)" ] && return 0 - sleep 5 - done - return 0 -} - -cleanup() { - [ "$INJECTED_FIXTURE" = true ] || return 0 - echo - echo "[case-123] cleanup" - kubectl -n "$NS" delete nodefeatures.nfd.k8s-sigs.io "$NF" --ignore-not-found >/dev/null 2>&1 - restore_setting - settle_back -} -trap cleanup EXIT - -# ---------------------------------------------------------------- the platform InstanceType - -IT="" -for m in nvidia amd; do - IT="$(usable_type_for "$m")" - if [ -n "$IT" ]; then - PLATFORM="$m" - break - fi -done - -if [ -n "$IT" ]; then - record PASS "an admitted platform InstanceType is available" \ - "instancetype ${IT} reports manufacturer '${PLATFORM}'" -else - NODE="$(pick_worker_node)" - if [ -n "$NODE" ]; then - INJECTED_FIXTURE=true - echo "[case-123] injecting CPU vendor '${VENDOR}' on node ${NODE} and turning CPU-manufacturer awareness on" - kubectl apply -f - >/dev/null 2>&1 <&1 | tr '\n' ' ')" - if [ -z "$out" ]; then - record FAIL "$check" "the apply produced no output at all, so nothing was refused" - return 0 - fi - for w in "$want" "$@"; do - case "$out" in - *"$w"*) ;; - *) record FAIL "$check" "refused with the wrong message, or accepted (missing '${w}'): ${out:0:220}"; return 0 ;; - esac - done - record PASS "$check" "refused naming: ${want}${*:+ and $*}" - return 0 -} - -accepts() { - local check="$1" roles="$2" out - out="$(manifest "$roles" | kubectl apply --dry-run=server -f - 2>&1 | tr '\n' ' ')" - case "$out" in - *"created (server dry run)"* | *"configured (server dry run)"*) - record PASS "$check" "accepted by both the schema and the webhook" ;; - *) record FAIL "$check" "refused: ${out:0:220}" ;; - esac - return 0 -} - -# ---------------------------------------------------------------- the vllm-router rows - -if [ -n "$IT" ]; then - # Row 0: the admitted vllm-router + vLLM EPD shape on nvidia/amd is accepted. - ROUTER_NAME=vllm-router accepts \ - "vllm-router + vLLM with an encoder role is ACCEPTED on ${PLATFORM}" \ - "$(baseline_roles)" - - # Row 2: vllm-router paired with SGLang is refused naming the router and engine. - ROUTER_NAME=vllm-router ENGINE=SGLang refuses \ - "vllm-router paired with SGLang is refused" \ - 'router "vllm-router" is not admitted with engine "SGLang"' \ - "$(baseline_roles)" \ - 'the admitted rows are' - ENGINE=vLLM - - # Row 3: --epd-config is owned by the operator for vllm-router and refused in extraArgs. - ROUTER_EXTRA_ARGS="$(printf ' extraArgs:\n - --epd-config={"encoder_urls":[]}\n')" \ - ROUTER_NAME=vllm-router refuses \ - "user-supplied --epd-config on vllm-router is refused as operator-owned" \ - '"--epd-config" is set by the operator for router "vllm-router"' \ - "$(baseline_roles)" - ROUTER_EXTRA_ARGS="" - - # Row 4: vllm-router with no Encode role (plain prefill/decode) remains accepted. - ROUTER_NAME=vllm-router accepts \ - "vllm-router with NO encoder role is admitted" \ - "$(pair_roles)" -else - for label in \ - "vllm-router + vLLM with an encoder role is ACCEPTED" \ - "vllm-router paired with SGLang is refused" \ - "user-supplied --epd-config on vllm-router is refused as operator-owned" \ - "vllm-router with NO encoder role is admitted"; do - record SKIP "$label" "needs an nvidia or amd InstanceType this run did not have; see the first row" - done -fi - -# Results. -print_rows -[ "$FAILS" -eq 0 ] || { echo "[case-123] ${FAILS} check(s) FAILED"; exit 1; } -echo "[case-123] all checks passed" \ No newline at end of file diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-124.sh b/.agents/skills/gpustack-operator-e2e/cases/case-124.sh deleted file mode 100755 index 7ba1776b1..000000000 --- a/.agents/skills/gpustack-operator-e2e/cases/case-124.sh +++ /dev/null @@ -1,334 +0,0 @@ -#!/usr/bin/env bash -# -# CASE 124 — The admitted matrix admits ascend, nvidia and amd, and every other accelerator -# platform is refused, the refusal naming the platform beside the router and engine it -# accepted and the rows that do exist (NON-MUTATING: server-side dry runs only) -# AUTO-SKIPS without a cluster that has an unadmitted platform to submit -# -# case-124.sh -# -# Goal: The admitted matrix carries Ascend, NVIDIA and AMD and NOTHING ELSE. NVIDIA joined -# only once its live proof of concept passed; every other accelerator platform is -# absent, and that is the state of the work rather than an oversight: a combination is -# admitted only after a live cluster run passes. This case is the reading of that -# absence, and it reads the one refusal this design will actually meet -# in the field — because the platform a user has is exactly what they will write an -# encoder deployment on first. -# -# The refusal names the PLATFORM column, with the router and the engine it checked and -# rejected alongside it, then lists the admitted rows so the reader can see what to -# change. That last part is the reason this case exists at all: a refusal that says only -# "not admitted" leaves a user to find the table themselves, and the table is not -# something they can see. -# -# THE PLATFORM IS READ FROM THE INSTANCE TYPE, NOT DECLARED, so the case does not -# invent a manufacturer: it looks for a type the cluster actually has that reports one -# the matrix does not cover. Which manufacturers to look for is the OPERATOR'S OWN LIST, -# minus the three it admits: the case iterates every accelerator manufacturer this -# project declares and takes the first usable type it finds, rather than guessing a couple of -# names. Guessing is how a cluster that holds a Hygon or a THead card ends up skipped -# for want of a name nobody thought to probe. The general CPU fallback is left out: -# "generic" is what an unrecognised cpu-model.vendor_id collapses to, not an -# accelerator platform, and submitting it would assert nothing about the matrix. -# -# WHY THE FUNCTIONAL HALF LIVES ELSEWHERE. The admitted platforms' functional readings -# are their own cases (the NVIDIA and AMD network leg is case 120, the Ascend -# shared-directory leg is case 121), so there is nothing functional to assert here: a -# run on an unadmitted platform is refused at admission, which is exactly the rows -# above. This case states that at the end so a reader scanning only the table sees it -# too. -# -# Environment: Any cluster with a materialized scheduling chain (run case-1 first). Needs at least -# one usable InstanceType (not inactive, not terminating) whose reported manufacturer is -# one this operator declares and the matrix does not admit — every accelerator -# manufacturer the project declares, minus ascend, nvidia and amd, tried in that order -# (cambricon, hygon, iluvatar, metax, mthreads, thead). A cluster of Ascend, NVIDIA and -# AMD types alone has no such type and skips those rows. NO accelerator of any specific -# family is required, and none of the roles will ever run: every assertion is a -# server-side dry run. -# -# AUTO-SKIP: when no usable InstanceType reports one of those manufacturers, the platform -# rows record a SKIP naming the manufacturers that were looked for and the types that -# were seen, and the case exits 0. -# -# Inputs: MOCKED: nothing. No node label is injected and no setting is changed — this case -# reads the manufacturers the cluster's own InstanceTypes already report, so it needs -# no fixture and its trap deletes nothing. -# -# Expected: - on a cluster with an unadmitted platform, that platform is REFUSED naming the -# platform column, the router and the engine it was checked against, and the admitted -# rows; -# - two roles of DIFFERENT platforms are refused separately: the mixed-platform rule -# answers first, so the refusal names two manufacturers rather than the matrix; -# - an admitted combination (llm-d-router + vLLM on an admitted platform) is ACCEPTED -# by the same instrument, so the refusals are each their own rule; on ascend that -# shape carries the two shared-claim mounts the row renders, and on nvidia or amd it -# carries none, because the network rows forbid the shared mount; -# - on a cluster with no unadmitted platform, the platform rows SKIP naming the -# manufacturers that were looked for and the types that were seen, and the accepted -# baseline still runs; -# - the functional half is a SKIP pointing at the cases that carry it. -# -# Cleanup: Nothing to clean. The case creates no object, injects no label, changes no setting -# and stores nothing; every assertion is a server-side dry run. Safe to re-run. -set -uo pipefail - -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" - -NS="${1:-}" -if [ -z "$NS" ]; then - echo "usage: case-124.sh " >&2 - exit 2 -fi - -MD=case124-epd -IMAGE="${E2E_MD_IMAGE:-registry.k8s.io/pause:3.10}" -ROUTER_NAME=llm-d-router -ENGINE=vLLM - -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } -# shellcheck source=/dev/null -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" - -command -v jq >/dev/null 2>&1 || { echo "[case-124] jq is required." >&2; exit 2; } - - -# ---------------------------------------------------------------- the platforms this cluster has - -# TWO TYPES ARE SOUGHT, and neither is invented. The platform row needs a type reporting a -# manufacturer the matrix does NOT cover, and the baseline needs one that reports one it DOES. -# On a cluster holding any unadmitted accelerator the first is found and the case runs its rows; on -# a cluster whose only accelerated types are the three admitted platforms the first is empty, the -# rows skip naming what was seen, and the baseline still runs against the second. -# -# THE UNADMITTED MANUFACTURERS ARE THE OPERATOR'S OWN LIST MINUS THE THREE IT ADMITS, so every -# accelerator this project supports is covered without the case having to know which of them is -# common. Three of the nine declared are admitted here and are therefore excluded; the general CPU -# fallback is not a manufacturer of an accelerator and is not in the list either. -UNADMITTED_PLATFORMS="cambricon hygon iluvatar metax mthreads thead" - -# The manufacturer is read from status, not from spec: the platform of a role is whatever its -# InstanceType reports, which is the same fact the webhook reads. A type reporting nothing is not -# treated as a platform of its own — that is the not-yet-synced state, and the webhook refuses it. -usable_type_for() { # - kubectl get instancetypes.worker.gpustack.ai -o jsonpath='{range .items[*]}{.metadata.name}|{.metadata.deletionTimestamp}|{.spec.inactive}|{.status.detail.manufacturer}{"\n"}{end}' \ - 2>/dev/null \ - | while IFS='|' read -r name deleting inactive manufacturer; do - [ -n "$name" ] || continue - [ -z "$deleting" ] || continue - [ "$inactive" = true ] && continue - [ -n "$manufacturer" ] || continue - if [ "$manufacturer" = "$1" ]; then - echo "$name" - break - fi - done -} - -OTHER_IT="" -OTHER_MANUFACTURER="" -for m in $UNADMITTED_PLATFORMS; do - OTHER_IT="$(usable_type_for "$m")" - if [ -n "$OTHER_IT" ]; then - OTHER_MANUFACTURER="$m" - break - fi -done - -# The baseline is ANY admitted platform, tried in the order the matrix admitted them, and the -# manufacturer is kept beside the type because the shape differs by row: ascend carries the -# shared claim, nvidia and amd carry the network leg and forbid the mount. -IT="" -IT_MFG="" -for m in ascend nvidia amd; do - IT="$(usable_type_for "$m")" - if [ -n "$IT" ]; then - IT_MFG="$m" - break - fi -done - -SEEN="$(kubectl get instancetypes.worker.gpustack.ai \ - -o jsonpath='{range .items[*]}{.metadata.name}={.status.detail.manufacturer}{" "}{end}' 2>/dev/null)" - -if [ -z "$OTHER_IT" ] && [ -z "$IT" ]; then - record SKIP "the platform could be read" \ - "no usable InstanceType reported any manufacturer (the cluster holds: ${SEEN:-}), so there is no platform to submit and no baseline to compare against" - print_rows - exit 0 -fi - -# ---------------------------------------------------------------- the manifests - -# The engine and the router are the only dials this case turns; each is reset right after the row -# that turns it, because an assignment prefixed to a function call persists in bash after the call -# returns and would silently move every later row off the baseline. -manifest() { # roles-block - cat <&1 | tr '\n' ' ')" - if [ -z "$out" ]; then - record FAIL "$check" "the apply produced no output at all, so nothing was refused" - return 0 - fi - for w in "$want" "$@"; do - case "$out" in - *"$w"*) ;; - *) record FAIL "$check" "refused with the wrong message, or accepted (missing '${w}'): ${out:0:220}"; return 0 ;; - esac - done - record PASS "$check" "refused naming: ${want}${*:+ and $*}" - return 0 -} - -accepts() { - local check="$1" roles="$2" out - out="$(manifest "$roles" | kubectl apply --dry-run=server -f - 2>&1 | tr '\n' ' ')" - case "$out" in - *"created (server dry run)"* | *"configured (server dry run)"*) - record PASS "$check" "accepted by both the schema and the webhook" ;; - *) record FAIL "$check" "refused: ${out:0:220}" ;; - esac - return 0 -} - -# ---------------------------------------------------------------- row 0: the accepted baseline - -# Three roles being accepted at all is the shape this whole rule set exists to guard, so the -# baseline proves the refusals below are each their own rule rather than one blanket rejection. -# The shape is whichever admitted platform this cluster has: the ascend row carries the two -# shared-claim mounts, and the nvidia or amd rows carry none, because the network connector -# forbids the shared mount on its row. -if [ -n "$IT" ]; then - accepts "an admitted combination is ACCEPTED by the same instrument" \ - "$(baseline_roles "$IT" "$IT_MFG")" -else - record SKIP "an admitted combination is ACCEPTED by the same instrument" \ - "no usable InstanceType reports an admitted platform (the cluster holds: ${SEEN:-}), so there is no accepted shape to compare the refusals against" -fi - -# ---------------------------------------------------------------- the unadmitted-platform rows - -if [ -n "$OTHER_IT" ]; then - # THE HEADLINE OF THIS CASE. The matrix gap is read router-first, then engine, then platform, and - # it returns ONE message: llm-d-router and vLLM are both admitted here, so the gap that fires is - # the platform one, which names the platform AND quotes the router and engine it ACCEPTED beside - # it. The word "and" before the engine is the fragment that proves this is the platform arm and - # not the engine arm above it — the two messages read alike and only that word tells them apart. - # The admitted rows come from the same sentence, appended by the rule that fires it. - refuses "a platform the matrix does not admit is refused, all three named" \ - "is not admitted for encoder roles under router" \ - "$(baseline_roles "$OTHER_IT" "$OTHER_MANUFACTURER")" \ - 'and engine' \ - "the admitted rows are" - - # THE SAME PLATFORM WITH A DIFFERENT ENGINE, which reaches a different arm and stops earlier: - # the engine is checked before the platform, so this refusal is the engine message and never - # reaches the platform row above. The pair is what tells a user which of the two to change, and - # either alone leaves the other looking fine. - ENGINE=SGLang refuses "an unadmitted platform is refused naming the engine too" \ - 'is not admitted with engine "SGLang"' \ - "$(baseline_roles "$OTHER_IT" "$OTHER_MANUFACTURER")" - ENGINE="" - - # TWO PLATFORMS IN ONE DEPLOYMENT answer a DIFFERENT rule, and the difference is the point: the - # matrix lookup is per platform, so the mixed-platform rule runs first and names both - # manufacturers instead of the matrix. A case that only submitted one unadmitted platform would - # never tell the two refusals apart. - # - # THE MINIMUM THAT REACHES IT IS ONE ROLE OF EACH KIND ON TWO TYPES. Six roles would reach it - # too, but a second role of a kind trips the at-most-one-role-of-each-kind rule alongside it, and - # this case states elsewhere that a manifest which trips a rule its row is not about is a row - # that has stopped being about its rule. - if [ -n "$IT" ]; then - refuses "two platforms in one encoder deployment are refused by the mixing rule" \ - "more than one accelerator platform" \ - "$(baseline_roles "$IT" "$IT_MFG") - - name: decode - kind: Decode - instanceType: ${OTHER_IT} - image: ${IMAGE}" - else - record SKIP "two platforms in one encoder deployment are refused by the mixing rule" \ - "an unadmitted platform exists (${OTHER_MANUFACTURER}) but no usable InstanceType reports an admitted one, so there is nothing to mix it with" - fi -else - record SKIP "a platform the matrix does not admit is refused, all three named" \ - "no usable InstanceType reports any of the unadmitted manufacturers this operator declares (${UNADMITTED_PLATFORMS// /, }; the cluster holds: ${SEEN:-}), so there is no unadmitted platform to name" - record SKIP "an unadmitted platform is refused naming the engine too" \ - "no usable InstanceType reports any of the unadmitted manufacturers this operator declares (${UNADMITTED_PLATFORMS// /, }; the cluster holds: ${SEEN:-}), so there is no unadmitted platform to name" - record SKIP "two platforms in one encoder deployment are refused by the mixing rule" \ - "no usable InstanceType reports any of the unadmitted manufacturers this operator declares (${UNADMITTED_PLATFORMS// /, }; the cluster holds: ${SEEN:-}), so there is no second manufacturer to mix with" -fi - -# ------------------------------------------------- why the functional half lives elsewhere - -# Stated as a row rather than left to the header, because a case that reads as a refusal-only case -# with no functional rows is otherwise indistinguishable from one that forgot to write them. The -# admitted platforms' functional readings are their own cases, and no unadmitted platform has an -# object to run until a live proof of concept admits its row. -record SKIP "an unadmitted platform serves a multimodal request through the router" \ - "the matrix admits ascend, nvidia and amd only, so any other platform has no object to run. The admitted platforms' functional readings are case 120 (the NVIDIA and AMD network leg) and case 121 (the Ascend shared-directory leg); this case asserts the refusal for every platform that has no row, and nothing else" - -# Results. -print_rows -[ "$FAILS" -eq 0 ] || { echo "[case-124] ${FAILS} check(s) FAILED"; exit 1; } -echo "[case-124] all checks passed" \ No newline at end of file diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-125.sh b/.agents/skills/gpustack-operator-e2e/cases/case-125.sh deleted file mode 100755 index 97670c91f..000000000 --- a/.agents/skills/gpustack-operator-e2e/cases/case-125.sh +++ /dev/null @@ -1,621 +0,0 @@ -#!/usr/bin/env bash -# -# CASE 125 — The llm-d-router picker carries the encoder stage: an encode filter, an unscored -# encode profile, and the multimodal decider (MUTATING, self-recovering) -# AUTO-SKIPS without an InstanceType that reports an admitted accelerator platform, which -# this case builds by injecting a CPU vendor label -# -# case-125.sh -# -# Goal: llm-d-router is the ONLY router both admitted rows name, so what its picker document -# says about an encoder is the part of the router that carries this whole feature. -# Cases 116 to 119 read admission, Services and mounts; case 120 and 121 send a real -# request through the router. Nothing reads the document itself. This case reads it, and -# reads the three facts in it that an encoder changes: -# -# - an ENCODE FILTER (label-selector-filter, selecting role-kind=encode) exists; -# - the ENCODE PROFILE names that filter and NOTHING ELSE: no queue scorer, no -# KV-cache scorer, no prefix scorer, no picker; -# - the MULTIMODAL DECIDER (always-disagg-multimodal-decider) is present, and the -# disagg-profile-handler routes BOTH "prefill" and "encode" at it. -# -# THE ENCODE PROFILE'S EMPTINESS IS THE ROW THAT MATTERS, and it is asserted by -# content rather than by absence of a key. An encoder keeps no KV cache, so the queue -# and KV-cache scorers read gauges it never exports and rank every candidate -# identically; the prefix scorer asks about token overlap, which does not exist before -# a request has tokens. A scorer that cannot separate candidates is not a weak -# preference, it is a cost paid on every media request. Upstream completes any profile -# naming none with its own max-score-picker at load time, so an empty plugin list is -# valid and still runs — which is why the row asserts the exact list and not merely -# "no prefix scorer". -# -# The decider's ABSENCE would be the other failure worth catching: without it the -# encoder stage is never selected and a multimodal request is answered by a -# prefiller running undivided, which looks like success and is not the disaggregation -# this feature exists to perform. -# -# Environment: Any cluster with a materialized scheduling chain (run case-1 first) and at least one -# Ready, schedulable worker node that is not a control-plane node. NO accelerator: the -# encoder shape reaches admission through the platform an injected CPU vendor label -# builds ("Ascend" — an admitted row whose connector is the mounted directory). The -# gpustack-settings Secret must exist in the system namespace (E2E_SYSTEM_NS, default -# gpustack-system). -# -# AUTO-SKIP: the subject needs the platform InstanceType, and the vendor label may not -# take (NFD owns cpu-model.vendor_id and may decline an injected override). It also -# needs the pool's entrance LocalQueue in , because that is what admits the groups -# and therefore what renders the router document. Either one absent records a SKIP -# naming it and exits 0. The router image may also be unpullable: the rows read the -# router's CONFIGMAP, which the operator renders before any image is pulled, so an -# unpullable image does not skip them — only the Pod-level row that checks the document -# reached a running process. -# -# Inputs: MOCKED: one node label (the CPU vendor) injected through a NodeFeature object and one -# settings key (CPU-manufacturer awareness) flipped on — both restored by the trap. -# REAL: one PV, one PVC (a hostPath volume, manually bound) and one ModelDeployment -# whose encode and prefill roles mount the claim at one path, exactly as case 119 -# builds it; the router ConfigMap the operator renders is the subject. -# -# Expected: - the deployment is admitted and created, and ALL THREE of its role Pods exist (the -# control deployment's two are checked against its own role count; none need run); -# - the router ConfigMap carries an EndpointPickerConfig with one filter per role, and -# the encode filter matches role-kind=encode; -# - the encode profile's plugin list is EXACTLY the encode filter's reference; -# - the always-disagg-multimodal-decider is present, and the disagg-profile-handler -# maps both prefill and encode to a decider; -# - the precise prefix-cache producer EXCLUDES role-kind=decode and role-kind=encode -# from its podLabelSelector, because neither publishes KV events; -# - for contrast, a deployment with NO encoder role renders no multimodal decider and -# no encode profile — the same document, read a second time. -# -# Cleanup: The trap deletes the deployment, releases any Workload still holding its replicas, -# deletes the PVC and the PV, the NodeFeature object, and restores the settings key to -# its exact original value, then waits for the chain to settle back: the derived type -# goes only once the setting cache flips and the pool re-derives. Idempotent, runs on -# pass AND fail, safe to re-run. -set -uo pipefail - -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" - -NS="${1:-}" -if [ -z "$NS" ]; then - echo "usage: case-125.sh " >&2 - exit 2 -fi - -SYSTEM_NS="${E2E_SYSTEM_NS:-gpustack-system}" -SETTING=instance-type-aware-cpu-manufacturer -VENDOR="${E2E_C125_VENDOR:-Ascend}" -PLATFORM="$(printf '%s' "$VENDOR" | tr '[:upper:]' '[:lower:]')" -NF=case125-cpu-vendor -MD=case125-epd -PD=case125-pd -CLAIM=case125-claim -PV=case125-pv -IMAGE="${E2E_MD_IMAGE:-registry.k8s.io/pause:3.10}" -MOUNT=/ec-cache -SETTLE_ROUNDS="${E2E_C125_SETTLE:-60}" -POD_ROUNDS="${E2E_C125_PODS:-40}" - -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } -# shellcheck source=/dev/null -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" - -command -v jq >/dev/null 2>&1 || { echo "[case-125] jq is required." >&2; exit 2; } - -pick_worker_node() { - kubectl get nodes -o json 2>/dev/null | python3 -c ' -import json,sys -for n in json.load(sys.stdin).get("items",[]): - cs=n.get("status",{}).get("conditions",[]) - if not any(c.get("type")=="Ready" and c.get("status")=="True" for c in cs): - continue - if n.get("spec",{}).get("unschedulable"): - continue - if n.get("metadata",{}).get("labels",{}).get("feature.gpustack.ai/acceleratable") == "true": - continue - ts=n.get("spec",{}).get("taints",[]) - if any(str(t.get("key","")).startswith("node-role.kubernetes.io/") for t in ts): - continue - print(n["metadata"]["name"]) - break -' -} - -epd_platform_type() { - kubectl get instancetypes.worker.gpustack.ai \ - -o jsonpath='{range .items[*]}{.metadata.name}|{.metadata.deletionTimestamp}|{.spec.inactive}|{.status.detail.manufacturer}{"\n"}{end}' \ - 2>/dev/null \ - | while IFS='|' read -r name deleting inactive manufacturer; do - [ -n "$name" ] || continue - [ -z "$deleting" ] || continue - [ "$inactive" = true ] && continue - if [ "$manufacturer" = "$PLATFORM" ]; then - echo "$name" - break - fi - done -} - -ORIG_SETTING="$(kubectl -n "$SYSTEM_NS" get secret gpustack-settings -o jsonpath="{.data.${SETTING}}" 2>/dev/null)" -set_awareness() { # true|false - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"${SETTING}\":\"$(printf '%s' "$1" | base64)\"}}" >/dev/null 2>&1 -} -restore_setting() { - local v=null - [ -n "$ORIG_SETTING" ] && v="\"${ORIG_SETTING}\"" - kubectl -n "$SYSTEM_NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"${SETTING}\":${v}}}" >/dev/null 2>&1 -} - -# Restoring the value is not the end of the restore: the worker reads it through a thirty-second -# cache, and the type derived from it outlives the patch by that cache plus one reconcile. Wait for -# the derived type to go, so the next case in a serial run does not list an aware type this run left -# behind. A cluster that started with the setting already on keeps its derived type, so there is -# nothing to wait for. -settle_back() { - [ "$(printf '%s' "$ORIG_SETTING" | base64 -d 2>/dev/null)" = "true" ] && return 0 - for _ in $(seq 1 "$SETTLE_ROUNDS"); do - [ -z "$(epd_platform_type)" ] && return 0 - sleep 5 - done - return 0 -} - -# One list call, because this runs from the EXIT trap: Kueue holds a finalizer on every Pod of a -# serving group and releases it only when the Workload is deleted. -force_release() { - local md="$1" row wl uids owners u - uids="$(kubectl -n "$NS" get pods -l "app.kubernetes.io/instance=${md}" \ - -o jsonpath='{range .items[*]}{.metadata.uid}{"\n"}{end}' 2>/dev/null)" - [ -n "$uids" ] || return 0 - while IFS= read -r row; do - [ -n "$row" ] || continue - wl="${row%%=*}" - owners="${row#*=}" - for u in $uids; do - case " $owners " in - *" $u "*) - kubectl -n "$NS" delete workloads.kueue.x-k8s.io "$wl" \ - --ignore-not-found --wait=false >/dev/null 2>&1 - break - ;; - esac - done - done </dev/null) -EOF - return 0 -} - -INJECTED=false -cleanup() { - echo - echo "[case-125] cleanup" - for md in "$MD" "$PD"; do - kubectl -n "$NS" delete modeldeployments.worker.gpustack.ai "$md" \ - --ignore-not-found --wait=false >/dev/null 2>&1 - done - for _ in $(seq 1 12); do - sleep 5 - force_release "$MD" - force_release "$PD" - [ -z "$(kubectl -n "$NS" get pods -l "app.kubernetes.io/instance=${MD}" --no-headers 2>/dev/null)" ] && - [ -z "$(kubectl -n "$NS" get pods -l "app.kubernetes.io/instance=${PD}" --no-headers 2>/dev/null)" ] && break - done - kubectl -n "$NS" delete pvc "$CLAIM" --ignore-not-found --wait=false >/dev/null 2>&1 - kubectl delete pv "$PV" --ignore-not-found --wait=false >/dev/null 2>&1 - if [ "$INJECTED" = true ]; then - kubectl -n "$NS" delete nodefeatures.nfd.k8s-sigs.io "$NF" --ignore-not-found >/dev/null 2>&1 - restore_setting - settle_back - fi -} -trap cleanup EXIT - -# ---------------------------------------------------------------- the fixture - -IT="$(epd_platform_type)" -if [ -n "$IT" ]; then - record PASS "a CPU-only pool reports an admitted platform" \ - "instancetype ${IT} reports manufacturer '${PLATFORM}'" -else - NODE="$(pick_worker_node)" - if [ -z "$NODE" ]; then - echo "[case-125] no Ready schedulable worker node; this case cannot place its vendor label" >&2 - exit 2 - fi - - echo "[case-125] injecting CPU vendor '${VENDOR}' on node ${NODE} and turning CPU-manufacturer awareness on" - INJECTED=true - - kubectl apply -f - >/dev/null 2>&1 </dev/null)" -ENTRANCE_LQ="" -for _ in $(seq 1 "$SETTLE_ROUNDS"); do - ENTRANCE_LQ="$(kubectl -n "$NS" get localqueue "${ENTRANCE:-}" -o name 2>/dev/null)" - [ -n "$ENTRANCE_LQ" ] && break - sleep 5 -done - -if [ -z "$ENTRANCE_LQ" ]; then - record SKIP "the pool's entrance LocalQueue exists" \ - "instancetype ${IT} names entrance '${ENTRANCE:-}' and no localqueue/${ENTRANCE:-} answered in ${NS} within $((SETTLE_ROUNDS * 5))s, so nothing is admitted, no role Pod is created and no router document is ever rendered" - print_rows - echo "[case-125] skipped: no entrance LocalQueue" - exit 0 -fi - -record PASS "the pool's entrance LocalQueue exists" "${ENTRANCE_LQ##*/} in ${NS}" - -# The claim both encoder-side roles mount: a hostPath volume under /tmp on the labeled node, -# manually bound, exactly as case 119 builds it. The router document is the subject here, so the -# claim only has to be well formed enough for admission to accept the shape. -kubectl apply -f - >/dev/null 2>&1 </dev/null 2>&1; then - record PASS "the encoder deployment is admitted and created" \ - "one encode, one prefill and one decode role on ${IT}, llm-d-router" -else - record FAIL "the encoder deployment is admitted and created" \ - "no modeldeployment/${MD} after the apply: $(printf '%s' "${APPLY_OUT:-}" | tr '\n' ' ' | cut -c1-220)" - print_rows - exit 1 -fi - -# The CONTROL deployment is the same router with no encoder at all. Reading the same document twice -# is what makes the encoder's three facts encoder facts rather than prefill/decode facts that happen -# to be there: every term the control lacks is a term the encoder added. -# -# Its apply output is KEPT rather than discarded. A refusal here would otherwise surface as the -# pod-count row below, which names a missing Pod and not the refusal that caused it — the same -# wrong-cause report the LocalQueue poll was fixed for. -PD_APPLY_OUT="$(cat <&1 -apiVersion: worker.gpustack.ai/v1alpha1 -kind: ModelDeployment -metadata: - name: ${PD} - namespace: ${NS} -spec: - engine: - name: vLLM - image: ${IMAGE} - model: - name: Qwen/Qwen2.5-0.5B-Instruct - router: - name: llm-d-router - roles: - - name: prefill - kind: Prefill - instanceType: ${IT} - replicas: 1 - image: ${IMAGE} - - name: decode - kind: Decode - instanceType: ${IT} - replicas: 1 - image: ${IMAGE} -YAML -)" - -# An empty output is not an accepted apply, so it counts as refused here too. -PD_REFUSED=no -case "$PD_APPLY_OUT" in - *created* | *configured*) ;; - *) PD_REFUSED=yes ;; -esac - -# THE ENCODER DEPLOYMENT RENDERS THREE ROLE PODS AND THE CONTROL TWO, and the count each row -# waits for is that deployment's own role count rather than a flat number: a two-of-three gate -# would pass a renderer that dropped the encoder, which is the failure this case exists to catch. -for md in "$MD" "$PD"; do - if [ "$md" = "$MD" ]; then want=3; else want=2; fi - count="" - for _ in $(seq 1 "$POD_ROUNDS"); do - count="$(kubectl -n "$NS" get pods -l "app.kubernetes.io/instance=${md}" --no-headers 2>/dev/null | grep -c . || true)" - [ "${count:-0}" -ge "$want" ] && break - sleep 5 - done - if [ "${count:-0}" -ge "$want" ]; then - record PASS "${md} rendered its role Pods" "${count} Pods for ${want} roles; the rows below read the router document, not the containers" - else - # A refused control apply is the cause of a missing control Pod, so it is named here instead of - # leaving the row to be read as a renderer that dropped a role. - cause="" - if [ "$md" = "$PD" ] && [ "$PD_REFUSED" = yes ]; then - cause="; its apply was refused: $(printf '%s' "$PD_APPLY_OUT" | tr '\n' ' ' | cut -c1-220)" - fi - record FAIL "${md} rendered its role Pods" \ - "only ${count:-0} of ${want} role Pods appeared within $((POD_ROUNDS * 5))s, so the router ConfigMap may never be rendered${cause}" - fi -done - -# ---------------------------------------------------------------- the picker document - -# The document as YAML, from the ConfigMap the operator renders into the router Pod. A router -# configured by argv renders none, so its ABSENCE is a distinct answer from an empty one and both -# are read rather than assumed. -picker_yaml() { # - kubectl -n "$NS" get configmap "$1-router" -o jsonpath='{.data.epp\.yaml}' 2>/dev/null -} - -DOC="$(picker_yaml "$MD")" -if [ -z "$DOC" ]; then - record FAIL "the router ConfigMap carries an EndpointPickerConfig" \ - "no configmap/${MD}-router carries a key epp.yaml, so there is no document to read" - print_rows - [ "$FAILS" -eq 0 ] || { echo "[case-125] ${FAILS} check(s) FAILED"; exit 1; } - echo "[case-125] all checks passed" - exit 0 -fi - -# Each check runs the document through jq as YAML (jq reads YAML only with -y on some builds), so -# the document is converted once with python3, which every case in this suite already requires. -json_of() { # yaml-on-stdin - python3 -c ' -import json,sys -try: - import yaml -except ImportError: - sys.stderr.write("pyyaml is required to read the router document") - raise SystemExit(3) -d = yaml.safe_load(sys.stdin.read()) -print(json.dumps(d) if d is not None else "{}") -' -} - -DOC_JSON="$(printf '%s' "$DOC" | json_of)" || { - echo "[case-125] cannot read the router document: pyyaml is required in the runner." >&2 - exit 2 -} -PD_JSON="$(printf '%s' "$(picker_yaml "$PD")" | json_of)" || PD_JSON="" - -if printf '%s' "$DOC_JSON" | jq -e '.kind == "EndpointPickerConfig" and (.plugins | type == "array")' \ - >/dev/null 2>&1; then - record PASS "the router ConfigMap carries an EndpointPickerConfig" \ - "$(printf '%s' "$DOC_JSON" | jq -r '.plugins | length') plugins and $(printf '%s' "$DOC_JSON" | jq -r '.schedulingProfiles | length') scheduling profiles" -else - record FAIL "the router ConfigMap carries an EndpointPickerConfig" \ - "kind=$(printf '%s' "$DOC_JSON" | jq -r '.kind // ""'), and the document has no plugins array to read" -fi - -# The encode filter: one label-selector-filter whose matchLabels select role-kind=encode. The label -# key is the role-kind label the operator stamps on every Pod, so this is the selector the router -# will actually match against, read off the document rather than assumed from the render. -FILTER_OK="$(printf '%s' "$DOC_JSON" | jq -r ' - [ .plugins[]? - | select(.type == "label-selector-filter") - | select(.parameters.matchLabels["modeldeployment.gpustack.ai/role-kind"] == "encode") - ] | length')" -if [ "${FILTER_OK:-0}" -ge 1 ]; then - record PASS "an encode filter selects the encoder's Pods" \ - "a label-selector-filter matches modeldeployment.gpustack.ai/role-kind=encode" -else - record FAIL "an encode filter selects the encoder's Pods" \ - "no label-selector-filter matches role-kind=encode; the picker would never route to an encoder" -fi - -# THE ENCODE PROFILE'S PLUGIN LIST, asserted exactly. Upstream completes any profile naming none with -# its own max-score-picker at load time, so [encode-pods] is a valid, runnable profile — and any -# additional plugin is a scorer that cannot separate encoder candidates. -ENCODE_PLUGINS="$(printf '%s' "$DOC_JSON" | jq -c \ - '[.schedulingProfiles[]? | select(.name == "encode") | .plugins[]? | .pluginRef] | unique')" -if [ "$ENCODE_PLUGINS" = '["encode-pods"]' ]; then - record PASS "the encode profile scores on nothing but its own filter" \ - "plugins=${ENCODE_PLUGINS}: upstream appends its max-score-picker to a profile naming none" -else - record FAIL "the encode profile scores on nothing but its own filter" \ - "encode profile plugins=${ENCODE_PLUGINS}, expected [\"encode-pods\"]: an encoder keeps no KV cache, so the queue and KV scorers read gauges it never exports and the prefix scorer asks about tokens that do not exist yet" -fi - -# The decider, and the handler that routes BOTH halves at one. Without the encode term in the -# handler the decider is present but never consulted, and the encoder stage is silently unused. -DECIDER="$(printf '%s' "$DOC_JSON" | jq -r \ - '[.plugins[]? | select(.type == "always-disagg-multimodal-decider")] | length')" -HANDLER="$(printf '%s' "$DOC_JSON" | jq -c \ - '[.plugins[]? | select(.type == "disagg-profile-handler") | .parameters] | first // {}')" -if [ "${DECIDER:-0}" -ge 1 ]; then - record PASS "the multimodal decider is present" "always-disagg-multimodal-decider is in the document" -else - record FAIL "the multimodal decider is present" \ - "no always-disagg-multimodal-decider: without it the encoder stage is never selected and a media request is answered undivided" -fi - -HANDLER_ENCODE="$(printf '%s' "$HANDLER" | jq -r '.profiles.encode // ""')" -HANDLER_PREFILL="$(printf '%s' "$HANDLER" | jq -r '.profiles.prefill // ""')" -HANDLER_DECIDER_ENCODE="$(printf '%s' "$HANDLER" | jq -r '.deciders.encode // ""')" -if [ "$HANDLER_ENCODE" = encode ] && [ "$HANDLER_PREFILL" = prefill ] \ - && [ "$HANDLER_DECIDER_ENCODE" = always-disagg-multimodal-decider ]; then - record PASS "the handler routes the encoder stage at the multimodal decider" \ - "profiles.encode=encode, profiles.prefill=prefill, deciders.encode=always-disagg-multimodal-decider" -else - record FAIL "the handler routes the encoder stage at the multimodal decider" \ - "profiles.encode='${HANDLER_ENCODE}' profiles.prefill='${HANDLER_PREFILL}' deciders.encode='${HANDLER_DECIDER_ENCODE}'" -fi - -# The precise prefix-cache producer's Pod discovery EXCLUDES the two kinds that publish no events. -# The exclusion is what makes the terms exist at all, so the row reads the selector string rather -# than the plugin's presence. -# -# THE KEY IS podLabelSelector, which is what the renderer writes. A producer configured by a -# different field would render a document this case could not read, and reading one that does not -# exist would answer "no producer" on a cluster that has one — so the key is named here rather than -# guessed, and the failure message quotes it. -DISCOVERY="$(printf '%s' "$DOC_JSON" | jq -r ' - [ .plugins[]? - | select(.type == "precise-prefix-cache-producer") - | .parameters.kvEventsConfig.podDiscoveryConfig.podLabelSelector // "" - ] | first // ""')" -case "$DISCOVERY" in - *role-kind!=decode*role-kind!=encode* | *role-kind!=encode*role-kind!=decode*) - record PASS "the prefix-cache producer excludes the kinds that publish no events" \ - "podLabelSelector: ${DISCOVERY}" - ;; - "") - # The renderer picks the PRECISE producer whenever the deployment opens a KV event port, and an - # encoder leg does — so this branch is the alternative a renderer can still emit, not the - # expected shape: the approximate producer discovers Pods by service and carries no role - # selector, so there is nothing to exclude and nothing wrong. It is a PASS, because a - # deployment that renders no events has no eventless kind to exclude. - APPROX="$(printf '%s' "$DOC_JSON" | jq -r \ - '[.plugins[]? | select(.type == "approx-prefix-cache-producer")] | length')" - if [ "${APPROX:-0}" -ge 1 ]; then - record PASS "the prefix-cache producer excludes the kinds that publish no events" \ - "the approximate producer renders instead, which discovers Pods by service and has no role selector to exclude" - else - record FAIL "the prefix-cache producer excludes the kinds that publish no events" \ - "neither producer is in the document, so neither kind is excluded and nothing is subscribed" - fi - ;; - *) - record FAIL "the prefix-cache producer excludes the kinds that publish no events" \ - "podLabelSelector '${DISCOVERY}' lacks role-kind!=decode or role-kind!=encode: both kinds publish no KV events and would be subscribers dialing a socket the engine never opens" - ;; -esac - -# ---------------------------------------------------------------- the control deployment - -# The same router, the same engine, one role fewer. Every term the control lacks is a term the -# encoder added, which is what makes the rows above encoder facts rather than router defaults. -if [ -z "$PD_JSON" ]; then - record SKIP "a deployment with no encoder role renders no encoder stage" \ - "no configmap/${PD}-router carries a key epp.yaml, so the control document could not be read; the encoder rows above stand on their own" -else - CTRL_DECIDER="$(printf '%s' "$PD_JSON" | jq -r \ - '[.plugins[]? | select(.type == "always-disagg-multimodal-decider")] | length')" - CTRL_PROFILE="$(printf '%s' "$PD_JSON" | jq -r \ - '[.schedulingProfiles[]? | select(.name == "encode")] | length')" - CTRL_FILTER="$(printf '%s' "$PD_JSON" | jq -r \ - '[.plugins[]? | select(.type == "label-selector-filter") - | select(.parameters.matchLabels["modeldeployment.gpustack.ai/role-kind"] == "encode")] | length')" - if [ "${CTRL_DECIDER:-0}" -eq 0 ] && [ "${CTRL_PROFILE:-0}" -eq 0 ] && [ "${CTRL_FILTER:-0}" -eq 0 ]; then - record PASS "a deployment with no encoder role renders no encoder stage" \ - "the control's picker carries no multimodal decider, no encode profile and no encode filter, so all three are the encoder's and not the router's defaults" - else - record FAIL "a deployment with no encoder role renders no encoder stage" \ - "the control still carries decider=${CTRL_DECIDER} profile=${CTRL_PROFILE} filter=${CTRL_FILTER}, which would mean the encoder rows above prove nothing about the encoder" - fi -fi - -# Results. -print_rows -[ "$FAILS" -eq 0 ] || { echo "[case-125] ${FAILS} check(s) FAILED"; exit 1; } -echo "[case-125] all checks passed" \ No newline at end of file diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-126.sh b/.agents/skills/gpustack-operator-e2e/cases/case-126.sh index 8429dd94f..5d6782c8a 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/case-126.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-126.sh @@ -1,27 +1,39 @@ #!/usr/bin/env bash # -# CASE 126 — a ModelDeployment without spec.engine.image is refused by the API server's schema, -# before any webhook runs (NON-MUTATING: server-side dry runs only) -# AUTO-SKIPS the accepted-baseline row without a usable InstanceType reporting an -# admitted platform +# CASE 126 — Two admission rules on one ModelDeployment: the engine image the schema requires, +# and the Router the SGLang external shape requires +# (NON-MUTATING: server-side dry runs only) +# AUTO-SKIPS the engine-image baseline row without a usable InstanceType reporting an +# admitted platform # # case-126.sh # -# Goal: spec.engine.image is a REQUIRED field: the CRD lists it beside engine.name in the -# schema's Required set, and the webhook rule that used to police the engine block -# was deleted with the version field because the schema carries the obligation now. -# The schema reports the obligation in two wordings: "Required value" when the key -# is absent, and "Invalid value: \"\"" when a mutating patch has materialized the -# key as an empty string. The patch is ordinary: the webhook defaults -# resources.accelerator for a role that states none, the typed object is -# re-marshaled, and engine.image has no omitempty, so the patch carries -# "image": "" before the schema runs. Schema validation runs BEFORE admission +# Goal: Two contracts, both answered at admission, both about the same object. +# +# THE ENGINE IMAGE IS A SCHEMA OBLIGATION. spec.engine.image is a REQUIRED field: the +# CRD lists it beside engine.name in the schema's Required set, and the webhook rule +# that used to police the engine block was deleted with the version field because the +# schema carries the obligation now. The schema reports the obligation in two +# wordings: "Required value" when the key is absent, and "Invalid value: \"\"" when +# a mutating patch has materialized the key as an empty string. The patch is +# ordinary: the webhook defaults resources.accelerator for a role that states none, +# the typed object is re-marshaled, and engine.image has no omitempty, so the patch +# carries "image": "" before the schema runs. Schema validation runs BEFORE admission # webhooks, so neither wording can be a webhook refusal wearing a schema's clothes, # and the field path is load-bearing in both. # -# THE ACCEPTED BASELINE IS WHAT MAKES THE REFUSALS MEAN SOMETHING. The same manifest -# with engine.image present is accepted by the same instrument, so a refusal below is -# about the missing field and not about the deployment being wrong in some other way. +# THE EXTERNAL SGLANG SHAPE IS A WEBHOOK OBLIGATION. An SGLang role that declares the +# external shape — attention width at least two, a node count of at least two, and +# the Rust server on — is served through a Router, so a deployment that declares one +# without a Router is REFUSED naming the rule, and the same role WITH a Router is +# ACCEPTED. The controls matter as much as the rows: a role that declares LESS stays +# leader-served Internal and is accepted without a Router, which proves the refusal +# is the external-shape rule and not a blanket rejection of SGLang multi-Pod roles. +# +# THE ACCEPTED BASELINE IS WHAT MAKES THE ENGINE-IMAGE REFUSALS MEAN SOMETHING. The +# same manifest with engine.image present is accepted by the same instrument, so a +# refusal below is about the missing field and not about the deployment being wrong in +# some other way. # # A ROLE IMAGE IS NOT A SUBSTITUTE. A role's own image overrides the engine-wide one # at render time — a different field with a different job — so the requirement sits @@ -30,18 +42,21 @@ # exactly the shape a reader could mistake for a substitute. # # Environment: Any reachable API server with the ModelDeployment CRD installed (run case-1 first -# to materialize the scheduling chain). No accelerator, no node mutation, no -# settings change. The accepted-baseline row additionally needs a usable -# InstanceType reporting an admitted platform (ascend, nvidia or amd), because the -# webhook reads the type a role names; the refusal rows never reach a webhook and -# run everywhere. +# to materialize the scheduling chain). No accelerator, no engine image, no GPU, no +# node mutation and no settings change: every assertion is a server-side dry run, so +# nothing is stored and no Pod is ever started. The engine-image baseline row +# additionally needs a usable InstanceType reporting an admitted platform (ascend, +# nvidia or amd), because the webhook reads the type a role names; its refusal rows +# never reach a webhook and run everywhere. # # AUTO-SKIP: without such an InstanceType the baseline row records SKIP and the # refusal rows still run. # # Inputs: MOCKED: nothing. REAL: the manifests below are the only input; the image is # E2E_MD_IMAGE (default registry.k8s.io/pause:3.10) and is never pulled — every -# assertion is a server-side dry run and nothing is stored. +# assertion is a server-side dry run and nothing is stored. The SGLang rows name the +# InstanceType the caller picked (E2E_MD_INSTANCE_TYPE) or a name that resolves to +# nothing, because they assert an admission rule and not a scheduling one. # # Expected: - the deployed operator build is recorded, never asserted: the image tag is the # only handle this case has on which CRD the cluster installed; @@ -57,14 +72,20 @@ # the patch materializes, the schema's minimum length refuses it once that lands, # and the row carries the observed answer either way; # - an engine naming only its name, with every role carrying its own image, is STILL -# REFUSED — a role image does not satisfy the engine-level requirement. +# REFUSED — a role image does not satisfy the engine-level requirement; +# - the SGLang controls — attention width one, a node count of one, and the Rust +# server off — are each ACCEPTED without a Router, because each of them stays +# leader-served Internal; +# - the external SGLang shape with NO router is REFUSED naming RouterRequired and +# the role, and the same shape WITH sglang-gateway or with llm-d-router is +# ACCEPTED. # # Cleanup: Nothing to clean. No object is created, no label injected, no setting changed; # every assertion is a server-side dry run. Safe to re-run. set -uo pipefail -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:-}" if [ -z "$NS" ]; then @@ -73,42 +94,24 @@ if [ -z "$NS" ]; then fi SYSTEM_NS="${E2E_SYSTEM_NS:-gpustack-system}" -MD=case126-epd +MD=case126-probe IMAGE="${E2E_MD_IMAGE:-registry.k8s.io/pause:3.10}" ENGINE=vLLM -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } # shellcheck source=/dev/null -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_md-lib.sh" +# ---------------------------------------------------------------- the type the rows name -# ---------------------------------------------------------------- the type the baseline names - -# The baseline row must pass BOTH the schema and the webhook, and the webhook reads the platform -# the InstanceType a role names reports, so the baseline needs a real type reporting a platform -# the matrix admits. The refusal rows below never reach the webhook — schema validation fails -# first — so they run with or without one. -usable_type_for() { # - kubectl get instancetypes.worker.gpustack.ai -o jsonpath='{range .items[*]}{.metadata.name}|{.metadata.deletionTimestamp}|{.spec.inactive}|{.status.detail.manufacturer}{"\n"}{end}' \ - 2>/dev/null \ - | while IFS='|' read -r name deleting inactive manufacturer; do - [ -n "$name" ] || continue - [ -z "$deleting" ] || continue - [ "$inactive" = true ] && continue - [ -n "$manufacturer" ] || continue - if [ "$manufacturer" = "$1" ]; then - echo "$name" - break - fi - done -} - +# The engine-image baseline row must pass BOTH the schema and the webhook, and the webhook reads +# the platform the InstanceType a role names reports, so the baseline needs a real type reporting a +# platform the matrix admits. The refusal rows below never reach the webhook — schema validation +# fails first — so they run with or without one. The type lookup is the library's: +# md_usable_type_for. IT="" MFG="" for m in ascend nvidia amd; do - found="$(usable_type_for "$m")" + found="$(md_usable_type_for "$m")" if [ -n "$found" ]; then IT="$found" MFG="$m" @@ -116,41 +119,84 @@ for m in ascend nvidia amd; do fi done -# ---------------------------------------------------------------- the precondition +# The SGLang rows name the InstanceType the caller picked, and nothing else: they assert an +# admission rule about the role's shape, not a scheduling one, so a name that resolves to nothing +# is as good as a real type and never turns the row into a scheduling question. +SGLANG_IT="${E2E_MD_INSTANCE_TYPE:-unset}" -# THE INSTRUMENT IS A DEPLOYED BUILD, and this row is what says which one. A CRD is installed by -# the release under test, so a cluster still carrying the previous schema would answer every row -# below about the previous schema: the accepted baseline would be refused for naming a field the -# old CRD does not have, and every refusal would quote "unknown field" instead of the missing -# property. The image tag is the only handle this case has on that, so it is read, recorded, and -# never asserted — pinning a tag here would break the moment the image is rebuilt. -WORKER_DEPLOY="$(kubectl -n "$SYSTEM_NS" get deploy gpustack-operator-worker -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null)" -if [ -n "$WORKER_DEPLOY" ]; then - record PASS "the deployed operator build is recorded" "worker image ${WORKER_DEPLOY}" -else - record SKIP "the deployed operator build is recorded" \ - "no gpustack-operator-worker Deployment in ${SYSTEM_NS}, so the rows below depend on whichever CRD this cluster installed without this case saying which" -fi +# ---------------------------------------------------------------- the manifests' parts + +# The engine blocks. The key is DELETED, not emptied, in the no-image block: the schema answers a +# MISSING required property, and a missing property is what "Required value" names. The empty-image +# block writes out the value the defaulting patch materializes. +ENGINE_OK=" name: ${ENGINE} + image: ${IMAGE}" +ENGINE_NO_IMAGE=" name: ${ENGINE}" +ENGINE_EMPTY=" name: ${ENGINE} + image: \"\"" +ENGINE_SGLANG=" name: SGLang + image: ${IMAGE}" + +# The decode role the engine-image rows vary. It names the engine image (or does not) and the +# accelerator count (or does not), and the pair of refusal rows below differ in exactly the thing a +# reader might think substitutes for the engine default, so their agreement is evidence. +ROLE_PLAIN=" - name: decode + kind: Decode + instanceType: ${IT:-case126-any}" + +ROLE_WITH_IMAGE=" - name: decode + kind: Decode + instanceType: ${IT:-case126-any} + image: ${IMAGE}" -# ---------------------------------------------------------------- the manifests - -# ROLE_IMAGE is the second dial, and the pair of refusal rows below is the reason it exists. With -# the role image present in one row and absent in the other, the two rows differ in exactly the -# thing a reader might think substitutes for the engine default, so their agreement is evidence. -# Two rows rendering the SAME manifest would agree for the trivial reason and prove nothing. -WITH_IMAGE=" image: ${IMAGE}" -NO_IMAGE="" -ROLE_IMAGE="$WITH_IMAGE" -# ROLE_RESOURCES is the third dial. Pinning resources.accelerator: "1" in the role leaves the -# mutating webhook nothing to default, so it emits no patch and the manifest reaches the schema -# exactly as written. That is the no-patch shape, and its refusal wording differs from the -# patched one. ACCELERATOR_PIN is its non-empty value; empty is the ordinary shape, whose webhook -# defaults the accelerator and emits its patch. -ROLE_RESOURCES="" -ACCELERATOR_PIN=' resources: - accelerator: "1"' - -manifest() { # engine-block +# ROLE_PINNED is the no-patch shape: pinning resources.accelerator: "1" leaves the mutating +# webhook nothing to default, so it emits no patch and the manifest reaches the schema exactly as +# written. That is the shape whose refusal wording differs from the patched one. +ROLE_PINNED=" - name: decode + kind: Decode + instanceType: ${IT:-case126-any} + resources: + accelerator: \"1\"" + +# The Rust server switch. The external shape needs it on; the control that turns it off declares +# no env block at all, which is what "off" means here. +RUST_ENV=' env: + - name: SGLANG_RUST_SERVER + value: "1"' + +# The SGLang role the external-shape rows vary. The external shape is three facts: the attention +# width is at least two, the node count is at least two, and the Rust server is on. Each control +# below removes exactly one of them. +sglang_role() { # attention-width node-count env-block + cat <&1 | tr '\n' ' ' +} + +# accepts asserts that the server ACCEPTED the manifest: both the schema and the webhook took it. +# An empty apply output is a FAIL on its own: "created" must be observed, because a row that +# passes on no output at all reports an acceptance nothing produced. The instrument is the +# library's: it reads the manifest, prints the text of the row and returns the verdict. +accepts() { # check engine roles [router] + local check="$1" engine="$2" roles="$3" router="${4:-}" obj status=0 + obj="$(md_dryrun_accepts < <(manifest "$engine" "$roles" "$router"))" || status=1 + if [ "$status" -eq 0 ]; then + record PASS "$check" "$obj" + else + record FAIL "$check" "$obj" + fi + return 0 +} -# Assert that a manifest is refused with the schema's own wording, naming the field path. An -# empty apply output is a FAIL on its own: deleting anything from "" leaves "". +# refuses asserts that the server REFUSED the manifest AND that the refusal names EVERY fragment +# the row is about. A refusal carrying somebody else's message too still passes — several rules +# aggregate into one response — but a MISSING fragment means the manifest tripped a rule this row +# is not about. # -# THE WORDING IS MEASURED, NOT GUESSED, and the two shapes the server can answer produce two -# wordings: "spec.engine.image: Required value" when the key reaches the schema absent (the -# no-patch shape), and "spec.engine.image: Invalid value: \"\"" when a defaulting patch has -# materialized it as an empty string. Either passes. The field path is load-bearing in both, so a -# refusal that says "Required value" about some other property of this object fails even though -# it carries the phrase — this case would otherwise claim a requirement it never read. -refuses() { - local check="$1" engine_block="$2" role_block="$3" resources_block="${4:-}" out - out="$(ROLE_IMAGE="$role_block" ROLE_RESOURCES="$resources_block" manifest "$engine_block" | kubectl apply --dry-run=server -f - 2>&1 | tr '\n' ' ')" +# AN EMPTY OUTPUT MUST NOT PASS. The refusal is read out of the apply's output, and an output that +# is empty at all means nothing was refused: the row would report a refusal it never saw. +refuses() { # check engine roles router fragment... + local check="$1" engine="$2" roles="$3" router="$4" want="" obj status=0 + shift 4 + if [ "$#" -gt 0 ]; then + want="$1" + shift + fi + obj="$(md_dryrun_refuses "$want" "$@" < <(manifest "$engine" "$roles" "$router"))" || status=1 + if [ "$status" -eq 0 ]; then + record PASS "$check" "$obj" + else + record FAIL "$check" "$obj" + fi + return 0 +} + +# refuses_one_of asserts that the server REFUSED the manifest with ONE of the two wordings the +# schema can answer, and the row says which wording it answered. The two shapes the server can +# answer produce two wordings: "spec.engine.image: Required value" when the key reaches the schema +# absent (the no-patch shape), and "spec.engine.image: Invalid value: \"\"" when a defaulting patch +# has materialized it as an empty string. Either one is the same refusal, so either one passes; the +# field path is load-bearing in both, so a refusal that says "Required value" about some other +# property of this object fails even though it carries the phrase. +refuses_one_of() { # check engine roles router wording wording + local check="$1" engine="$2" roles="$3" router="$4" first="$5" second="$6" out w + out="$(dry_run "$engine" "$roles" "$router")" if [ -z "$out" ]; then record FAIL "$check" "the apply produced no output at all, so nothing was refused" return 0 fi - case "$out" in - *"spec.engine.image: Required value"*) - record PASS "$check" "refused with: spec.engine.image: Required value" - ;; - *'spec.engine.image: Invalid value: ""'*) - record PASS "$check" 'refused with: spec.engine.image: Invalid value: ""' - ;; - *) - record FAIL "$check" \ - "not refused with either 'spec.engine.image: Required value' or 'spec.engine.image: Invalid value: \"\"' (got: ${out:0:200})" - ;; - esac + for w in "$first" "$second"; do + case "$out" in + *"$w"*) + record PASS "$check" "refused with: ${w}" + return 0 + ;; + esac + done + record FAIL "$check" \ + "not refused with either '${first}' or '${second}' (got: ${out:0:200})" return 0 } -accepts() { - local check="$1" engine_block="$2" role_block="$3" out - out="$(ROLE_IMAGE="$role_block" manifest "$engine_block" | kubectl apply --dry-run=server -f - 2>&1 | tr '\n' ' ')" - case "$out" in - *"created (server dry run)"* | *"configured (server dry run)"*) - record PASS "$check" "accepted by both the schema and the webhook" ;; - *) record FAIL "$check" "refused: ${out:0:220}" ;; - esac - return 0 -} +# ---------------------------------------------------------------- the precondition -# ---------------------------------------------------------------- row 0: the accepted baseline +# THE INSTRUMENT IS A DEPLOYED BUILD, and this row is what says which one. A CRD is installed by +# the release under test, so a cluster still carrying the previous schema would answer every row +# below about the previous schema: the accepted baseline would be refused for naming a field the +# old CRD does not have, and every refusal would quote "unknown field" instead of the missing +# property. The image tag is the only handle this case has on that, so it is read, recorded, and +# never asserted — pinning a tag here would break the moment the image is rebuilt. +WORKER_DEPLOY="$(kubectl -n "$SYSTEM_NS" get deploy gpustack-operator-worker -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null)" +if [ -n "$WORKER_DEPLOY" ]; then + record PASS "the deployed operator build is recorded" "worker image ${WORKER_DEPLOY}" +else + record SKIP "the deployed operator build is recorded" \ + "no gpustack-operator-worker Deployment in ${SYSTEM_NS}, so the rows below depend on whichever CRD this cluster installed without this case saying which" +fi + +# ---------------------------------------------------------------- the engine-image surface -# Without this row the refusals below prove nothing: a manifest wrong in some unrelated way would -# refuse too. The baseline is the SAME manifest with engine.image present, accepted by the same -# instrument. It names a real platform type when the cluster has one, because the webhook reads the -# type a role names; the refusal rows below fail schema validation before any webhook is called, so -# they run whether or not a type exists. +# Row 0: the accepted baseline. Without this row the refusals below prove nothing: a manifest +# wrong in some unrelated way would refuse too. The baseline is the SAME manifest with engine.image +# present, accepted by the same instrument. It names a real platform type when the cluster has one, +# because the webhook reads the type a role names; the refusal rows below fail schema validation +# before any webhook is called, so they run whether or not a type exists. if [ -n "$IT" ]; then - accepts "an engine.image baseline with a role set is ACCEPTED (${MFG})" "$ENGINE_OK" "$WITH_IMAGE" + accepts "an engine.image baseline with a role set is ACCEPTED (${MFG})" "$ENGINE_OK" "$ROLE_WITH_IMAGE" else record SKIP "an engine.image baseline with a role set is ACCEPTED" \ "no usable InstanceType reports an admitted platform (ascend, nvidia, amd), so there is no shape the webhook would admit; the refusal rows below do not need one" fi -# ---------------------------------------------------------------- spec.engine.image absent - -# THE HEADLINE OF THIS CASE, IN ITS ORDINARY SHAPE. The engine names its engine and nothing else, +# spec.engine.image absent, in its ordinary shape. The engine names its engine and nothing else, # and the role states no accelerator, so the mutating webhook defaults it and emits a patch that # materializes the image key as an empty string before the schema runs. Schema validation still # runs BEFORE admission webhooks, so this refusal cannot be the deleted webhook rule wearing a # schema's clothes; the wording this shape answers is the Invalid value one. -refuses "an engine with no image key is refused by the schema" "$ENGINE_NO_IMAGE" "$NO_IMAGE" - -# ---------------------------------------------------------------- spec.engine.image empty - -# RECORDED, NOT ASSERTED. An explicit "" is the value the defaulting patch materializes, written -# out in the manifest itself: once the schema carries a minimum length on engine.image, the server -# refuses it with Invalid value; on a schema without the minimum, the key is present and the -# server accepts. The row records whichever answer this build gives and carries it in its message. -# It is the one row here whose verdict is an observation, not a criterion: the case must read the -# same whether or not that minimum has landed. -out="$(ROLE_IMAGE="$WITH_IMAGE" manifest ' name: vLLM - image: ""' | kubectl apply --dry-run=server -f - 2>&1 | tr '\n' ' ')" +refuses_one_of "an engine with no image key is refused by the schema" \ + "$ENGINE_NO_IMAGE" "$ROLE_PLAIN" "" \ + "spec.engine.image: Required value" 'spec.engine.image: Invalid value: ""' + +# spec.engine.image empty. RECORDED, NOT ASSERTED. An explicit "" is the value the defaulting patch +# materializes, written out in the manifest itself: once the schema carries a minimum length on +# engine.image, the server refuses it with Invalid value; on a schema without the minimum, the key +# is present and the server accepts. The row records whichever answer this build gives and carries +# it in its message. It is the one row here whose verdict is an observation, not a criterion: the +# case must read the same whether or not that minimum has landed. +out="$(dry_run "$ENGINE_EMPTY" "$ROLE_WITH_IMAGE")" if [ -z "$out" ]; then record SKIP "an empty engine.image is recorded" \ "the apply produced no output at all, so there was no outcome to record" @@ -266,24 +347,50 @@ else esac fi -# ------------------------------------------------- a role image is not a substitute +# A role image is not a substitute. The tempting shortcut: every role names its own image, so +# what is the engine's for? A role's image OVERRIDES the engine-wide one at render time — a +# different field with a different job — so a deployment whose every role is pinned still has no +# engine-wide default, and that is refused. This row and the headline render DIFFERENT manifests — +# the headline's role names no image at all, this row's role does. That difference is the whole +# point: the two agree on the refusal while differing in exactly the field a reader might think +# substitutes for the engine default. +refuses_one_of "an engine with no image key is refused even when every role names an image" \ + "$ENGINE_NO_IMAGE" "$ROLE_WITH_IMAGE" "" \ + "spec.engine.image: Required value" 'spec.engine.image: Invalid value: ""' + +# The no-patch shape. The ordinary shape above is worded through the defaulting patch. This row +# differs from the headline by exactly one thing: the role pins resources.accelerator: "1", so the +# webhook has nothing to default and emits no patch. The image key then reaches the schema still +# absent, and the wording is the other one refuses_one_of accepts: Required value. The headline and +# this row cover both shapes the server can answer, because the two shapes produce different +# wordings. +refuses_one_of "an engine with no image key is refused even when the role pins its accelerator" \ + "$ENGINE_NO_IMAGE" "$ROLE_PINNED" "" \ + "spec.engine.image: Required value" 'spec.engine.image: Invalid value: ""' + +# ---------------------------------------------------------------- the SGLang external shape -# The tempting shortcut: every role names its own image, so what is the engine's for? A role's -# image OVERRIDES the engine-wide one at render time — a different field with a different job — so -# a deployment whose every role is pinned still has no engine-wide default, and that is refused. -# This row and the headline render DIFFERENT manifests — the headline's role names no image at all, -# this row's role does. That difference is the whole point: the two agree on the refusal while -# differing in exactly the field a reader might think substitutes for the engine default. -refuses "an engine with no image key is refused even when every role names an image" "$ENGINE_NO_IMAGE" "$WITH_IMAGE" +# THE CONTROLS COME FIRST. Each of them declares less than the external shape and is therefore +# leader-served Internal, so each is accepted with NO router. Without these rows every refusal +# below would pass for the wrong reason: a blanket rejection of SGLang multi-Pod roles would look +# exactly like the rule under test. +accepts "the control: attention width one is ACCEPTED without a router" \ + "$ENGINE_SGLANG" "$(sglang_role 1 2 "$RUST_ENV")" "" +accepts "the control: one node is ACCEPTED without a router" \ + "$ENGINE_SGLANG" "$(sglang_role 2 1 "$RUST_ENV")" "" +accepts "the control: the Rust server off is ACCEPTED without a router" \ + "$ENGINE_SGLANG" "$(sglang_role 2 2 "")" "" -# ------------------------------------------------- the no-patch shape +# The refusal: the external shape with no router names the rule and the field it is about. +refuses "the external SGLang shape with no router is REFUSED naming the rule" \ + "$ENGINE_SGLANG" "$(sglang_role 2 2 "$RUST_ENV")" "" \ + "RouterRequired" 'spec.router' -# The ordinary shape above is worded through the defaulting patch. This row differs from the -# headline by exactly one thing: the role pins resources.accelerator: "1", so the webhook has -# nothing to default and emits no patch. The image key then reaches the schema still absent, and -# the wording is the other one refuses() accepts: Required value. The headline and this row cover -# both shapes the server can answer, because the two shapes produce different wordings. -refuses "an engine with no image key is refused even when the role pins its accelerator" "$ENGINE_NO_IMAGE" "$NO_IMAGE" "$ACCELERATOR_PIN" +# The acceptances: the same external shape served through either router. +accepts "the external SGLang shape with sglang-gateway is ACCEPTED" \ + "$ENGINE_SGLANG" "$(sglang_role 2 2 "$RUST_ENV")" "sglang-gateway" +accepts "the external SGLang shape with llm-d-router is ACCEPTED" \ + "$ENGINE_SGLANG" "$(sglang_role 2 2 "$RUST_ENV")" "llm-d-router" # Results. print_rows diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-127.sh b/.agents/skills/gpustack-operator-e2e/cases/case-127.sh index 9e3531abd..903af7ded 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-127.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-127.sh @@ -85,15 +85,15 @@ # Creates no cluster-scoped object, changes no node, label or setting. set -uo pipefail -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:-}" if [ -z "$NS" ]; then echo "usage: case-127.sh " >&2 exit 2 fi -CASE_ID=127 +export CASE_ID=127 IMAGE="${E2E_MD_IMAGE:-registry.k8s.io/pause:3.10}" IT="${E2E_MD_INSTANCE_TYPE:-}" HOLD="${E2E_C127_HOLD:-240}" @@ -112,50 +112,11 @@ MD="case127-${SFX}" MD_WARM="case127-warm-${SFX}" WORK="$(mktemp -d)" -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } # shellcheck source=/dev/null -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" - -results() { - print_rows - [ "$FAILS" -eq 0 ] || { echo "[case-${CASE_ID}] ${FAILS} check(s) FAILED"; return 1; } - echo "[case-${CASE_ID}] all checks passed" - return 0 -} +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_md-lib.sh" # The first InstanceType a deployment can name: not terminating, not inactive. See case-49 for why -# the first one listed is not good enough. -usable_instance_type() { - kubectl get instancetypes.worker.gpustack.ai \ - -o jsonpath='{range .items[*]}{.metadata.name}|{.metadata.deletionTimestamp}|{.spec.inactive}{"\n"}{end}' \ - 2>/dev/null \ - | while IFS='|' read -r name deleting inactive; do - [ -n "$name" ] || continue - [ -z "$deleting" ] || continue - [ "$inactive" = true ] && continue - echo "$name" - break - done -} - -# The first usable InstanceType whose reported manufacturer is the one named: the warm-restart leg -# needs a pool whose cache directory carries a real vendor segment, not the cpu fallback. -usable_type_for() { # - kubectl get instancetypes.worker.gpustack.ai -o jsonpath='{range .items[*]}{.metadata.name}|{.metadata.deletionTimestamp}|{.spec.inactive}|{.status.detail.manufacturer}{"\n"}{end}' \ - 2>/dev/null \ - | while IFS='|' read -r name deleting inactive manufacturer; do - [ -n "$name" ] || continue - [ -z "$deleting" ] || continue - [ "$inactive" = true ] && continue - [ -n "$manufacturer" ] || continue - if [ "$manufacturer" = "$1" ]; then - echo "$name" - break - fi - done -} +# the first one listed is not good enough. The lookup is the library's: md_usable_instance_type. has_localqueue() { kubectl -n "$NS" get localqueue -o name 2>/dev/null | head -n 1 | grep -q . @@ -243,36 +204,32 @@ ${extra} YAML } +# The instrument is the library's: it reads the manifest, prints the text of the row and returns the +# verdict. The row stays here, so the table and the verdict never leave the case that owns them. admission_accepts() { # - local check="$1" out - out="$(admission_manifest | kubectl apply --dry-run=server -f - 2>&1 | tr '\n' ' ')" - case "$out" in - *"created (server dry run)"* | *"configured (server dry run)"*) - record PASS "$check" "accepted by the schema and the webhook" ;; - *) - record FAIL "$check" "refused: ${out:0:220}" ;; - esac + local check="$1" obj status=0 + obj="$(md_dryrun_accepts < <(admission_manifest))" || status=1 + if [ "$status" -eq 0 ]; then + record PASS "$check" "$obj" + else + record FAIL "$check" "$obj" + fi return 0 } admission_refuses_timeout() { # - local check="$1" out - out="$(admission_manifest | kubectl apply --dry-run=server -f - 2>&1 | tr '\n' ' ')" - if [ -z "$out" ]; then - record FAIL "$check" "the apply produced no output at all, so nothing was refused" - return 0 + local check="$1" obj status=0 + obj="$(md_dryrun_refuses "startupTimeoutSeconds" < <(admission_manifest))" || status=1 + if [ "$status" -eq 0 ]; then + record PASS "$check" "$obj" + else + record FAIL "$check" "$obj" fi - case "$out" in - *"startupTimeoutSeconds"*) - record PASS "$check" "refused naming startupTimeoutSeconds: ${out:0:180}" ;; - *) - record FAIL "$check" "not refused naming startupTimeoutSeconds (got: ${out:0:200})" ;; - esac return 0 } if [ -z "$IT" ]; then - IT="$(usable_instance_type)" + IT="$(md_usable_instance_type)" fi MOUNT="" @@ -343,7 +300,7 @@ POD="" if [ -z "$IT" ]; then record SKIP "$LEG_ROW" "no usable InstanceType in the cluster, so nothing would be admitted" elif ! has_localqueue; then - record SKIP "$LEG_ROW" "no LocalQueue in ${NS}, so the group would never be admitted; see case-93's Environment note" + record SKIP "$LEG_ROW" "no LocalQueue in ${NS}, so the group would never be admitted; see case-78's Environment note" else APPLY_OUT="$(rendered_manifest | kubectl apply -f - 2>&1 | tr '\n' ' ')" for _ in $(seq 1 90); do @@ -539,8 +496,11 @@ if [ -z "$ENGINE_IMAGE" ] || [ -z "$ENGINE_MODEL" ]; then "E2E_C127_ENGINE_IMAGE and E2E_C127_MODEL are not both set, so there is no real engine to compile anything; the warm-restart leg is skipped" else WARM_IT="" + # The first usable InstanceType whose reported manufacturer is the one named: the warm-restart leg + # needs a pool whose cache directory carries a real vendor segment, not the cpu fallback. The + # lookup is the library's: md_usable_type_for. for m in ascend nvidia amd; do - WARM_IT="$(usable_type_for "$m")" + WARM_IT="$(md_usable_type_for "$m")" [ -n "$WARM_IT" ] && break done if [ -z "$WARM_IT" ]; then diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-128.sh b/.agents/skills/gpustack-operator-e2e/cases/case-128.sh index a2644c312..76e61d657 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-128.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-128.sh @@ -66,8 +66,8 @@ # binding first. set -uo pipefail -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:-}" if [ -z "$NS" ]; then @@ -99,11 +99,8 @@ ELASTIC_TARGET=case128-target-elastic S1=case128-scaler1 S2=case128-scaler2 -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } # shellcheck source=/dev/null -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_md-lib.sh" cleanup() { kubectl -n "$NS" delete "$SCALER_RESOURCE" "$DEFAULTS_SCALER" "$S1" "$S2" \ @@ -181,60 +178,42 @@ expect() { # # THE PHRASE IS THE RULE'S OWN WORDING where the rule owns the wording, and the FIELD PATH where # the schema owns it, because the schema and the webhook word the same fault differently and a # row that pinned one word would fail on the other layer's phrasing. +# +# The instrument is the library's: it reads the manifest on stdin, prints the text of the row and +# returns the verdict. The row stays here, so the table and the verdict never leave the case that +# owns them. refuses() { # - local check="$1" phrase="$2" out - out="$(cat | kubectl apply --dry-run=server -f - 2>&1 | tr '\n' ' ')" - if [ -z "$out" ]; then - record FAIL "$check" "the apply produced no output at all, so nothing was refused" - return 0 + local check="$1" phrase="$2" obj status=0 + obj="$(md_dryrun_refuses "$phrase")" || status=1 + if [ "$status" -eq 0 ]; then + record PASS "$check" "$obj" + else + record FAIL "$check" "$obj" fi - case "$out" in - *"created (server dry run)"* | *"configured (server dry run)"*) - record FAIL "$check" "the server ACCEPTED it: ${out:0:200}" ;; - *"$phrase"*) - record PASS "$check" "refused with: ${out:0:200}" ;; - *) - record FAIL "$check" "refused, but not with the expected wording (${phrase}): ${out:0:200}" ;; - esac return 0 } # Assert that a scaler manifest is ACCEPTED, which is what makes the refusals above mean # something: a manifest wrong in some unrelated way would refuse too. accepts() { # - local check="$1" out - out="$(cat | kubectl apply --dry-run=server -f - 2>&1 | tr '\n' ' ')" - case "$out" in - *"created (server dry run)"* | *"configured (server dry run)"*) - record PASS "$check" "accepted by both the schema and the webhook" ;; - *) - record FAIL "$check" "refused: ${out:0:220}" ;; - esac + local check="$1" obj status=0 + obj="$(md_dryrun_accepts)" || status=1 + if [ "$status" -eq 0 ]; then + record PASS "$check" "$obj" + else + record FAIL "$check" "$obj" + fi return 0 } # ---------------------------------------------------------------- the usable InstanceType # The rows that read a target need one, because the webhook reads the platform the type a role -# names reports. The type must not be deleting and must not be inactive. -usable_type_for() { # - kubectl get instancetypes.worker.gpustack.ai -o jsonpath='{range .items[*]}{.metadata.name}|{.metadata.deletionTimestamp}|{.spec.inactive}|{.status.detail.manufacturer}{"\n"}{end}' \ - 2>/dev/null \ - | while IFS='|' read -r name deleting inactive manufacturer; do - [ -n "$name" ] || continue - [ -z "$deleting" ] || continue - [ "$inactive" = true ] && continue - [ -n "$manufacturer" ] || continue - if [ "$manufacturer" = "$1" ]; then - echo "$name" - break - fi - done -} - +# names reports. The type must not be deleting and must not be inactive. The lookup is the +# library's: md_usable_type_for. IT="" for m in ascend nvidia amd; do - found="$(usable_type_for "$m")" + found="$(md_usable_type_for "$m")" if [ -n "$found" ]; then IT="$found" break diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-129.sh b/.agents/skills/gpustack-operator-e2e/cases/case-129.sh index 0636da1b8..cf161599d 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-129.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-129.sh @@ -76,8 +76,8 @@ # and on exit. A target the caller named is never deleted. set -uo pipefail -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${E2E_C129_NS:-${1:-}}" if [ -z "$NS" ]; then @@ -111,11 +111,6 @@ READY_BOUND="${E2E_C129_READY_BOUND:-900}" STEP_BOUND="${E2E_C129_STEP_BOUND:-300}" RETIRE_BOUND="${E2E_C129_RETIRE_BOUND:-300}" -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } -# shellcheck source=/dev/null -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" # shellcheck source=/dev/null . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_serving-lib.sh" @@ -249,6 +244,14 @@ skip_rest() { # record SKIP "every phase answers requests and fails none" "$reason" } +# Print the rows and say the verdict. Every exit goes through here, so the case says the same +# thing wherever it stops. +finish() { + print_rows + [ "$FAILS" -eq 0 ] || { echo "[case-129] ${FAILS} check(s) FAILED"; exit 1; } + echo "[case-129] all checks passed" +} + # ---------------------------------------------------------------- the target BUILT="" @@ -319,10 +322,7 @@ fi if [ -n "$ENV_REASON" ]; then record SKIP "the target serves and the probe gets an answer" "$ENV_REASON" skip_rest "$ENV_REASON" - print_rows - [ "$FAILS" -eq 0 ] || { echo "[case-129] ${FAILS} check(s) FAILED"; exit 1; } - echo "[case-129] all checks passed" - exit 0 + finish; exit 0 fi # The target has to serve before anything is measured: a scaler that reads the counter of an @@ -344,10 +344,7 @@ if [ "$phase" != "Ready" ]; then "the target never reached Ready (its phase is ${phase:-unknown}) and it has no Pod, so nothing was admitted" fi skip_rest "the target never reached Ready (its phase is ${phase:-unknown})" - print_rows - [ "$FAILS" -eq 0 ] || { echo "[case-129] ${FAILS} check(s) FAILED"; exit 1; } - echo "[case-129] all checks passed" - exit 0 + finish; exit 0 fi # THE SERVED MODEL NAME. A request for a model the endpoint does not serve is refused with 404, so @@ -383,10 +380,7 @@ if ! kubectl -n "$NS" wait "pod/$PROBE" --for=condition=Ready --timeout=300s >/d record SKIP "the target serves and the probe gets an answer" \ "the probe Pod '${PROBE}' never became Ready on image '${PROBE_IMAGE}', so no request was sent" skip_rest "the probe Pod '${PROBE}' never became Ready on image '${PROBE_IMAGE}', so no request was sent" - print_rows - [ "$FAILS" -eq 0 ] || { echo "[case-129] ${FAILS} check(s) FAILED"; exit 1; } - echo "[case-129] all checks passed" - exit 0 + finish; exit 0 fi # THE PROBE SCRIPT. It holds each phase for its whole duration: a round of concurrent requests @@ -489,10 +483,7 @@ else record FAIL "the probe files reach the Pod" \ "the script is ${script_bytes:-0} bytes and the phase list is ${phase_bytes:-0} bytes in the Pod, so no request was sent" skip_rest "the probe files did not reach the Pod, so no request was sent" - print_rows - [ "$FAILS" -eq 0 ] || { echo "[case-129] ${FAILS} check(s) FAILED"; exit 1; } - echo "[case-129] all checks passed" - exit 0 + finish; exit 0 fi # ---------------------------------------------------------------- the scaler @@ -739,6 +730,4 @@ else fi # Results. -print_rows -[ "$FAILS" -eq 0 ] || { echo "[case-129] ${FAILS} check(s) FAILED"; exit 1; } -echo "[case-129] all checks passed" +finish diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-13.sh b/.agents/skills/gpustack-operator-e2e/cases/case-13.sh index b80fc4ef3..cc13cf0ef 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-13.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-13.sh @@ -29,8 +29,8 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: case-13.sh }" INST=gpustack-e2e-ssh-slice @@ -39,6 +39,9 @@ MEM_PCT=60 LOCAL_PORT=22022 KEYDIR="$(mktemp -d)" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_ssh-lib.sh" + command -v ssh >/dev/null 2>&1 || { echo "== CASE 13 — SKIPPED =="; echo "no ssh client on the runner"; exit 0; } # --- Skip gate: real sliced accelerator required. --- @@ -74,20 +77,14 @@ print(int(m.group(1)) * (1024 if m.group(2) == 'G' else 1) if m else 0) " 2>/dev/null) echo "[case-13] sliceable InstanceType ${IT} (card memory ${CARDMEM} = ${PHYS_MIB}MiB)" -PF_PID="" restore() { echo echo "[case-13] cleanup" - [ -n "$PF_PID" ] && kill "$PF_PID" 2>/dev/null kubectl -n default delete instance "$INST" --ignore-not-found --wait=false 2>/dev/null || true kubectl -n default delete secret "$SECRET" --ignore-not-found 2>/dev/null || true rm -rf "$KEYDIR" } -trap restore EXIT - -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } +trap 'ssh_cleanup; restore' EXIT print_and_exit() { echo @@ -107,9 +104,7 @@ print_and_exit() { } # 1. SSH key + secret, then a sliced SSH-enabled Instance on the sliceable pool. -ssh-keygen -t ed25519 -f "$KEYDIR/id" -N "" -q -kubectl -n default delete secret "$SECRET" --ignore-not-found >/dev/null 2>&1 -kubectl -n default create secret generic "$SECRET" --from-file=authorized_keys="$KEYDIR/id.pub" >/dev/null +ssh_setup_key_secret default "$SECRET" "$KEYDIR/id" echo "[case-13] creating Instance ${INST}: ${MEM_PCT}% memory slice, SSH enabled, on ${IT}" cat </dev/null apiVersion: worker.gpustack.ai/v1alpha1 @@ -175,12 +170,8 @@ fi # 5. SSH login (real sshd -> ForceCommand /chroot.sh -> nsenter into main -> setpriv login shell): # the slice is visible, the shell has no capabilities, and host mknod is denied. -kubectl -n default port-forward "pod/$POD" "${LOCAL_PORT}:22" >/dev/null 2>&1 & -PF_PID=$! -for _ in $(seq 1 20); do (exec 3<>"/dev/tcp/127.0.0.1/${LOCAL_PORT}") 2>/dev/null && { exec 3>&- 3<&-; break; }; sleep 1; done -ssh_out=$(ssh -T -p "$LOCAL_PORT" -i "$KEYDIR/id" -o IdentitiesOnly=yes \ - -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=15 \ - root@127.0.0.1 2>/dev/null <<'CMDS' +ssh_open_port_forward default "pod/$POD" 22 "$LOCAL_PORT" +ssh_out=$(ssh_exec "$KEYDIR/id" "$LOCAL_PORT" 2>/dev/null <<'CMDS' echo "CAPEFF=$(grep CapEff /proc/self/status | awk '{print $2}')" echo "CAPBND=$(grep CapBnd /proc/self/status | awk '{print $2}')" echo "SMI=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader 2>/dev/null | grep -oE '[0-9]+' | head -1)" @@ -188,7 +179,7 @@ mknod /tmp/case13blk b 8 0 2>/dev/null; echo "MKNOD_RC=$?" exit CMDS ) -kill "$PF_PID" 2>/dev/null; PF_PID="" +ssh_cleanup ssh_smi=$(printf '%s\n' "$ssh_out" | sed -n 's/^SMI=//p') ssh_capeff=$(printf '%s\n' "$ssh_out" | sed -n 's/^CAPEFF=//p') diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-130.sh b/.agents/skills/gpustack-operator-e2e/cases/case-130.sh deleted file mode 100755 index 5f0a5236f..000000000 --- a/.agents/skills/gpustack-operator-e2e/cases/case-130.sh +++ /dev/null @@ -1,241 +0,0 @@ -#!/usr/bin/env bash -# -# CASE 130 — SGLang External DP at admission: the Router is required (NON-MUTATING: server-side dry runs only) -# -# case-130.sh -# -# Goal: Read the SGLang External DP admission rule on a live API server rather than in a -# table a user cannot see. An SGLang role that declares the external shape — attention -# width at least two, a node count of at least two, and the Rust server on — is served -# through a Router, so a deployment that declares one without a Router is REFUSED -# naming the rule, and the same role WITH a Router is ACCEPTED. The controls matter -# as much as the rows: a role that declares LESS stays leader-served Internal and is -# accepted without a Router, which proves the refusal is the external-shape rule and -# not a blanket rejection of SGLang multi-Pod roles. -# Environment: Any cluster with a materialized scheduling chain (run case-1 first). No accelerator, -# no engine image and no GPU are needed: every assertion is a server-side dry run, so -# nothing is stored and no Pod is ever started. -# Inputs: REAL: the assertions are server-side dry runs of the ModelDeployment the case builds. -# MOCKED: nothing. -# Expected: - the external SGLang shape with NO router is REFUSED naming RouterRequired and the -# role; -# - the same shape WITH sglang-gateway is ACCEPTED, and with llm-d-router too; -# - the CONTROL rows — attention width one, and a node count of one, and the Rust -# server off — are each ACCEPTED without a Router, because each of them stays -# leader-served Internal. -# Cleanup: Nothing to clean up: no object is stored, no node is touched and no setting is -# changed. The case exits 0 when every check passed. -set -uo pipefail - -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" - -NS="${1:-}" -if [ -z "$NS" ]; then - echo "usage: case-130.sh " >&2 - exit 2 -fi - -MD=case130-sedp -IMAGE="${E2E_MD_IMAGE:-registry.k8s.io/pause:3.10}" -IT="${E2E_MD_INSTANCE_TYPE:-unset}" - -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } -# shellcheck source=/dev/null -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" - -command -v jq >/dev/null 2>&1 || { echo "[case-130] jq is required." >&2; exit 2; } - -# ---------------------------------------------------------------- the manifests - -# manifest builds the deployment under test. The router block is omitted when the router name is -# empty, because the row under test is the one where there IS no router, and a router block with -# an empty name would be a different object. -manifest() { # router-name roles-block - local router="$1" - cat <&1 | tr '\n' ' ')" - if [ -z "$out" ]; then - record FAIL "$check" "the apply produced no output at all, so nothing was refused" - return 0 - fi - for w in "$want" "$@"; do - case "$out" in - *"$w"*) ;; - *) record FAIL "$check" "refused with the wrong message, or accepted (missing '${w}'): ${out:0:220}"; return 0 ;; - esac - done - record PASS "$check" "refused naming: ${want}${*:+ and $*}" - return 0 -} - -# accepts asserts that the apply was ACCEPTED by both the schema and the webhook. -accepts() { - local check="$1" router="$2" roles="$3" out - out="$(manifest "$router" "$roles" | kubectl apply --dry-run=server -f - 2>&1 | tr '\n' ' ')" - case "$out" in - *"created (server dry run)"* | *"configured (server dry run)"*) - record PASS "$check" "accepted by both the schema and the webhook" ;; - *) record FAIL "$check" "refused: ${out:0:220}" ;; - esac - return 0 -} - -# ---------------------------------------------------------------- row 0: the controls - -# The controls come FIRST. Each of them declares less than the external shape and is therefore -# leader-served Internal, so each is accepted with NO router. Without these rows every refusal -# below would pass for the wrong reason: a blanket rejection of SGLang multi-Pod roles would look -# exactly like the rule under test. -accepts "the control: attention width one is ACCEPTED without a router" "" "$(attn_width_one_roles)" -accepts "the control: one node is ACCEPTED without a router" "" "$(one_node_roles)" -accepts "the control: the Rust server off is ACCEPTED without a router" "" "$(no_rust_server_roles)" - -# ---------------------------------------------------------------- row 1: the refusal - -refuses "the external SGLang shape with no router is REFUSED naming the rule" \ - "RouterRequired" "" "$(external_roles)" \ - 'spec.router' - -# ---------------------------------------------------------------- row 2: the acceptance - -accepts "the external SGLang shape with sglang-gateway is ACCEPTED" "sglang-gateway" "$(external_roles)" -accepts "the external SGLang shape with llm-d-router is ACCEPTED" "llm-d-router" "$(external_roles)" - -# Results. -print_rows -[ "$FAILS" -eq 0 ] || { echo "[case-130] ${FAILS} check(s) FAILED"; exit 1; } -echo "[case-130] all checks passed" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-14.sh b/.agents/skills/gpustack-operator-e2e/cases/case-14.sh index 75f9dda12..077b39992 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-14.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-14.sh @@ -35,8 +35,8 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: case-14.sh }" INST_A=gpustack-e2e-coexist-a @@ -113,10 +113,6 @@ restore() { } trap restore EXIT -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } - mkslice() { # mkslice [image] [command-json] cat </dev/null apiVersion: worker.gpustack.ai/v1alpha1 diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-15.sh b/.agents/skills/gpustack-operator-e2e/cases/case-15.sh index 8de8b6201..3aa93f432 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-15.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-15.sh @@ -24,8 +24,8 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: case-15.sh }" INST=gpustack-e2e-ssh-exclusive @@ -33,6 +33,9 @@ SECRET=gpustack-e2e-ssh-exclusive-key LOCAL_PORT=22015 KEYDIR="$(mktemp -d)" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_ssh-lib.sh" + command -v ssh >/dev/null 2>&1 || { echo "== CASE 15 — SKIPPED =="; echo "no ssh client on the runner"; exit 0; } # --- Skip gate: a real whole-card (exclusive) accelerator must be advertised. --- @@ -78,20 +81,14 @@ fi WHOLE_MIN=$(python3 -c "print(int(${PHYS_MIB:-0} * 0.9))" 2>/dev/null) echo "[case-15] acceleratable InstanceType ${IT} (card memory ${CARDMEM} = ${PHYS_MIB}MiB; whole-card >= ${WHOLE_MIN}MiB)" -PF_PID="" restore() { echo echo "[case-15] cleanup" - [ -n "$PF_PID" ] && kill "$PF_PID" 2>/dev/null kubectl -n default delete instance "$INST" --ignore-not-found --wait=false 2>/dev/null || true kubectl -n default delete secret "$SECRET" --ignore-not-found 2>/dev/null || true rm -rf "$KEYDIR" } -trap restore EXIT - -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } +trap 'ssh_cleanup; restore' EXIT print_and_exit() { echo @@ -110,9 +107,7 @@ print_and_exit() { } # 1. SSH key + secret, then an exclusive (no slice %) SSH-enabled Instance. -ssh-keygen -t ed25519 -f "$KEYDIR/id" -N "" -q -kubectl -n default delete secret "$SECRET" --ignore-not-found >/dev/null 2>&1 -kubectl -n default create secret generic "$SECRET" --from-file=authorized_keys="$KEYDIR/id.pub" >/dev/null +ssh_setup_key_secret default "$SECRET" "$KEYDIR/id" echo "[case-15] creating exclusive whole-card SSH Instance ${INST} on ${IT}" cat </dev/null apiVersion: worker.gpustack.ai/v1alpha1 @@ -166,19 +161,15 @@ case "$shape" in esac # 3. SSH login: the shell sees the WHOLE card (not a slice) and is capability-stripped. -kubectl -n default port-forward "pod/$POD" "${LOCAL_PORT}:22" >/dev/null 2>&1 & -PF_PID=$! -for _ in $(seq 1 20); do (exec 3<>"/dev/tcp/127.0.0.1/${LOCAL_PORT}") 2>/dev/null && { exec 3>&- 3<&-; break; }; sleep 1; done -ssh_out=$(ssh -T -p "$LOCAL_PORT" -i "$KEYDIR/id" -o IdentitiesOnly=yes \ - -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=15 \ - root@127.0.0.1 2>/dev/null <<'CMDS' +ssh_open_port_forward default "pod/$POD" 22 "$LOCAL_PORT" +ssh_out=$(ssh_exec "$KEYDIR/id" "$LOCAL_PORT" 2>/dev/null <<'CMDS' echo "CAPEFF=$(grep CapEff /proc/self/status | awk '{print $2}')" echo "SMI=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader 2>/dev/null | grep -oE '[0-9]+' | head -1)" mknod /tmp/case15blk b 8 0 2>/dev/null; echo "MKNOD_RC=$?" exit CMDS ) -kill "$PF_PID" 2>/dev/null; PF_PID="" +ssh_cleanup ssh_smi=$(printf '%s\n' "$ssh_out" | sed -n 's/^SMI=//p') ssh_capeff=$(printf '%s\n' "$ssh_out" | sed -n 's/^CAPEFF=//p') diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-16.sh b/.agents/skills/gpustack-operator-e2e/cases/case-16.sh index 9d6a813a5..2d44df91f 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-16.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-16.sh @@ -1,39 +1,104 @@ #!/usr/bin/env bash # -# CASE 16 — InstanceTypeFlavor catalog + declarative queue ownership (MUTATING, self-recovering) +# CASE 16 — InstanceTypeFlavor catalog, queue ownership, and CPU-manufacturer awareness (MUTATING, self-recovering) # # case-16.sh # -# Goal: On the general pool, prove the queue-ownership split (InstanceTypeReconciler owns the +# Goal: On the general (CPU) pool, prove the queue-ownership split (InstanceTypeReconciler owns the # CQ's existence + schedule labels + teardown; NodeQueueReconciler owns its quota / -# StopPolicy; NodeFlavorReconciler authors the derived type) — +# StopPolicy; NodeFlavorReconciler authors the derived type) and the shape of the +# aggregated catalog — # A: the aggregated, list-only InstanceTypeFlavor resource projects the fleet's pools, the # generic (CPU-only) pool appearing with acceleratable=false; # B: deleting a backing ClusterQueue while its InstanceType still lives self-heals — the # reconciler recreates it (a NEW uid) and the chain never stays down; # C: deleting an admin InstanceType holds its finalizer until the operator has deleted the -# backing ClusterQueue and Kueue has actually removed it (delete-then-wait), then releases. -# Environment: Any cluster with a materialized general pool. No GPU. Uses a throwaway admin type for the -# teardown probe so the derived chain is untouched. +# backing ClusterQueue and Kueue has actually removed it (delete-then-wait), then releases; +# D: the CPU ResourceFlavor is named gpustack--${gKey}-${os}-${arch}-${count}c (the +# double-dash finest grain) and always carries a cpuDetail note recording the raw CPU +# manufacturer/product/family; +# E: instance-type-aware-cpu-manufacturer drives the collapse<->split of the aggregated view +# while the ResourceFlavor stays the finest, setting-independent grain — awareness OFF +# collapses every CPU into one CPU-agnostic row named gpustack--generic (acceleratable=false, +# generalGroup=generic); awareness ON replaces it with a real-CPU-keyed row named +# gpustack--${gKey} (generalGroup=${gKey}, not "generic"), and the generic row is gone. +# Environment: Any cluster with a materialized general (CPU) pool. No GPU. The ResourceFlavor and the +# InstanceTypeFlavor catalog exist whenever a node is managed, so this runs anywhere; the +# catalog recomputes live at list time, so the toggle needs no redeploy. A throwaway admin +# type carries the teardown probe so the derived chain is untouched. # Inputs: All real, nothing mocked — -# - reads the InstanceTypeFlavor catalog; +# - reads the InstanceTypeFlavor catalog and the CPU ResourceFlavor; # - deletes the derived general pool's backing ClusterQueue, then lets it self-heal; -# - creates + deletes a throwaway admin InstanceType e2e-case16-teardown (generalGroup e2e16td). +# - creates + deletes a throwaway admin InstanceType e2e-case16-teardown (generalGroup e2e16td); +# - patches the gpustack-settings Secret's instance-type-aware-cpu-manufacturer key true/false +# (the editable-setting store the aggregated apiserver reads, with a <=30s value cache). # Expected: - A — >=1 InstanceTypeFlavor row, and a generic (acceleratable=false) one; # - B — the ClusterQueue is recreated with a NEW uid while the InstanceType survives; # - C — the throwaway's backing ClusterQueue is created, then both it and the InstanceType -# are removed on delete, and no sample sees the InstanceType gone while its queue remains. -# Cleanup: Trap force-strips the throwaway's finalizer if stuck, deletes the throwaway type + its -# ClusterQueue, and waits for the derived general InstanceType to return to Active. +# are removed on delete, and no sample sees the InstanceType gone while its queue remains; +# - D — a gpustack--...-Nc CPU flavor whose note.gpustack.ai/cpuDetail parses as JSON; +# - E — awareness off shows exactly the gpustack--generic (generalGroup=generic) row; +# awareness on shows a gpustack--${gKey} (generalGroup!=generic) row and no generic row. +# Cleanup: Trap restores the awareness setting to its original value, waits for the catalog to +# re-collapse so the reconciler's setting cache has flipped back, deletes the derived +# InstanceTypes (+ their ClusterQueues) created while awareness was on, strips and deletes +# the throwaway type and its ClusterQueue, and waits for the derived general InstanceType +# to return to Active. Idempotent, runs on pass AND fail. set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: case-16.sh }" +# shellcheck disable=SC2034 # Read by the sourced library, which shellcheck does not follow. +SYSTEM_NS="$NS" # the settings Secret lives in the namespace this case was given +AWARE_KEY=instance-type-aware-cpu-manufacturer +DERIVED_LABEL=schedule.gpustack.ai/derived-from-node +PROBE=e2e-case16-teardown # throwaway admin InstanceType for the teardown probe + +# The Settings store and the helpers that read and write it. + +# --- Helpers (defined before the trap so cleanup can use them). --- + +# generic_row prints the name of the collapsed, CPU-agnostic catalog row (acceleratable=false, +# generalGroup=="generic"), else empty. +generic_row() { + kubectl get instancetypeflavors.worker.gpustack.ai -o json 2>/dev/null | python3 -c " +import json,sys +try: items=json.load(sys.stdin).get('items',[]) +except Exception: items=[] +for it in items: + s=it.get('spec',{}) + if not s.get('acceleratable') and s.get('generalGroup')=='generic': + print(it['metadata']['name']); break +" 2>/dev/null +} + +# split_row prints 'name|generalGroup' of a per-CPU catalog row (acceleratable=false, generalGroup a +# real CPU key, not the "generic" sentinel), else empty. +split_row() { + kubectl get instancetypeflavors.worker.gpustack.ai -o json 2>/dev/null | python3 -c " +import json,sys +try: items=json.load(sys.stdin).get('items',[]) +except Exception: items=[] +for it in items: + s=it.get('spec',{}); gg=s.get('generalGroup','') + if not s.get('acceleratable') and gg and gg!='generic': + print(it['metadata']['name']+'|'+gg); break +" 2>/dev/null +} + +# derived_its prints the names of the operator-authored (derived-from-node) InstanceTypes. +derived_its() { + kubectl get instancetypes.worker.gpustack.ai -l "${DERIVED_LABEL}=true" \ + -o jsonpath='{.items[*].metadata.name}' 2>/dev/null +} + +# wait_generic waits for the collapsed generic row to appear (rides out the <=30s setting cache). +wait_generic() { for _ in $(seq 1 24); do [ -n "$(generic_row)" ] && return 0; sleep 3; done; return 1; } # The derived general InstanceType == its backing ClusterQueue name (one isolated CQ per pool). IT=$(kubectl get instancetypes.worker.gpustack.ai \ @@ -41,12 +106,38 @@ IT=$(kubectl get instancetypes.worker.gpustack.ai \ [ -n "$IT" ] || { echo "no general InstanceType found — run case-1 first to materialize the chain"; exit 1; } echo "general InstanceType / ClusterQueue: ${IT}" -PROBE=e2e-case16-teardown # throwaway admin InstanceType for the teardown probe +# The CPU ResourceFlavor is keyed by the node's real CPU (gpustack--${gKey}-...-${count}c, plus a +# -fnv64-<16 hex> topology profile on a TAS cluster), +# independent of any collapsed InstanceType name — the finest, setting-independent grain. +CPURF=$(kubectl get resourceflavors.kueue.x-k8s.io -o name 2>/dev/null | grep -E '/gpustack--.*-[0-9]+c(-fnv64-[0-9a-f]{16})?$' | head -1) +[ -n "$CPURF" ] || { echo "no CPU ResourceFlavor found — run case-1 first to materialize the chain"; exit 1; } + +# Capture the original setting and the derived types present now, so cleanup puts the setting back +# exactly as it found it and removes only what the aware window created. +settings_save "$AWARE_KEY" +orig="$(setting_get "$AWARE_KEY")" +before_its=" $(derived_its) " cleanup() { echo - echo "[case-16] cleanup: removing the throwaway type and waiting for the derived chain to recover" - # Strip our finalizer only if it is stuck Terminating, then delete both objects best-effort. + echo "[case-16] cleanup: restoring ${AWARE_KEY}=${orig:-}, removing the throwaway type and waiting for the derived chain to recover" + settings_restore + # Let the setting cache flip back (the catalog re-collapsing is the signal both the aggregated read + # and the reconciler read have flipped) before removing the aware-named derived types, so the + # reconciler will not recreate one after we delete it. + [ "$orig" = "true" ] || wait_generic || true + for it in $(derived_its); do + case "$before_its" in + *" $it "*) : ;; + *) + echo "[case-16] removing derived type created during the aware window: ${it}" + kubectl patch instancetype "$it" --type=merge -p '{"metadata":{"finalizers":[]}}' >/dev/null 2>&1 || true + kubectl delete instancetype "$it" --wait=false >/dev/null 2>&1 || true + kubectl delete clusterqueue "$it" --wait=false >/dev/null 2>&1 || true + ;; + esac + done + # Strip the throwaway's finalizer only if it is stuck Terminating, then delete both objects best-effort. kubectl get instancetype "$PROBE" >/dev/null 2>&1 && \ kubectl patch instancetype "$PROBE" --type=merge -p '{"metadata":{"finalizers":[]}}' >/dev/null 2>&1 || true kubectl delete instancetype "$PROBE" --wait=false >/dev/null 2>&1 || true @@ -58,20 +149,20 @@ cleanup() { } trap cleanup EXIT -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } - # --- A. The aggregated InstanceTypeFlavor catalog lists the pools; the generic one is acceleratable=false. --- n_flavors=$(kubectl get instancetypeflavors.worker.gpustack.ai --no-headers 2>/dev/null | grep -c . || true) -[ "${n_flavors:-0}" -ge 1 ] \ - && record PASS "InstanceTypeFlavor lists the fleet's pools" "${n_flavors} row(s)" \ - || record FAIL "InstanceTypeFlavor lists the fleet's pools" "no rows — the aggregated catalog must project the ResourceFlavors" +if [ "${n_flavors:-0}" -ge 1 ]; then + record PASS "InstanceTypeFlavor lists the fleet's pools" "${n_flavors} row(s)" +else + record FAIL "InstanceTypeFlavor lists the fleet's pools" "no rows — the aggregated catalog must project the ResourceFlavors" +fi generic=$(kubectl get instancetypeflavors.worker.gpustack.ai \ -o jsonpath='{.items[?(@.spec.acceleratable==false)].spec.generalGroup}' 2>/dev/null | tr ' ' '\n' | grep -m1 .) -[ -n "$generic" ] \ - && record PASS "generic pool surfaces as acceleratable=false" "generalGroup=${generic}" \ - || record FAIL "generic pool surfaces as acceleratable=false" "no acceleratable=false InstanceTypeFlavor row" +if [ -n "$generic" ]; then + record PASS "generic pool surfaces as acceleratable=false" "generalGroup=${generic}" +else + record FAIL "generic pool surfaces as acceleratable=false" "no acceleratable=false InstanceTypeFlavor row" +fi # --- B. Accidental delete of the backing ClusterQueue self-heals (recreated with a new uid). --- uid_before=$(kubectl get clusterqueue "$IT" -o jsonpath='{.metadata.uid}' 2>/dev/null) @@ -90,9 +181,11 @@ for _ in $(seq 1 10); do [ -n "$(kubectl get instancetype "$IT" -o name 2>/dev/null)" ] && { it_alive=1; break; } sleep 2 done -{ [ -n "$recreated" ] && [ -n "$it_alive" ]; } \ - && record PASS "accidental CQ delete self-heals" "${IT} recreated (uid ${uid_now}), InstanceType survived" \ - || record FAIL "accidental CQ delete self-heals" "recreated='${recreated:-no}' it_alive='${it_alive:-no}' — a live InstanceType must recreate its deleted queue" +if [ -n "$recreated" ] && [ -n "$it_alive" ]; then + record PASS "accidental CQ delete self-heals" "${IT} recreated (uid ${uid_now}), InstanceType survived" +else + record FAIL "accidental CQ delete self-heals" "recreated='${recreated:-no}' it_alive='${it_alive:-no}' — a live InstanceType must recreate its deleted queue" +fi # Wait for the recreated queue to refill + report Active before moving on. for _ in $(seq 1 40); do @@ -122,9 +215,11 @@ for _ in $(seq 1 30); do kubectl get clusterqueue "$PROBE" >/dev/null 2>&1 && { cq_made=1; break; } sleep 2 done -[ -n "$cq_made" ] \ - && record PASS "InstanceType creates its backing ClusterQueue" "${PROBE} queue present" \ - || record FAIL "InstanceType creates its backing ClusterQueue" "${PROBE} queue never appeared" +if [ -n "$cq_made" ]; then + record PASS "InstanceType creates its backing ClusterQueue" "${PROBE} queue present" +else + record FAIL "InstanceType creates its backing ClusterQueue" "${PROBE} queue never appeared" +fi echo "[case-16] deleting InstanceType ${PROBE}; the finalizer must hold until its queue is gone" kubectl delete instancetype "$PROBE" --wait=false >/dev/null 2>&1 || true @@ -148,8 +243,58 @@ else record FAIL "delete-then-wait teardown" "InstanceType='${it_left:-gone}' ClusterQueue='${cq_left:-gone}' — teardown must delete the queue then release the finalizer" fi +# --- D. The ResourceFlavor is the finest, double-dash grain and always records its raw CPU detail. --- +rfname=${CPURF#*/} +if echo "$rfname" | grep -qE '^gpustack--.+-[0-9]+c(-fnv64-[0-9a-f]{16})?$'; then + record PASS "CPU ResourceFlavor is the finest, double-dash grain" "${rfname}" +else + record FAIL "CPU ResourceFlavor is the finest, double-dash grain" "${rfname} — expected gpustack--\${gKey}-\${os}-\${arch}-\${count}c[-fnv64-]" +fi + +cpudetail=$(kubectl get "$CPURF" -o json 2>/dev/null | python3 -c " +import json,sys +try: a=json.load(sys.stdin).get('metadata',{}).get('annotations',{}) or {} +except Exception: a={} +v=a.get('note.gpustack.ai/cpuDetail','') +if not v: print('MISSING'); raise SystemExit +try: d=json.loads(v) +except Exception: print('BADJSON'); raise SystemExit +print('OK|manufacturer=%s product=%s'%(d.get('manufacturer','') or '', d.get('product','') or '')) +" 2>/dev/null) +case "$cpudetail" in + OK*) record PASS "CPU ResourceFlavor carries a cpuDetail note" "${cpudetail#OK|}" ;; + *) record FAIL "CPU ResourceFlavor carries a cpuDetail note" "${cpudetail:-} — a CPU flavor must always record its raw CPU detail" ;; +esac + +# --- E. Awareness OFF collapses the CPU pool to one CPU-agnostic generic row. --- +echo "[case-16] awareness OFF: expecting the catalog to collapse the CPU pool to gpustack--generic" +setting_set "$AWARE_KEY" false +gname="" +wait_generic && gname=$(generic_row) +if [ "$gname" = "gpustack--generic" ]; then + record PASS "awareness off collapses to one generic row" "${gname} (acceleratable=false, generalGroup=generic)" +else + record FAIL "awareness off collapses to one generic row" "generic row='${gname:-}' — expected gpustack--generic" +fi + +# --- F. Awareness ON splits the CPU pool into a real-CPU-keyed row (generic row gone). --- +echo "[case-16] awareness ON: expecting the generic row to split into a real-CPU-keyed row" +setting_set "$AWARE_KEY" true +split="" +for _ in $(seq 1 24); do + split=$(split_row) + [ -n "$split" ] && [ -z "$(generic_row)" ] && break + split="" + sleep 3 +done +if [ -n "$split" ]; then + record PASS "awareness on splits the CPU pool by manufacturer" "${split%%|*} (generalGroup=${split#*|}, generic row gone)" +else + record FAIL "awareness on splits the CPU pool by manufacturer" "no real-CPU row (or the generic row lingered) — the catalog must recompute per the setting" +fi + echo -echo "== CASE 16 — InstanceTypeFlavor catalog + declarative queue ownership ==" +echo "== CASE 16 — InstanceTypeFlavor catalog, queue ownership, and CPU-manufacturer awareness ==" { echo "STATUS|CHECK|OBJECT" printf '%s\n' "${ROWS[@]}" @@ -158,8 +303,10 @@ echo "== CASE 16 — InstanceTypeFlavor catalog + declarative queue ownership == if [ "$FAILS" -ne 0 ]; then echo echo "FAILED ${FAILS} check(s). The InstanceTypeReconciler owns the backing queue's existence" - echo "(recreate on accidental delete; delete-then-wait teardown); the InstanceTypeFlavor catalog" - echo "projects the pools. See instance_type.go / node_queue.go / extensionapis." + echo "(recreate on accidental delete; delete-then-wait teardown) and the InstanceTypeFlavor catalog" + echo "projects the pools. The ResourceFlavor is the finest, setting-independent grain (always" + echo "carrying cpuDetail); instance-type-aware-cpu-manufacturer collapses (off) or splits (on) only the" + echo "aggregated InstanceTypeFlavor/InstanceType/ClusterQueue view." exit 1 fi echo "CASE 16 PASS" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-18.sh b/.agents/skills/gpustack-operator-e2e/cases/case-18.sh deleted file mode 100755 index 24f440e47..000000000 --- a/.agents/skills/gpustack-operator-e2e/cases/case-18.sh +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env bash -# -# CASE 18 — CPU-manufacturer awareness reshapes the catalog (MUTATING, self-recovering: flips an editable setting) -# -# case-18.sh -# -# Goal: On the general (CPU) pool, prove instance-type-aware-cpu-manufacturer drives the -# collapse<->split of the aggregated view while the ResourceFlavor stays the finest, -# setting-independent grain — -# A: the CPU ResourceFlavor is named gpustack--${gKey}-${os}-${arch}-${count}c (the -# double-dash finest grain) and always carries a cpuDetail note recording the raw CPU -# manufacturer/product/family; -# B: with awareness OFF the InstanceTypeFlavor catalog collapses every CPU into one -# CPU-agnostic row named gpustack--generic (acceleratable=false, generalGroup=generic); -# C: flipping awareness ON splits it — the generic row is replaced by a real-CPU-keyed row -# named gpustack--${gKey} (generalGroup=${gKey}, not "generic"), and the generic row is gone. -# Environment: Any cluster with a materialized general (CPU) pool. No GPU. The ResourceFlavor and the -# InstanceTypeFlavor catalog exist whenever a node is managed, so this runs anywhere; the -# catalog recomputes live at list time, so the toggle needs no redeploy. -# Inputs: Nothing mocked — patches the gpustack-settings Secret's -# instance-type-aware-cpu-manufacturer key true/false (the editable-setting store the -# aggregated apiserver reads, with a <=30s value cache) and reads the live catalog. -# Expected: - A — a gpustack--...-Nc CPU flavor whose note.gpustack.ai/cpuDetail parses as JSON; -# - B — awareness off shows exactly the gpustack--generic (generalGroup=generic) row; -# - C — awareness on shows a gpustack--${gKey} (generalGroup!=generic) row and no generic row. -# Cleanup: Trap restores the setting to its original value, waits for the catalog to re-collapse so -# the reconciler's setting cache has flipped back, then deletes any operator-derived -# InstanceType (+ its ClusterQueue) created while awareness was on, so the cluster is left -# exactly as found. -set -uo pipefail - -# Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail -# on transport alone, and a check that takes such a failure for an answer reports a verdict -# about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" - -NS="${1:?usage: case-18.sh }" -AWARE_KEY=instance-type-aware-cpu-manufacturer -DERIVED_LABEL=schedule.gpustack.ai/derived-from-node - -# --- Helpers (defined before the trap so cleanup can use them). --- - -# generic_row prints the name of the collapsed, CPU-agnostic catalog row (acceleratable=false, -# generalGroup=="generic"), else empty. -generic_row() { - kubectl get instancetypeflavors.worker.gpustack.ai -o json 2>/dev/null | python3 -c " -import json,sys -try: items=json.load(sys.stdin).get('items',[]) -except Exception: items=[] -for it in items: - s=it.get('spec',{}) - if not s.get('acceleratable') and s.get('generalGroup')=='generic': - print(it['metadata']['name']); break -" 2>/dev/null -} - -# split_row prints 'name|generalGroup' of a per-CPU catalog row (acceleratable=false, generalGroup a -# real CPU key, not the "generic" sentinel), else empty. -split_row() { - kubectl get instancetypeflavors.worker.gpustack.ai -o json 2>/dev/null | python3 -c " -import json,sys -try: items=json.load(sys.stdin).get('items',[]) -except Exception: items=[] -for it in items: - s=it.get('spec',{}); gg=s.get('generalGroup','') - if not s.get('acceleratable') and gg and gg!='generic': - print(it['metadata']['name']+'|'+gg); break -" 2>/dev/null -} - -# derived_its prints the names of the operator-authored (derived-from-node) InstanceTypes. -derived_its() { - kubectl get instancetypes.worker.gpustack.ai -l "${DERIVED_LABEL}=true" \ - -o jsonpath='{.items[*].metadata.name}' 2>/dev/null -} - -# set_aware patches the delegated settings Secret's awareness key to $1 (true|false). -set_aware() { - local b64; b64=$(printf '%s' "$1" | base64 | tr -d '\n') - kubectl -n "$NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"${AWARE_KEY}\":\"${b64}\"}}" >/dev/null 2>&1 -} - -# wait_generic waits for the collapsed generic row to appear (rides out the <=30s setting cache). -wait_generic() { for _ in $(seq 1 24); do [ -n "$(generic_row)" ] && return 0; sleep 3; done; return 1; } - -# The CPU ResourceFlavor is keyed by the node's real CPU (gpustack--${gKey}-...-${count}c, plus a -# -fnv64-<16 hex> topology profile on a TAS cluster), -# independent of any collapsed InstanceType name — the finest, setting-independent grain. -CPURF=$(kubectl get resourceflavors.kueue.x-k8s.io -o name 2>/dev/null | grep -E '/gpustack--.*-[0-9]+c(-fnv64-[0-9a-f]{16})?$' | head -1) -[ -n "$CPURF" ] || { echo "no CPU ResourceFlavor found — run case-1 first to materialize the chain"; exit 1; } - -# Capture the original setting (absent/other → the documented "false" default) and the derived types -# present now, so cleanup restores the setting and removes only what the aware window created. -orig=$(kubectl -n "$NS" get secret gpustack-settings -o jsonpath="{.data.${AWARE_KEY}}" 2>/dev/null | base64 -d 2>/dev/null) -[ "$orig" = "true" ] || orig=false -before_its=" $(derived_its) " - -cleanup() { - echo - echo "[case-18] cleanup: restoring ${AWARE_KEY}=${orig}" - set_aware "$orig" - # Let the setting cache flip back (the catalog re-collapsing is the signal both the aggregated read - # and the reconciler read have flipped) before removing the aware-named derived types, so the - # reconciler will not recreate one after we delete it. - [ "$orig" = "true" ] || wait_generic || true - for it in $(derived_its); do - case "$before_its" in - *" $it "*) : ;; - *) - echo "[case-18] removing derived type created during the aware window: ${it}" - kubectl patch instancetype "$it" --type=merge -p '{"metadata":{"finalizers":[]}}' >/dev/null 2>&1 || true - kubectl delete instancetype "$it" --wait=false >/dev/null 2>&1 || true - kubectl delete clusterqueue "$it" --wait=false >/dev/null 2>&1 || true - ;; - esac - done -} -trap cleanup EXIT - -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } - -# --- A. The ResourceFlavor is the finest, double-dash grain and always records its raw CPU detail. --- -rfname=${CPURF#*/} -echo "$rfname" | grep -qE '^gpustack--.+-[0-9]+c(-fnv64-[0-9a-f]{16})?$' \ - && record PASS "CPU ResourceFlavor is the finest, double-dash grain" "${rfname}" \ - || record FAIL "CPU ResourceFlavor is the finest, double-dash grain" "${rfname} — expected gpustack--\${gKey}-\${os}-\${arch}-\${count}c[-fnv64-]" - -cpudetail=$(kubectl get "$CPURF" -o json 2>/dev/null | python3 -c " -import json,sys -try: a=json.load(sys.stdin).get('metadata',{}).get('annotations',{}) or {} -except Exception: a={} -v=a.get('note.gpustack.ai/cpuDetail','') -if not v: print('MISSING'); raise SystemExit -try: d=json.loads(v) -except Exception: print('BADJSON'); raise SystemExit -print('OK|manufacturer=%s product=%s'%(d.get('manufacturer','') or '', d.get('product','') or '')) -" 2>/dev/null) -case "$cpudetail" in - OK*) record PASS "CPU ResourceFlavor carries a cpuDetail note" "${cpudetail#OK|}" ;; - *) record FAIL "CPU ResourceFlavor carries a cpuDetail note" "${cpudetail:-} — a CPU flavor must always record its raw CPU detail" ;; -esac - -# --- B. Awareness OFF collapses the CPU pool to one CPU-agnostic generic row. --- -echo "[case-18] awareness OFF: expecting the catalog to collapse the CPU pool to gpustack--generic" -set_aware false -gname="" -wait_generic && gname=$(generic_row) -[ "$gname" = "gpustack--generic" ] \ - && record PASS "awareness off collapses to one generic row" "${gname} (acceleratable=false, generalGroup=generic)" \ - || record FAIL "awareness off collapses to one generic row" "generic row='${gname:-}' — expected gpustack--generic" - -# --- C. Awareness ON splits the CPU pool into a real-CPU-keyed row (generic row gone). --- -echo "[case-18] awareness ON: expecting the generic row to split into a real-CPU-keyed row" -set_aware true -split="" -for _ in $(seq 1 24); do - split=$(split_row) - [ -n "$split" ] && [ -z "$(generic_row)" ] && break - split="" - sleep 3 -done -if [ -n "$split" ]; then - record PASS "awareness on splits the CPU pool by manufacturer" "${split%%|*} (generalGroup=${split#*|}, generic row gone)" -else - record FAIL "awareness on splits the CPU pool by manufacturer" "no real-CPU row (or the generic row lingered) — the catalog must recompute per the setting" -fi - -echo -echo "== CASE 18 — CPU-manufacturer awareness reshapes the catalog ==" -{ - echo "STATUS|CHECK|OBJECT" - printf '%s\n' "${ROWS[@]}" -} | column -t -s '|' - -if [ "$FAILS" -ne 0 ]; then - echo - echo "FAILED ${FAILS} check(s). The ResourceFlavor is the finest, setting-independent grain (always" - echo "carrying cpuDetail); instance-type-aware-cpu-manufacturer collapses (off) or splits (on) only the" - echo "aggregated InstanceTypeFlavor/InstanceType/ClusterQueue view. See pkg/nodefeature/helper.go and" - echo "pkg/worker/extensionapis/worker/instance_type_flavor.go." - exit 1 -fi -echo "CASE 18 PASS" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-19.sh b/.agents/skills/gpustack-operator-e2e/cases/case-19.sh index 5aa623768..1b865f2c4 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-19.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-19.sh @@ -31,18 +31,20 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: case-19.sh }" +# shellcheck disable=SC2034 # Read by the sourced library, which shellcheck does not follow. +SYSTEM_NS="$NS" # the settings Secret lives in the namespace this case was given AWARE_KEY=instance-type-aware-cpu-manufacturer DERIVED_LABEL=schedule.gpustack.ai/derived-from-node INST=gpustack-e2e-case19 MULTI_INST=${INST}-two CREATED=() -set_aware() { local b64; b64=$(printf '%s' "$1" | base64 | tr -d '\n') - kubectl -n "$NS" patch secret gpustack-settings --type=merge -p "{\"data\":{\"${AWARE_KEY}\":\"${b64}\"}}" >/dev/null 2>&1; } +# The Settings store and the helpers that read and write it. + derived_its() { kubectl get instancetypes.worker.gpustack.ai -l "${DERIVED_LABEL}=true" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null; } note() { kubectl get resourceflavor "$1" -o jsonpath="{.metadata.annotations.note\.gpustack\.ai/$2}" 2>/dev/null; } bounce_worker() { kubectl -n "$NS" rollout restart deploy/gpustack-operator-worker >/dev/null 2>&1; @@ -67,8 +69,11 @@ ARCH=$(kubectl get resourceflavor "$ARF" -o jsonpath='{.metadata.labels.kubernet AWARE_IT="gpustack--${GKEY}--${AKEY}-${OS}-${ARCH}" echo "[case-19] accelerated flavor ${ARF} → aware type ${AWARE_IT} (product=${PRODUCT} memory=${MEMORY} cores=${CORES})" -# Capture the exact setting value, including an absent key, before any mutation. -orig=$(kubectl -n "$NS" get secret gpustack-settings -o json | jq -c --arg key "$AWARE_KEY" '.data[$key]') || exit 1 +# Capture the setting as the store carries it before any mutation, so cleanup puts back exactly what +# was there; an absent key reads as an empty value. +kubectl -n "$NS" get secret gpustack-settings >/dev/null 2>&1 || { echo "[case-19] the gpustack-settings Secret is not in ${NS}"; exit 1; } +settings_save "$AWARE_KEY" +orig="$(setting_get "$AWARE_KEY")" before_json=$(kubectl get instancetypes.worker.gpustack.ai -o json) || exit 1 before_type=$(printf '%s' "$before_json" | jq -c --arg name "$AWARE_IT" '.items[] | select(.metadata.name == $name)') for name in "$INST" "$MULTI_INST"; do @@ -102,15 +107,13 @@ cleanup() { [ "$(printf '%s' "$current" | jq -cS '.spec | {unitResources,localStorage}')" = \ "$(printf '%s' "$before_type" | jq -cS '.spec | {unitResources,localStorage}')" ] || failed=1 fi - kubectl -n "$NS" patch secret gpustack-settings --type=merge \ - -p "$(jq -nc --arg key "$AWARE_KEY" --argjson value "$orig" '{data:{($key):$value}}')" >/dev/null || failed=1 + settings_restore bounce_worker || failed=1 - current=$(kubectl -n "$NS" get secret gpustack-settings -o json | jq -c --arg key "$AWARE_KEY" '.data[$key]') || failed=1 + current="$(setting_get "$AWARE_KEY")" # Startup initializes absent keys. Restore absence after the final restart as well. - if [ "$orig" = null ] && [ "$current" != null ]; then - kubectl -n "$NS" patch secret gpustack-settings --type=merge \ - -p "$(jq -nc --arg key "$AWARE_KEY" '{data:{($key):null}}')" >/dev/null || failed=1 - current=$(kubectl -n "$NS" get secret gpustack-settings -o json | jq -c --arg key "$AWARE_KEY" '.data[$key]') || failed=1 + if [ -z "$orig" ] && [ -n "$current" ]; then + setting_unset "$AWARE_KEY" + current="$(setting_get "$AWARE_KEY")" fi [ "$current" = "$orig" ] || failed=1 current=$(derived_its) || failed=1 @@ -124,13 +127,9 @@ cleanup() { } trap 'cleanup || exit 1' EXIT -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } - # 1. Enable awareness and force the aware re-derive (setting cache + create-only authoring). echo "[case-19] enabling awareness and restarting the worker to force the aware re-derive" -set_aware true +setting_set "$AWARE_KEY" true bounce_worker active="" for _ in $(seq 1 40); do diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-2.sh b/.agents/skills/gpustack-operator-e2e/cases/case-2.sh index e96dd6e80..e656ad1cd 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-2.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-2.sh @@ -1,63 +1,92 @@ #!/usr/bin/env bash # -# CASE 2 — Running Instance admits, then drain stops it (not recreate) (MUTATING, self-recovering) +# CASE 2 — Portless Instance admits, writes status, skips Service; the drain stops it, the managed toggle scopes onboarding (MUTATING, self-recovering) # # case-2.sh # -# Goal: A running general Instance's Workload admits on the CPU-only queue despite requesting -# resources the queue does not cover, and when its pool is drained the Instance is -# STOPPED (spec.stop=true) — not recreated with a stuck Pending Pod. +# Goal: One portless CPU Instance carries three contracts of the general (CPU) pool: +# - Kueue admits its Workload on the cpu-only queue even though the Pod asks for +# resources the queue does not cover, and the Instance reaches Ready; +# - a portless Instance writes its status and creates no Service (a Service without +# ports is refused by the API server, which failed every reconcile before the +# status block, so the Pod ran while the Instance status stayed empty); +# - draining the pool STOPS the running Instance (spec.stop=true) instead of +# recreating it with a stuck Pending Pod, keeps the pool's InstanceType alive with +# its backing ClusterQueue emptied and held, and restoring the nodes refills the +# queue and reactivates the type. # Environment: A real cluster with a materialized general pool (the fake client cannot reproduce -# Kueue admission + eviction). No GPU. Targets the general (CPU) pool fed by every -# managed node, so it behaves the same on a 1-node or an N-node cluster. +# Kueue admission + eviction, nor the API server's Service validation). No GPU. Targets +# the general (CPU) pool fed by every managed node, so it behaves the same on a 1-node +# or an N-node cluster, and needs a CONTINUOUSLY RUNNING operator (it must NOT be +# restarted between the toggle and the assertions). # Inputs: All real, nothing mocked — # - reads the derived general InstanceType's unit spec, which the Instance webhook needs; -# - a running Instance gpustack-e2e-instance (alpine sleep + ephemeral volume) in ns default; +# - a portless Instance gpustack-e2e-instance (alpine sleep + ephemeral volume, no +# spec.ports) in ns default; # - drains the pool by toggling gpustack.ai/managed=false on every -gpustack-worker -# NodeFeature (a general pool's capacity is Node CPU count, not a bumpable label). +# NodeFeature (a general pool's capacity is Node CPU count, not a bumpable label), +# then toggles the same labels back to true. # Expected: - the Instance Pod's own Workload reaches Admitted=True on the cpu-only queue # (quotaCheckStrategy checks only the covered cpu dimension and ignores the uncovered # memory/ephemeral-storage); # - the Instance is running (phase Ready, spec.stop unset) before the drain; -# - the drain deletes the pool's general ResourceFlavor; -# - the Instance flips to spec.stop=true (STOPPED, not recreated). -# The worker log line naming the stop branch is printed for diagnosis, not asserted. +# - the portless Instance writes its status and no Service is created for it; +# - the drain deletes the pool's general ResourceFlavor and the Instance flips to +# spec.stop=true (STOPPED, not recreated); +# - the InstanceType survives the flavor loss with its queue emptied and held, and it +# reactivates when the nodes come back. +# The worker log line naming the stop branch and the count of "spec.ports: Required +# value" refusals are printed for diagnosis, not asserted. # Cleanup: Trap restores gpustack.ai/managed=true on all nodes, deletes the test Instance, and # waits for the InstanceType to return to Active WITH a non-zero CPU capacity so a # following case finds a healthy chain rather than a pool whose flavor is still being -# rebuilt. +# rebuilt. Idempotent, runs on pass AND fail. set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: case-2.sh }" +# The one Instance every scope below reads: it is portless (no spec.ports), so "no Service" is a +# meaningful expectation, and it is the fixture the admission and stop checks run against. +INST=gpustack-e2e-instance # Target the general (CPU) pool: it is non-accelerated (no GPU needed) and every managed node feeds # it, so this behaves the same on a 1-node local cluster and an N-node real one. IT=$(kubectl get instancetypes.worker.gpustack.ai \ -o jsonpath='{.items[?(@.spec.acceleratable==false)].metadata.name}' 2>/dev/null | tr ' ' '\n' | grep -m1 'gpustack-') [ -n "$IT" ] || { echo "no general InstanceType found — run case-1 first to materialize the chain"; exit 1; } +# One of the pool's CPU ResourceFlavors (named gpustack--${gKey}-${os}-${arch}-${count}c, plus a +# -fnv64-<16 hex> topology profile on a TAS cluster — keyed by +# the node's real CPU, so its name is independent of the collapsed InstanceType name), kept as a +# full type/name path so `kubectl get "$RF"` works — watched so we see it deleted on de-manage. +RF=$(kubectl get resourceflavors.kueue.x-k8s.io -o name 2>/dev/null | grep -E '/gpustack--.*-[0-9]+c(-fnv64-[0-9a-f]{16})?$' | head -1) # Every -gpustack-worker NodeFeature: draining ALL of them is what removes a pool that spans # more than one node (de-managing a single node leaves the others' flavors behind). WORKER_NFS=$(kubectl -n "$NS" get nodefeatures -o name 2>/dev/null | grep -E -- '-gpustack-worker$') -[ -n "$WORKER_NFS" ] || { echo "no -gpustack-worker NodeFeatures found"; exit 1; } -echo "general InstanceType: ${IT}; draining $(echo "$WORKER_NFS" | grep -c .) node(s)" - # The derived general InstanceType is created with its unit spec, and that spec is immutable # afterwards, so there is nothing to set: read it as a precondition. The Instance webhook sizes the # Pod from it. unit_ram=$(kubectl get instancetypes.worker.gpustack.ai "$IT" -o jsonpath='{.spec.unitResources.ram}' 2>/dev/null) -[ -n "$unit_ram" ] || { echo "no unit spec on ${IT} — the Instance webhook needs unitRAM to size the Pod"; exit 1; } +echo "general InstanceType: ${IT}; ResourceFlavor: ${RF#*/}; draining $(echo "$WORKER_NFS" | grep -c .) node(s)" + +# Preconditions, read once. Each scope is gated on its own: a missing piece fails the rows that +# need it and leaves the other scopes' rows intact, so no scope's abort suppresses another's rows. +NO_UNIT="" +[ -n "$unit_ram" ] || NO_UNIT="no unit spec on ${IT} — the Instance webhook needs unitRAM to size the Pod" +NO_NODES="" +[ -n "$WORKER_NFS" ] || NO_NODES="no -gpustack-worker NodeFeatures found — the pool cannot be toggled" +NO_RF="" +[ -n "$RF" ] || NO_RF="no CPU ResourceFlavor found — run case-1 first to materialize the chain" restore() { echo echo "[case-2] restoring gpustack.ai/managed=true on all nodes, deleting test Instance, waiting for rebuild" echo "$WORKER_NFS" | xargs -r -I{} kubectl -n "$NS" patch {} --type=merge \ -p '{"spec":{"labels":{"gpustack.ai/managed":"true"}}}' 2>/dev/null || true - kubectl -n default delete instance gpustack-e2e-instance --ignore-not-found 2>/dev/null || true + kubectl -n default delete instance "$INST" --ignore-not-found 2>/dev/null || true # managed=true alone reactivates the pool. The drain emptied its queue through the held flavor # migration -- HoldAndDrain first, then the resource groups -- and the emptied queue stays held until # a flavor returns: it takes the returning flavor while still held and only then gets its saved @@ -78,94 +107,212 @@ restore() { } trap restore EXIT -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } - -# 1. Create a running Instance referencing the Active InstanceType. alpine is kept alive (sleep) -# so its Kueue Workload holds quota; the ephemeral volume lets non-type validation pass. -cat </dev/null \ + | grep -m1 -x 'gpustack-e2e-instance|True') + [ -n "$a" ] && { admitted=1; break; } + sleep 3 + done + if [ -n "$admitted" ]; then + record PASS "workload admitted" "the Workload owned by Pod gpustack-e2e-instance is Admitted=True (holds quota)" + else + record FAIL "workload admitted" "the Workload owned by Pod gpustack-e2e-instance never reached Admitted=True — the cpu-only CQ must admit despite the Pod's memory/ephemeral-storage (quotaCheckStrategy: IgnoreUndeclared)" + fi + + # The drain has to meet a RUNNING Instance, which is what this case claims to stop, and spec.stop + # unset is not that: an Instance whose Pod is still pulling its image or failing readiness is + # Starting, and a drain that lands then exercises the stop of a Starting Instance instead. Wait for + # the phase to reach Ready before draining. + ready="" + for _ in $(seq 1 40); do + IFS='|' read -r phase0 stop0 <<<"$(kubectl -n default get instance "$INST" \ + -o jsonpath='{.status.phase}|{.spec.stop}' 2>/dev/null)" + [ "$phase0" = "Ready" ] && [ "$stop0" != "true" ] && { ready=1; break; } + sleep 3 + done + if [ -n "$ready" ]; then + record PASS "instance running pre-drain" "phase=Ready spec.stop=${stop0:-}" + else + record FAIL "instance running pre-drain" "phase=${phase0:-} spec.stop=${stop0:-} — the drain would not meet a running Instance" + fi + + # THE portless assertion: the Instance status is written (phase non-empty). Before the fix the + # reconcile errored on Service creation before the status block, leaving .status empty while the + # Pod ran. + phase="" + for _ in $(seq 1 40); do + phase=$(kubectl -n default get instance "$INST" -o jsonpath='{.status.phase}' 2>/dev/null) + [ "$phase" = "Ready" ] && break + sleep 3 + done + case "$phase" in + Ready) record PASS "instance status written" "phase=Ready (portless Instance progresses)" ;; + Starting) record PASS "instance status written" "phase=Starting (status written; not Ready within the wait)" ;; + *) record FAIL "instance status written" "phase='${phase:-}' — status never written (portless Service error blocked it)" ;; + esac + + # No Service must be created for a portless Instance. + if kubectl -n default get svc "$INST" >/dev/null 2>&1; then + record FAIL "no Service for portless Instance" "svc/${INST} exists — a portless workload needs none" + else + record PASS "no Service for portless Instance" "no svc/${INST} (Service creation skipped)" + fi + + # A Workload reports Admitted=True a few seconds BEFORE the CQ's reservingWorkloads counter + # reflects it. Gate the drain on the counted reservation so the drain meets a queue that holds + # this Instance's reservation, which is the state in which Kueue has an admitted Workload to + # evict. An unreserved queue is also put on HoldAndDrain before it is emptied, and the Instance is + # stopped on that StopPolicy either way, so this gate decides what the drain evicts, not whether + # the stop happens. + for _ in $(seq 1 20); do + rw=$(kubectl get clusterqueue "$IT" -o jsonpath='{.status.reservingWorkloads}' 2>/dev/null) + [ "${rw:-0}" -ge 1 ] && break + sleep 2 + done +else + record FAIL "workload admitted" "$NO_UNIT" + record FAIL "instance running pre-drain" "$NO_UNIT" + record FAIL "instance status written" "$NO_UNIT" + record FAIL "no Service for portless Instance" "$NO_UNIT" +fi + +# 3. Drain: exclude the nodes from management so the general flavor is deleted and the pool's queue +# is drained; the InstanceType itself survives. Toggle via the NodeFeature (NFD reverts a direct +# node label). +if [ -z "$NO_NODES" ]; then + echo "[case-2] draining: gpustack.ai/managed=false on all worker NodeFeatures" + echo "$WORKER_NFS" | xargs -r -I{} kubectl -n "$NS" patch {} --type=merge \ + -p '{"spec":{"labels":{"gpustack.ai/managed":"false"}}}' + + # The guaranteed drain effect: the pool's general ResourceFlavor is deleted (node-index driven, + # independent of the running Instance/Workload). This confirms the managed-toggle propagated. + rf_gone="" + for _ in $(seq 1 40); do + n=$(kubectl get resourceflavors.kueue.x-k8s.io -o name 2>/dev/null | grep -cE '/gpustack--.*-[0-9]+c(-fnv64-[0-9a-f]{16})?$') + [ "${n:-0}" -eq 0 ] && { rf_gone=1; break; } + sleep 3 + done + if [ -n "$rf_gone" ]; then + record PASS "drain removes the pool ResourceFlavor" "no CPU (…-Nc) flavor left" + else + record FAIL "drain removes the pool ResourceFlavor" "a CPU (…-Nc) flavor persists — managed-toggle did not propagate" + fi + + # THE assertion: the Instance whose backing queue is now draining (HoldAndDrain) gets STOPPED, + # not recreated. On a drain the queue evicts the Pod; the reconciler must set spec.stop (which + # deletes the Pod) instead of recreating a Pod the drained queue can never admit. A ClusterQueue + # watch re-enqueues the Instance so the stop is prompt even when no Pod event fires (poll — async). + if [ -z "$NO_UNIT" ]; then + stopped="" + for _ in $(seq 1 40); do + s=$(kubectl -n default get instance "$INST" -o jsonpath='{.spec.stop}' 2>/dev/null) + [ "$s" = "true" ] && { stopped=1; break; } + sleep 3 + done + if [ -n "$stopped" ]; then + record PASS "instance STOPPED (not recreated)" "spec.stop=true" + else + record FAIL "instance STOPPED (not recreated)" "spec.stop still ${s:-} — a running Instance must stop when its type drains" + fi + else + record FAIL "instance STOPPED (not recreated)" "$NO_UNIT" + fi + + # The node's general ResourceFlavor is DELETED (no tombstone). NFD can take a few minutes to + # propagate a NodeFeature label change to the Node object on a busy cluster, and the flavor + # deletion only follows once the Node is seen unmanaged — wait generously. + if [ -z "$NO_RF" ]; then + gone="" + for _ in $(seq 1 90); do + kubectl get "$RF" >/dev/null 2>&1 || { gone=1; break; } + sleep 3 + done + if [ -n "$gone" ]; then + record PASS "flavor deleted on de-manage" "${RF#*/} gone (no drain tombstone)" + else + record FAIL "flavor deleted on de-manage" "${RF#*/} still present — NodeFlavorReconciler did not drop the unmanaged node" + fi + else + record FAIL "flavor deleted on de-manage" "$NO_RF" + fi -# 2. The Pod must be admitted by Kueue (holding quota is what lets the drain evict it). The Pod is -# named after the Instance and owns its Workload, so the Workload is found by that owner: any -# other admitted Workload in the namespace says nothing about this Instance. -admitted="" -for _ in $(seq 1 20); do - a=$(kubectl -n default get workloads.kueue.x-k8s.io \ - -o jsonpath='{range .items[*]}{.metadata.ownerReferences[0].name}{"|"}{.status.conditions[?(@.type=="Admitted")].status}{"\n"}{end}' 2>/dev/null \ - | grep -m1 -x 'gpustack-e2e-instance|True') - [ -n "$a" ] && { admitted=1; break; } - sleep 3 -done -[ -n "$admitted" ] && record PASS "workload admitted" "the Workload owned by Pod gpustack-e2e-instance is Admitted=True (holds quota)" \ - || record FAIL "workload admitted" "the Workload owned by Pod gpustack-e2e-instance never reached Admitted=True — the cpu-only CQ must admit despite the Pod's memory/ephemeral-storage (quotaCheckStrategy: IgnoreUndeclared)" - -# The drain has to meet a RUNNING Instance, which is what this case claims to stop, and spec.stop -# unset is not that: an Instance whose Pod is still pulling its image or failing readiness is -# Starting, and a drain that lands then exercises the stop of a Starting Instance instead. Wait for -# the phase to reach Ready before draining. -ready="" -for _ in $(seq 1 40); do - IFS='|' read -r phase0 stop0 <<<"$(kubectl -n default get instance gpustack-e2e-instance \ - -o jsonpath='{.status.phase}|{.spec.stop}' 2>/dev/null)" - [ "$phase0" = "Ready" ] && [ "$stop0" != "true" ] && { ready=1; break; } - sleep 3 -done -[ -n "$ready" ] && record PASS "instance running pre-drain" "phase=Ready spec.stop=${stop0:-}" \ - || record FAIL "instance running pre-drain" "phase=${phase0:-} spec.stop=${stop0:-} — the drain would not meet a running Instance" - -# A Workload reports Admitted=True a few seconds BEFORE the CQ's reservingWorkloads counter reflects -# it. Gate the drain on the counted reservation so the drain meets a queue that holds this -# Instance's reservation, which is the state in which Kueue has an admitted Workload to evict. An -# unreserved queue is also put on HoldAndDrain before it is emptied (the path case-3 exercises), and -# the Instance is stopped on that StopPolicy either way, so this gate decides what the drain evicts, -# not whether the stop happens. -for _ in $(seq 1 20); do - rw=$(kubectl get clusterqueue "$IT" -o jsonpath='{.status.reservingWorkloads}' 2>/dev/null) - [ "${rw:-0}" -ge 1 ] && break - sleep 2 -done - -# 3. Drain: exclude the node from management so the general flavor is deleted and the pool's queue -# is drained; the InstanceType itself survives (CASE 3). Toggle via the NodeFeature (NFD reverts -# a direct node label). -echo "[case-2] draining: gpustack.ai/managed=false on all worker NodeFeatures" -echo "$WORKER_NFS" | xargs -r -I{} kubectl -n "$NS" patch {} --type=merge \ - -p '{"spec":{"labels":{"gpustack.ai/managed":"false"}}}' - -# 4. The guaranteed drain effect: the pool's general ResourceFlavor is deleted (node-index driven, -# independent of the running Instance/Workload). This confirms the managed-toggle propagated; -# the fuller teardown (InstanceType survival + queue emptying) is CASE 3. -rf_gone="" -for _ in $(seq 1 40); do - n=$(kubectl get resourceflavors.kueue.x-k8s.io -o name 2>/dev/null | grep -cE '/gpustack--.*-[0-9]+c(-fnv64-[0-9a-f]{16})?$') - [ "${n:-0}" -eq 0 ] && { rf_gone=1; break; } - sleep 3 -done -[ -n "$rf_gone" ] && record PASS "drain removes the pool ResourceFlavor" "no CPU (…-Nc) flavor left" \ - || record FAIL "drain removes the pool ResourceFlavor" "a CPU (…-Nc) flavor persists — managed-toggle did not propagate" - -# 5. THE assertion: the Instance whose backing queue is now draining (HoldAndDrain) gets STOPPED, -# not recreated. On a drain the queue evicts the Pod; the reconciler must set spec.stop (which -# deletes the Pod) instead of recreating a Pod the drained queue can never admit. A ClusterQueue -# watch re-enqueues the Instance so the stop is prompt even when no Pod event fires (poll — async). -stopped="" -for _ in $(seq 1 40); do - s=$(kubectl -n default get instance gpustack-e2e-instance -o jsonpath='{.spec.stop}' 2>/dev/null) - [ "$s" = "true" ] && { stopped=1; break; } - sleep 3 -done -[ -n "$stopped" ] && record PASS "instance STOPPED (not recreated)" "spec.stop=true" \ - || record FAIL "instance STOPPED (not recreated)" "spec.stop still ${s:-} — a running Instance must stop when its type drains" + # The InstanceType SURVIVES (not deleted); its backing queue's quota is emptied. The emptied + # queue must still be held: restoring its StopPolicy before a flavor returns reopens a queue with + # no quota. Both fields come from one read, so they describe the same instant; an absent + # stopPolicy is Kueue's default, None. + survived="" + for _ in $(seq 1 60); do + exists=$(kubectl get instancetypes.worker.gpustack.ai "$IT" -o name 2>/dev/null) + cqexists=$(kubectl get clusterqueue "$IT" -o name 2>/dev/null) + read -r rg stop < <(kubectl get clusterqueue "$IT" -o json 2>/dev/null | python3 -c "import json,sys +try: + spec = json.load(sys.stdin).get('spec', {}) + print(len(spec.get('resourceGroups') or []), spec.get('stopPolicy') or 'None') +except Exception: print(-1, '?')" 2>/dev/null) + if [ -n "$exists" ] && [ -n "$cqexists" ] && [ "$rg" = "0" ] && [ "$stop" != "None" ]; then survived=1; break; fi + sleep 3 + done + if [ -n "$survived" ]; then + record PASS "InstanceType survives flavor loss; queue emptied and held" "${IT} kept, resourceGroups=0, stopPolicy=${stop} (drain-when-no-flavors, idle)" + else + record FAIL "InstanceType survives flavor loss; queue emptied and held" "exists='${exists:-gone}' cqexists='${cqexists:-gone}' resourceGroups='${rg}' stopPolicy='${stop}' — the type must survive with its queue emptied and held, not tear down or reopen" + fi +else + record FAIL "drain removes the pool ResourceFlavor" "$NO_NODES" + record FAIL "instance STOPPED (not recreated)" "$NO_NODES" + record FAIL "flavor deleted on de-manage" "$NO_NODES" + record FAIL "InstanceType survives flavor loss; queue emptied and held" "$NO_NODES" +fi + +# 4. Restore the nodes: the pool refills its queue and the InstanceType reactivates. +if [ -z "$NO_NODES" ]; then + echo "[case-2] restoring gpustack.ai/managed=true; expecting the pool to refill and reactivate" + echo "$WORKER_NFS" | xargs -r -I{} kubectl -n "$NS" patch {} --type=merge \ + -p '{"spec":{"labels":{"gpustack.ai/managed":"true"}}}' >/dev/null 2>&1 + reactivated="" + for _ in $(seq 1 90); do + p=$(kubectl get instancetypes.worker.gpustack.ai "$IT" -o jsonpath='{.status.phase}' 2>/dev/null) + rg=$(kubectl get clusterqueue "$IT" -o json 2>/dev/null | python3 -c "import json,sys +try: print(len(json.load(sys.stdin).get('spec',{}).get('resourceGroups') or [])) +except Exception: print(0)" 2>/dev/null) + if [ "$p" = "Active" ] && [ "${rg:-0}" -gt 0 ]; then reactivated=1; break; fi + sleep 3 + done + if [ -n "$reactivated" ]; then + record PASS "pool reactivates when nodes return" "${IT} Active, resourceGroups refilled" + else + record FAIL "pool reactivates when nodes return" "phase='${p:-?}' resourceGroups='${rg:-0}' — the pool must refill and reactivate" + fi +else + record FAIL "pool reactivates when nodes return" "$NO_NODES" +fi # Which branch set spec.stop is printed for diagnosis only. The stop itself is the behavior and is # asserted above; a log sentence is wording the controller is free to change, so it is not a verdict @@ -173,8 +320,14 @@ done cnt=$(kubectl -n "$NS" logs deploy/gpustack-operator-worker --since=10m 2>/dev/null | grep -c "stop instance as its instance type is gone, deleting, or draining") echo "[case-2] worker log lines naming the stop-on-drain branch: ${cnt:-0}" +# The API server's refusal of a portless Service is printed for diagnosis only. A zero count cannot +# fail the case on its own: it is also what a log line that stops naming the Instance produces, so +# the verdict rests on the status and Service rows above. +errs=$(kubectl -n "$NS" logs deploy/gpustack-operator-worker --since=5m 2>/dev/null | grep -c "spec.ports: Required value") +echo "[case-2] worker log lines with 'spec.ports: Required value' in the last 5m: ${errs:-0}" + echo -echo "== CASE 2 — Running Instance admits, then drain stops it (not recreate) ==" +echo "== CASE 2 — Portless Instance admits, writes status, skips Service; the drain stops it, the managed toggle scopes onboarding ==" { echo "STATUS|CHECK|OBJECT" printf '%s\n' "${ROWS[@]}" @@ -182,7 +335,10 @@ echo "== CASE 2 — Running Instance admits, then drain stops it (not recreate) if [ "$FAILS" -ne 0 ]; then echo - echo "FAILED ${FAILS} check(s). See references/drain-recycle.md (why a real cluster & the drain trigger)." + echo "FAILED ${FAILS} check(s). A portless Instance must be admitted, write its status and create" + echo "no Service; a drained pool must stop the running Instance and keep the InstanceType alive" + echo "with its queue emptied and held. Confirm the operator was NOT restarted between toggle and" + echo "assertion. See references/drain-recycle.md (why a real cluster & the drain trigger)." exit 1 fi echo "CASE 2 PASS" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-20.sh b/.agents/skills/gpustack-operator-e2e/cases/case-20.sh deleted file mode 100755 index 5a561f0f9..000000000 --- a/.agents/skills/gpustack-operator-e2e/cases/case-20.sh +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/env bash -# -# CASE 20 — Sibling InstanceTypes on one pool stay status-consistent (MUTATING, self-recovering; AUTO-SKIPS without a real sliceable accelerator) -# -# case-20.sh -# -# Goal: Several InstanceTypes that declare the SAME os / arch / acceleratorGroup / acceleratable -# share ONE ResourceFlavor + Devices pool, so a Devices-ledger change must re-enqueue EVERY -# one of them and leave their four-view status identical — the InstanceTypeReconciler's -# enqueueInstanceTypeWhenDevicesChanged lists all types carrying the pool's feature key, not -# just the one whose name matches. Deploying a sliced workload moves the ledger and is the -# trigger. -# Environment: Needs a REAL LOGICALLY sliceable accelerator pool (a Devices ledger a logical slice -# actually moves; the same path CASE 8 uses). A partition-only pool does NOT qualify — it -# serves no logical slice, and such a request is now rejected at admission. AUTO-SKIPS -# (exit 0, prints why) on a GPU-less / non-logically-sliceable cluster. -# Inputs: All real, nothing mocked — -# - two throwaway admin InstanceTypes e2e-case20-a / e2e-case20-b declaring the derived -# accelerated pool's acceleratorGroup/os/arch (+ a unit spec); -# - one Pod e2e-case20-load requesting a 50% slice on the pool's entrance LocalQueue. -# Expected: - all siblings (the derived one + a + b) report the SAME four-view status before the load; -# - the sliced Pod admits + runs (a slice is taken from the pool); -# - after the load every sibling's four-view is still identical AND has moved from the -# pre-load value — proof the ledger change re-enqueued them all. -# Cleanup: Trap deletes the load Pod and both throwaway types (+ their ClusterQueues), then waits for -# the derived type's status to return to its pre-load value. -set -uo pipefail - -# Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail -# on transport alone, and a check that takes such a failure for an answer reports a verdict -# about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" - -NS="${1:?usage: case-20.sh }" -A=e2e-case20-a -B=e2e-case20-b -POD=gpustack-e2e-case20-load - -# sig — the pool's four-view remaining signature (EX|SH|SL|PT), or empty. -sig() { - kubectl get instancetype "$1" -o jsonpath='{.status.accelerator.remaining}|{.status.acceleratorShared.remaining}|{.status.acceleratorSliced.remaining}|{.status.acceleratorPartitioned.remaining}' 2>/dev/null -} - -# --- Skip gate: a real LOGICALLY sliceable accelerated pool (derived InstanceType, sliced capacity > 0). -# A hardware-partitioning capability is deliberately NOT accepted: the load below is a logical slice, -# which a partitioned card cannot serve. --- -read -r DERIVED AKEY OS ARCH LQ MANUF <<<"$(kubectl get instancetypes.worker.gpustack.ai -o json 2>/dev/null | python3 -c " -import json,sys -for it in json.load(sys.stdin).get('items',[]): - s=it.get('spec',{}); st=it.get('status',{}); n=it['metadata']['name']; d=st.get('detail',{}); sd=d.get('slicedDetail',{}) - sliced=st.get('acceleratorSliced',{}).get('capacity','0') - sliceable=(sd.get('logical',{}).get('count',0) or 0)>0 - if n.startswith('e2e-case20'): continue # skip this case's own throwaways (e.g. a prior aborted run) - if s.get('acceleratable') and s.get('acceleratorGroup') and sliceable and sliced not in ('','0') and st.get('entrance'): - print(n, s['acceleratorGroup'], s.get('os',''), s.get('arch',''), st['entrance'], d.get('manufacturer','')); break -")" -if [ -z "${DERIVED:-}" ] || [ -z "${LQ:-}" ]; then - echo "== CASE 20 — SKIPPED ==" - echo "No real logically sliceable accelerated pool (an acceleratable InstanceType with a non-zero logical" - echo "slice count, a non-zero sliced capacity and an entrance LocalQueue) — this case needs real accelerator" - echo "hardware whose cards are NOT in a hardware partitioning mode. Run it on such a GPU cluster." - exit 0 -fi -# The sliced resource + its memory/cores percentage keys, read off a pool node's allocatable (the -# device-manager advertises "/.sliced[.memory-percentage|.cores-percentage]"). -read -r SLICED MEMPCT CORESPCT <<<"$(kubectl get nodes -o json 2>/dev/null | python3 -c " -import json,sys -akey='${AKEY}' -for n in json.load(sys.stdin).get('items',[]): - if n['metadata'].get('labels',{}).get('acceleratable.feature.gpustack.ai/'+akey)!='true': continue - for k in n.get('status',{}).get('allocatable',{}): - if k.endswith('.sliced'): - print(k, k+'.memory-percentage', k+'.cores-percentage'); raise SystemExit -")" -[ -n "${SLICED:-}" ] || { echo "[case-20] no *.sliced resource advertised on a node carrying ${AKEY}"; exit 1; } -echo "[case-20] pool: derived=${DERIVED} acceleratorGroup=${AKEY} os/arch=${OS}/${ARCH} entrance=${LQ} sliced=${SLICED}" - -# The sliced load Pod runs the HAMi logical-slicing runtime (libvgpu.so via LD_PRELOAD) — without the -# vendor runtimeClass mounting its driver-lib dependencies a bare image exits 127 and never runs. -# Derive it from the pool manufacturer (identity map: nvidia->nvidia, mthreads->mthreads). -RUNTIMECLASS="" -if [ -n "$MANUF" ] && kubectl get runtimeclass.node.k8s.io "$MANUF" >/dev/null 2>&1; then RUNTIMECLASS="$MANUF"; fi -RTC_LINE=""; [ -n "$RUNTIMECLASS" ] && RTC_LINE="runtimeClassName: ${RUNTIMECLASS}" -echo "[case-20] load Pod runtimeClass: ${RUNTIMECLASS:-}" - -cleanup() { - echo - echo "[case-20] cleanup: deleting load Pod + throwaway types" - kubectl -n default delete pod "$POD" --ignore-not-found --force --grace-period=0 >/dev/null 2>&1 || true - for t in "$A" "$B"; do - kubectl patch instancetype "$t" --type=merge -p '{"metadata":{"finalizers":[]}}' >/dev/null 2>&1 || true - kubectl delete instancetype "$t" --wait=false >/dev/null 2>&1 || true - kubectl delete clusterqueue "$t" --wait=false >/dev/null 2>&1 || true - done - # Let the freed slice settle back so a following case sees the pool whole again. - for _ in $(seq 1 30); do [ "$(sig "$DERIVED")" = "$BASE" ] && break; sleep 3; done -} -trap cleanup EXIT - -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } - -BASE=$(sig "$DERIVED") -[ -n "$BASE" ] && [ "$BASE" != "|||" ] || { echo "[case-20] derived pool has no four-view status yet"; exit 1; } -echo "[case-20] baseline four-view (EX|SH|SL|PT) on ${DERIVED} = ${BASE}" - -# 1. Declare two sibling admin InstanceTypes on the SAME pool identity. -for t in "$A" "$B"; do - kubectl apply -f - >/dev/null 2>&1 </dev/null 2>&1 -apiVersion: v1 -kind: Pod -metadata: { name: ${POD}, namespace: default, labels: { kueue.x-k8s.io/queue-name: ${LQ} } } -spec: - schedulerName: default-scheduler - ${RTC_LINE} - restartPolicy: Never - containers: - - name: main - image: ubuntu:24.04 - command: ["sleep", "86400"] - resources: - limits: { ${SLICED}: "1", ${MEMPCT}: "50", ${CORESPCT}: "50" } - requests: { ${SLICED}: "1", ${MEMPCT}: "50", ${CORESPCT}: "50" } -EOF -running="" -for _ in $(seq 1 50); do - [ "$(kubectl -n default get pod "$POD" -o jsonpath='{.status.phase}' 2>/dev/null)" = "Running" ] && { running=1; break; } - sleep 3 -done -[ -n "$running" ] \ - && record PASS "logical-slice workload admits + runs" "${POD} Running (a slice is taken from the pool)" \ - || record FAIL "logical-slice workload admits + runs" "${POD} not Running — Kueue admission / node-devices AdmissionCheck / scheduling failed" - -# 3. The ledger change must re-enqueue EVERY sibling: all stay identical AND move from the baseline. -moved="" -for _ in $(seq 1 40); do - sa=$(sig "$A"); sb=$(sig "$B"); sd=$(sig "$DERIVED") - if [ -n "$sd" ] && [ "$sd" = "$sa" ] && [ "$sa" = "$sb" ] && [ "$sd" != "$BASE" ]; then moved=1; break; fi - sleep 3 -done -[ -n "$moved" ] \ - && record PASS "ledger change re-enqueues all siblings consistently" "all three = ${sd} (moved from ${BASE}, equal)" \ - || record FAIL "ledger change re-enqueues all siblings consistently" "derived=${sd:-?} a=${sa:-?} b=${sb:-?} base=${BASE} — a Devices change must refresh every type on the pool" - -echo -echo "== CASE 20 — Sibling InstanceTypes on one pool stay status-consistent ==" -{ - echo "STATUS|CHECK|OBJECT" - printf '%s\n' "${ROWS[@]}" -} | column -t -s '|' - -if [ "$FAILS" -ne 0 ]; then - echo - echo "FAILED ${FAILS} check(s). Types sharing os/arch/acceleratorGroup/acceleratable share one pool;" - echo "enqueueInstanceTypeWhenDevicesChanged must enqueue ALL of them on a Devices change so their" - echo "four-view status stays identical. See pkg/worker/controllers/worker/instance_type.go." - exit 1 -fi -echo "CASE 20 PASS" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-21.sh b/.agents/skills/gpustack-operator-e2e/cases/case-21.sh index 7d81a734c..cb3ddb937 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-21.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-21.sh @@ -35,8 +35,8 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: case-21.sh }" INST=gpustack-e2e-ssh-noninteractive @@ -46,6 +46,9 @@ FWD_PORT=22121 MARK=__case21_exec__ KEYDIR="$(mktemp -d)" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_ssh-lib.sh" + for _tool in ssh ssh-keygen sftp; do command -v "$_tool" >/dev/null 2>&1 || { echo "== CASE 21 — SKIPPED =="; echo "missing ssh client tool: $_tool"; exit 0; } done @@ -54,22 +57,14 @@ IT=$(kubectl get instancetypes.worker.gpustack.ai \ -o jsonpath='{.items[?(@.spec.acceleratable==false)].metadata.name}' 2>/dev/null | tr ' ' '\n' | grep -m1 'gpustack-') [ -n "$IT" ] || { echo "no general InstanceType found — run case-1 first to materialize the chain"; exit 1; } -PF_PID="" -FWD_PID="" restore() { echo echo "[case-21] cleanup" - [ -n "$FWD_PID" ] && kill "$FWD_PID" 2>/dev/null - [ -n "$PF_PID" ] && kill "$PF_PID" 2>/dev/null kubectl -n default delete instance "$INST" --ignore-not-found --wait=false 2>/dev/null || true kubectl -n default delete secret "$SECRET" --ignore-not-found 2>/dev/null || true rm -rf "$KEYDIR" } -trap restore EXIT - -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } +trap 'ssh_cleanup; restore' EXIT print_and_exit() { echo @@ -95,9 +90,7 @@ unit_ram=$(kubectl get instancetypes.worker.gpustack.ai "$IT" -o jsonpath='{.spe [ -n "$unit_ram" ] || { echo "no unit spec on ${IT} — the Instance webhook needs unitRAM to size the Pod"; exit 1; } # 1. SSH key + secret, then a CPU-only SSH-enabled Instance on the general pool. -ssh-keygen -t ed25519 -f "$KEYDIR/id" -N "" -q -kubectl -n default delete secret "$SECRET" --ignore-not-found >/dev/null 2>&1 -kubectl -n default create secret generic "$SECRET" --from-file=authorized_keys="$KEYDIR/id.pub" >/dev/null +ssh_setup_key_secret default "$SECRET" "$KEYDIR/id" echo "[case-21] creating CPU-only SSH Instance ${INST} on ${IT}" cat </dev/null apiVersion: worker.gpustack.ai/v1alpha1 @@ -132,18 +125,13 @@ fi record PASS "CPU-only SSH Instance reaches 2/2 Running" "${POD} main+sshd Running" # 3. Open a port-forward to the sshd sidecar for the SSH assertions below. -kubectl -n default port-forward "pod/$POD" "${LOCAL_PORT}:22" >/dev/null 2>&1 & -PF_PID=$! -for _ in $(seq 1 20); do (exec 3<>"/dev/tcp/127.0.0.1/${LOCAL_PORT}") 2>/dev/null && { exec 3>&- 3<&-; break; }; sleep 1; done - -SSH_OPTS=(-p "$LOCAL_PORT" -i "$KEYDIR/id" -o IdentitiesOnly=yes - -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=15) +ssh_open_port_forward default "pod/$POD" 22 "$LOCAL_PORT" # 4. Exec channel: `ssh host ''` must run the command in `main`, capability-stripped, no banner. # The whole command rides $SSH_ORIGINAL_COMMAND; a stock server runs it, the old chroot.sh discarded # it and launched an interactive login shell instead (so stdout came back empty). EXEC_CMD='echo MARK='"$MARK"'; grep -E "^Cap(Eff|Bnd):" /proc/self/status; mknod /tmp/case21blk b 8 0 2>/dev/null; echo MKNOD_RC=$?' -exec_stdout=$(ssh -T "${SSH_OPTS[@]}" root@127.0.0.1 "$EXEC_CMD" 2>"$KEYDIR/exec.err") +exec_stdout=$(ssh_exec "$KEYDIR/id" "$LOCAL_PORT" "$EXEC_CMD" 2>"$KEYDIR/exec.err") exec_stderr=$(cat "$KEYDIR/exec.err" 2>/dev/null) exec_capeff=$(printf '%s\n' "$exec_stdout" | awk '/^CapEff:/{print $2}') @@ -172,7 +160,7 @@ else fi # 5. Interactive path (regression): no command, stdin over a heredoc → banner + login shell, unchanged. -int_out=$(ssh -T "${SSH_OPTS[@]}" root@127.0.0.1 2>&1 <<'CMDS' +int_out=$(ssh_exec "$KEYDIR/id" "$LOCAL_PORT" 2>&1 <<'CMDS' echo INT_MARK=__case21_int__ exit CMDS @@ -187,15 +175,13 @@ fi # $SSH_ORIGINAL_COMMAND set to the configured sftp-server path. chroot.sh stages the bundled static # sftp-server into `main` and serves it there (not the musl sidecar), so a put/get round-trips and the # uploaded file is visible in `main`'s own filesystem. The old chroot.sh discarded the subsystem request. -SFTP_OPTS=(-P "$LOCAL_PORT" -i "$KEYDIR/id" -o IdentitiesOnly=yes - -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=15) SFTP_MARK=__case21_sftp__ printf '%s\n' "$SFTP_MARK" >"$KEYDIR/up.txt" -sftp "${SFTP_OPTS[@]}" root@127.0.0.1 >/dev/null 2>&1 </dev/null 2>&1 </dev/null) +sftp_seen=$(ssh_exec "$KEYDIR/id" "$LOCAL_PORT" 'cat /workspace/case21-sftp.txt' 2>/dev/null) if [ -f "$KEYDIR/down.txt" ] && diff -q "$KEYDIR/up.txt" "$KEYDIR/down.txt" >/dev/null 2>&1; then record PASS "sftp subsystem round-trips a file" "put/get returned identical bytes via the bundled sftp-server" else @@ -209,17 +195,14 @@ fi # 7. Loopback TCP port-forward: forward a local port through the Instance to the in-Pod sshd (:22) and # read its protocol greeting back. Proves TCP forwarding round-trips (direct-tcpip, ForceCommand-agnostic). -ssh -N -T "${SSH_OPTS[@]}" -o ExitOnForwardFailure=yes \ - -L "127.0.0.1:${FWD_PORT}:127.0.0.1:22" root@127.0.0.1 >/dev/null 2>&1 & -FWD_PID=$! +ssh_open_loopback_forward "$KEYDIR/id" "$LOCAL_PORT" "$FWD_PORT" 127.0.0.1 22 fwd_greeting="" for _ in $(seq 1 15); do g=$( (exec 3<>"/dev/tcp/127.0.0.1/${FWD_PORT}" && head -c 8 <&3) 2>/dev/null ) if printf '%s' "$g" | grep -q '^SSH-2.0'; then fwd_greeting="$g"; break; fi sleep 1 done -kill "$FWD_PID" 2>/dev/null; FWD_PID="" -kill "$PF_PID" 2>/dev/null; PF_PID="" +ssh_cleanup if [ -n "$fwd_greeting" ]; then record PASS "loopback TCP port-forward round-trips" "forwarded port reached the in-Pod sshd (${fwd_greeting})" else diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-22.sh b/.agents/skills/gpustack-operator-e2e/cases/case-22.sh index 1defa9cf6..6c80cc8e3 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-22.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-22.sh @@ -59,8 +59,8 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: case-22.sh }" PODPFX=gpustack-e2e-exshare @@ -203,10 +203,6 @@ restore() { } trap restore EXIT -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } - # mkpod [extra-limits] [image] — a Pod requesting 1 unit of , # pinned to the GPU node, tagged with the LocalQueue label only when is non-empty (variant A vs the # raw variants). [extra-limits] adds request/limit entries (e.g. the sliced memory budget); [image] diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-23.sh b/.agents/skills/gpustack-operator-e2e/cases/case-23.sh index ff32af7a8..435024f2f 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-23.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-23.sh @@ -56,441 +56,52 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: MIG_NODE_SSH= case-23.sh }" PODPFX=gpustack-e2e-mig -DM_DS=gpustack-operator-device-manager-nvidia -MIGSSH_TIMEOUT="${MIG_SSH_TIMEOUT:-90}" + +# shellcheck disable=SC2034 # Read by the sourced library, which shellcheck does not follow. +CASE_ID=23 +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_pool-lib.sh" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_partition-lib.sh" # --------------------------------------------------------------------------------------------------- -# Env gate — the node address the case must NOT guess. Without it the case cannot toggle MIG mode, so -# it stops here (exit 2, "input required") rather than proceeding blind. Supply MIG_NODE_SSH and re-run. +# The hardware gate and the cluster correlation. The library owns both: the node address is never +# guessed, the card's partitioning mode decides whether the case can run at all, and the k8s node +# behind that address is correlated with the pool that backs it. # --------------------------------------------------------------------------------------------------- -if [ -z "${MIG_NODE_SSH:-}" ]; then - echo "== CASE 23 — INPUT REQUIRED (not run) ==" - echo - echo "This case toggles a node's NVIDIA MIG mode over SSH, so it needs the node's SSH address." - echo "It will not guess or auto-discover it. Provide it and re-run:" - echo - echo " MIG_NODE_SSH= bash .claude/skills/gpustack-operator-e2e/cases/case-23.sh ${NS}" - echo - echo "Optional overrides:" - echo " MIG_NODE_NAME= # disambiguate when several nodes carry an NVIDIA card" - echo " MIG_NODE_SSH_OPTS='-i ...' # extra ssh options (identity file, port, jump host, …)" - echo " MIG_GPU_INDEX= # which card to toggle (default 0)" - echo " MIG_SSH_TIMEOUT= # per-ssh timeout (default 90)" - echo - echo "The address stays out of this script and out of the repo — it is passed at run time only." - exit 2 -fi - -GPU_INDEX="${MIG_GPU_INDEX:-0}" - -# node_ssh — run a command on the node (non-interactive, bounded). The caller passes a full -# command line; sudo is the caller's responsibility (nvidia-smi mode switches need it). -# One shared control connection for the whole case. wait_card_idle alone polls up to 40 times, and -# every poll otherwise pays a full TCP + auth handshake to a remote node; multiplexing collapses them -# onto a single handshake. ControlPersist outlives the `timeout` killing a slow client, so the master -# survives an individual call being cut short. node_ssh_close tears it down from the trap. -MIGSSH_CTL="${TMPDIR:-/tmp}/gpustack-e2e-mig-$$.sock" -node_ssh() { - # shellcheck disable=SC2086 - timeout "${MIGSSH_TIMEOUT}" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=15 -o BatchMode=yes \ - -o ControlMaster=auto -o ControlPath="$MIGSSH_CTL" -o ControlPersist=300 \ - ${MIG_NODE_SSH_OPTS:-} "$MIG_NODE_SSH" "$@" -} -node_ssh_close() { - ssh -O exit -o ControlPath="$MIGSSH_CTL" "${MIG_NODE_SSH:-}" >/dev/null 2>&1 || true -} - -# --- Confirm SSH reachability + sudo before mutating anything. --- -if ! node_ssh true >/dev/null 2>&1; then - echo "== CASE 23 — SKIPPED ==" - echo "Cannot SSH to '${MIG_NODE_SSH}' (BatchMode, ${MIGSSH_TIMEOUT}s). Check the address / key / MIG_NODE_SSH_OPTS." - exit 0 -fi - -# --- MIG-capability gate: the card must report a MIG mode field ('Enabled'/'Disabled'), not '[N/A]'. --- -INITIAL_MODE="$(node_ssh sudo nvidia-smi -i "$GPU_INDEX" --query-gpu=mig.mode.current --format=csv,noheader 2>/dev/null | tr -d '[:space:]')" -case "$INITIAL_MODE" in - Enabled|Disabled) ;; - *) - echo "== CASE 23 — SKIPPED ==" - echo "Card ${GPU_INDEX} on ${MIG_NODE_SSH} does not report a MIG mode (got '${INITIAL_MODE:-}') — not MIG-capable." - echo "MIG needs an A100/A30/H100/H200/B-series (or newer) data-center card. Run this case on such a node." - exit 0 - esac -echo "[case-23] node ${MIG_NODE_SSH} card ${GPU_INDEX}: MIG mode currently ${INITIAL_MODE}" - -# --- The nvidia GPU node in the cluster + its accelerated pool. Correlate the SSH host to a k8s node so -# the ledger/InstanceType assertions read the right object. Require a single nvidia-accelerated node -# unless MIG_NODE_NAME disambiguates. --- -GPU_NODE="${MIG_NODE_NAME:-}" -if [ -z "$GPU_NODE" ]; then - _nv=() - # Read the ledger and its exit status separately, and parse in a command substitution: a failed - # query or an unparseable answer would otherwise read as an empty list, which this gate turns into - # a SKIP. Only a query that answered with no nvidia group may skip. - if ! _devs="$(kubectl get devices.worker.gpustack.ai -o json 2>&1)"; then - echo "== CASE 23 — FAILED (setup) ==" - echo "Reading the Devices ledger failed, so this case cannot tell 'no nvidia hardware' from 'the" - echo "query did not answer'. It refuses to report a skip on either:" - printf '%s\n' "$_devs" | head -5 - exit 1 - fi - if ! _parsed="$(printf '%s' "$_devs" | python3 -c " -import json,sys -for d in json.load(sys.stdin).get('items',[]): - for g in d.get('spec',{}).get('groups',[]): - if g.get('manufacturer')=='nvidia' and g.get('accelerators'): - print(d['metadata']['name']); break -")"; then - echo "== CASE 23 — FAILED (setup) ==" - echo "The Devices ledger was read but could not be parsed, so this case cannot tell 'no nvidia" - echo "hardware' from 'the answer was unreadable'. It refuses to report a skip on either." - exit 1 - fi - while IFS= read -r _n; do [ -n "$_n" ] && _nv+=("$_n"); done <<<"$_parsed" - if [ "${#_nv[@]}" -eq 0 ]; then - echo "== CASE 23 — SKIPPED ==" - echo "No Devices object reports an nvidia accelerator group — the operator chain is not observing a GPU node." - exit 0 - fi - if [ "${#_nv[@]}" -gt 1 ]; then - echo "[case-23] several nvidia GPU nodes (${_nv[*]}); set MIG_NODE_NAME= and re-run" - exit 2 - fi - GPU_NODE="${_nv[0]}" -fi -echo "[case-23] correlated k8s node: ${GPU_NODE}" - -# Accelerated InstanceType for this node's nvidia group + its entrance LocalQueue + the manufacturer. -GROUPID=$(kubectl get devices "$GPU_NODE" -o json 2>/dev/null | python3 -c " -import json,sys -for g in json.load(sys.stdin).get('spec',{}).get('groups',[]): - if g.get('manufacturer')=='nvidia' and g.get('accelerators'): - print(g.get('id','')); break -" 2>/dev/null) -read -r IT LQ MANUF <<<"$(kubectl get instancetypes.worker.gpustack.ai -o json 2>/dev/null | NODE_GID="$GROUPID" NODE_JSON="$(kubectl get node "$GPU_NODE" -o json 2>/dev/null)" python3 -c " -import json,sys,os -gid=os.environ.get('NODE_GID','') -nl=(json.loads(os.environ.get('NODE_JSON') or '{}').get('metadata',{}) or {}).get('labels',{}) or {} -# A pool BACKS this node when every discriminator it carries is a label the node carries too. Those are -# the schedule labels the webhook stamps from PoolScheduleLabels, so this is the pool's whole identity — -# os/arch, the acceleratable boolean, the accelerator group and, under CPU-aware grouping, the general -# group. schedule.gpustack.ai/* is the pool's own bookkeeping and never a node label. -def backs(it): - d={k:v for k,v in (it['metadata'].get('labels') or {}).items() if not k.startswith('schedule.gpustack.ai/')} - return bool(d) and all(nl.get(k)==v for k,v in d.items()) -def in_group(s): - return bool(gid) and (s.get('acceleratorGroup') or '').endswith('-'+gid) -items=json.load(sys.stdin).get('items',[]) -# The pool identity this node belongs to, not its rendered name — see the index's note on pool lookup. -for it in items: - s=it.get('spec',{}); st=it.get('status',{}) - if s.get('acceleratable') and in_group(s) and backs(it) and st.get('entrance'): - print(it['metadata']['name'], st['entrance'], (st.get('detail') or {}).get('manufacturer','nvidia')); sys.exit(0) -for it in items: - s=it.get('spec',{}); st=it.get('status',{}) - if s.get('acceleratable') and st.get('entrance'): - print(it['metadata']['name'], st['entrance'], (st.get('detail') or {}).get('manufacturer','nvidia')); sys.exit(0) -")" -[ -n "${IT:-}" ] && [ -n "${LQ:-}" ] || { echo "[case-23] no accelerated InstanceType with an entrance LocalQueue — chain not materialized"; exit 1; } -MANUF="${MANUF:-nvidia}" -# The two disjoint accelerator families. LOGICAL slicing (software, the vendor preload library) is -# served only by a card that is NOT in a hardware partitioning mode; PHYSICAL partitioning (the MIG -# hardware) only by a card that is. Enabling MIG moves the card from the first family to the second. -SLICED="${MANUF}.com/gpu.sliced" -PARTITIONED="${MANUF}.com/gpu.partitioned" -echo "[case-23] accelerated InstanceType ${IT} (entrance LocalQueue ${LQ}, group ${GROUPID:-?}, logical base ${SLICED}, partition base ${PARTITIONED})" - -# The vendor runtimeClass mounts the driver libs the workload needs (nvidia-smi -L, and — on the logically sliced -# path — the LD_PRELOAD'd libvgpu.so, which exits 127 without it). Derive it from the manufacturer. -RUNTIMECLASS="" -if kubectl get runtimeclass.node.k8s.io "$MANUF" >/dev/null 2>&1; then RUNTIMECLASS="$MANUF"; fi -RTC_LINE=""; [ -n "$RUNTIMECLASS" ] && RTC_LINE="runtimeClassName: ${RUNTIMECLASS}" -echo "[case-23] test-pod runtimeClass: ${RUNTIMECLASS:-}" +part_require_node_ssh "case-23.sh" +part_require_mig_capable +part_discover # --------------------------------------------------------------------------------------------------- -# Bookkeeping + helpers. +# Cleanup. The trap restores what the case changed: the test Pods go, and the card goes back into the +# mode the case found it in. # --------------------------------------------------------------------------------------------------- -FAILS=0 -ROWS=() -TESTPODS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } - -running() { [ "$(kubectl -n default get pod "$1" -o jsonpath='{.status.phase}' 2>/dev/null)" = "Running" ]; } -wait_running() { for _ in $(seq 1 40); do running "$1" && return 0; sleep 3; done; return 1; } - -# workload_refusal — the Pod's own Kueue Workload verdict, when an AdmissionCheck has actually -# rendered one: "admission-check(/)" for Retry or Rejected, empty otherwise. This is the -# product saying "I refuse to admit this", so it is conclusive the moment it appears. -workload_refusal() { - kubectl -n default get workloads.kueue.x-k8s.io -o json 2>/dev/null | python3 -c " -import json,sys -pod='$1' -for wl in json.load(sys.stdin).get('items',[]): - if not any(o.get('name')==pod for o in wl.get('metadata',{}).get('ownerReferences',[])): - continue - st=wl.get('status',{}) - if any(c.get('type')=='Admitted' and c.get('status')=='True' for c in st.get('conditions',[])): - break - for c in st.get('admissionChecks',[]): - if c.get('state') in ('Retry','Rejected'): - print('admission-check(%s/%s)' % (c.get('name',''), c.get('state'))) - break - break -" 2>/dev/null -} - -# pod_events — the reasons of the events recorded against THIS Pod, one per line. Scoped to the -# Pod's UID as well as its name: events outlive their Pod by about an hour while this case reuses the -# same Pod names every round, so a name-only selector reports a previous round's events as this Pod's. -# Returns non-zero when the identity or the event query cannot be answered. -pod_events() { - local uid - uid="$(kubectl -n default get pod "$1" -o jsonpath='{.metadata.uid}' 2>/dev/null)" || return 1 - [ -n "$uid" ] || return 1 - kubectl -n default get events \ - --field-selector "involvedObject.name=$1,involvedObject.uid=${uid}" \ - -o jsonpath='{range .items[*]}{.reason}{"\n"}{end}' 2>/dev/null || return 1 -} - -# held_reason — a DECISIVE signal that the Pod was refused: a terminal failure, an Unschedulable -# verdict, a device-plugin admission refusal, or an AdmissionCheck that declined it. Empty when none is -# present. The bare Kueue scheduling gate is deliberately NOT among these — see assert_held. -held_reason() { - local p="$1" phase cond ev wl - phase="$(kubectl -n default get pod "$p" -o jsonpath='{.status.phase}' 2>/dev/null)" - [ "$phase" = Failed ] && { echo "Failed"; return; } - cond="$(kubectl -n default get pod "$p" -o jsonpath='{.status.conditions[?(@.type=="PodScheduled")].reason}' 2>/dev/null)" - [ "$cond" = Unschedulable ] && { echo "Unschedulable"; return; } - # A failed event query reads as "no event", which can only withhold a PASS, never grant one. - ev="$(pod_events "$p" | grep -iE 'UnexpectedAdmissionError|FailedScheduling' | head -1)" - [ -n "$ev" ] && { echo "$ev"; return; } - wl="$(workload_refusal "$p")" - [ -n "$wl" ] && { echo "$wl"; return; } - echo "" -} - -# assert_held [polls] — confirm the Pod is HELD rather than merely slow; PASS text on success, -# empty on "cannot confirm". -# -# Kueue's pod integration gates EVERY Pod born in this namespace, so the bare kueue.x-k8s.io/admission -# scheduling gate proves nothing on its own: a Pod that is about to admit normally carries it too, for -# the second or two before Kueue clears it. Evidence therefore comes in two grades: -# -# DECISIVE — a verdict has been rendered against the Pod (held_reason). Conclusive on sight. -# ENDURING — nothing but the birth gate. It means "held" only once it OUTLASTS the window a healthy -# admission needs, so it must survive every poll of the window. -# -# The window is always polled to its end unless the Pod resolves, because the device-plugin's -# UnexpectedAdmissionError backstop fires only AFTER Kueue clears the gate. Reaching Running at any -# point returns empty: the Pod was never held. -assert_held() { - local p="$1" t="${2:-20}" hr gate last_gate="" gated_throughout=1 - for _ in $(seq 1 "$t"); do - running "$p" && { echo ""; return; } - hr="$(held_reason "$p")" - [ -n "$hr" ] && { echo "$hr"; return; } - gate="$(kubectl -n default get pod "$p" -o jsonpath='{.spec.schedulingGates[*].name}' 2>/dev/null)" - if [ -n "$gate" ]; then last_gate="$gate"; else gated_throughout=0; fi - sleep 3 - done - [ "$gated_throughout" = 1 ] && [ -n "$last_gate" ] && - { echo "scheduling-gated(${last_gate}) sustained across ${t} polls"; return; } - echo "" -} - -# node_gi_count — the number of live GPU instances the card actually holds (node ground truth). -# It prints the count and returns 0 only when the node answered. Two exit statuses are real answers of -# "none": 0 with "No MIG-enabled devices found." and 6 with "No GPU instances found: Not Found". -# Anything else (unreachable node, timeout, sudo refused) prints nothing and returns non-zero, because -# a probe that never answered must never be read as an idle card. -node_gi_count() { - local out rc - out="$(node_ssh sudo nvidia-smi mig -lgi 2>&1)"; rc=$? - case "$rc" in - 0 | 6) ;; - *) return 1 ;; - esac - # grep -c prints "0" AND exits non-zero on no match, so keep the count and discard the status. - printf '%s\n' "$out" | grep -cE '^\|[[:space:]]+[0-9]+' || true -} - -# wait_card_idle — poll until the card holds no live GPU instances, bounded ~120s. A MIG mode switch -# needs an idle card (the disable prerequisite), so callers wait for the last workload's instance to -# reclaim before toggling the mode. An unanswered probe keeps it waiting: unknown is not idle. -# Returns 0 when idle, 1 on timeout. -wait_card_idle() { - local n - for _ in $(seq 1 40); do - n="$(node_gi_count)" && [ "$n" = 0 ] && return 0 - sleep 3 - done - return 1 -} - -# exec_out ... — run a command in the Pod and echo its output, retried. -# -# A Pod reports Running slightly before its container is attachable, so an exec issued the instant -# wait_running returns can come back empty — or fail outright with `container not found` — on a Pod -# whose allocation is perfectly correct. That matters most in pod_mig_devices below: `grep -c` prints -# 0 even when the exec failed, so an attach race reads as "no MIG device inside the Pod" and fails a -# correct allocation. -exec_out() { - local pod="$1"; shift - local out - for _ in $(seq 1 8); do - out=$(kubectl -n default exec "$pod" -- "$@" 2>/dev/null) - [ -n "$out" ] && { printf '%s\n' "$out"; return 0; } - sleep 3 - done - return 1 -} - -# pod_mig_devices — count of MIG devices nvidia-smi -L reports inside the Pod (want exactly 1). -pod_mig_devices() { - local out - out="$(exec_out "$1" nvidia-smi -L | grep -c 'MIG')" - echo "${out:-0}" -} - -# node_key — the node's allocatable quantity for an extended resource (empty if absent). -node_key() { kubectl get node "$GPU_NODE" -o jsonpath="{.status.allocatable['${1//./\\.}']}" 2>/dev/null; } - -# node_key_gone — true when a key is absent OR reads 0. Reconciler-owned counting keys are -# genuinely REMOVED when their family leaves the node, but a device-plugin pool key only zeroes out — -# the kubelet keeps the entry until it restarts — so "gone" must accept both for the pool keys. -node_key_gone() { local v; v="$(node_key "$1")"; [ -z "$v" ] || [ "$v" = 0 ]; } - -# partition_profile_keys — the per-profile partition keys the node currently advertises, i.e. every -# ".partitioned.-" (never ".partitioned.units", which is a counting key, not a -# profile). The kind segment is the manufacturer's own name for hardware partitioning and is read off -# the node rather than assumed, so the case does not hardcode a vendor's spelling of it. -partition_profile_keys() { - kubectl get node "$GPU_NODE" -o json 2>/dev/null | PFX="${PARTITIONED}." python3 -c " -import json,os,sys -pfx=os.environ['PFX'] -a=json.load(sys.stdin).get('status',{}).get('allocatable',{}) -print(' '.join(k for k in a if k.startswith(pfx) and not k.endswith('.units'))) -" 2>/dev/null -} - -# profile_key — the advertised per-profile key whose profile segment is , or empty. -profile_key() { - local want="$1" k - for k in $(partition_profile_keys); do - case "$k" in *-"$want") echo "$k"; return;; esac - done - echo "" -} - -# refresh_dm — the Device Manager writes the capability at startup, and its detect loop compares only -# {manufacturer, id, unhealthy}, which a partitioning-mode toggle does not change — so a mode flip is -# picked up by RESTARTING the DaemonSet. Deleting the Devices object is deliberately NOT done: an -# existing group's capability is rewritten in place, and deleting it here would hide a regression of -# that. Waits for the rollout and for the object to be present. -# Both halves are measured, not assumed: on an 8-card node the flip alone left the object at the same -# generation and resourceVersion, unchanged again minutes later, while each restart moved it exactly -# one generation — logical to partitioned and back — with the object's uid and creationTimestamp never -# changing. So the restart is required, and it is also sufficient. -refresh_dm() { - echo "[case-23] refreshing Device Manager (rollout restart ${DM_DS}; the Devices object is kept)" - kubectl -n "$NS" rollout restart "ds/${DM_DS}" >/dev/null 2>&1 || true - kubectl -n "$NS" rollout status "ds/${DM_DS}" --timeout=180s >/dev/null 2>&1 || true - for _ in $(seq 1 30); do kubectl get devices.worker.gpustack.ai "$GPU_NODE" >/dev/null 2>&1 && break; sleep 3; done -} - -# set_mig_mode <0|1> — toggle the card's MIG mode over SSH and wait until it converges (or a reset pends). -# Returns 0 on convergence, 1 otherwise (caller records the failure with guidance). -# Converged here means only that the CARD reports the new mode. The toggle on its own is invisible to -# the operator, so a caller that stops at this point asserts against the capability the card had -# BEFORE the flip and reads a stale answer as a real one. Follow every successful toggle with -# refresh_dm. -set_mig_mode() { - local want="$1" target; [ "$want" = 1 ] && target=Enabled || target=Disabled - echo "[case-23] nvidia-smi -i ${GPU_INDEX} -mig ${want} on ${MIG_NODE_SSH}" - node_ssh sudo nvidia-smi -i "$GPU_INDEX" -mig "$want" >/dev/null 2>&1 || true - for _ in $(seq 1 20); do - local cur; cur="$(node_ssh sudo nvidia-smi -i "$GPU_INDEX" --query-gpu=mig.mode.current --format=csv,noheader 2>/dev/null | tr -d '[:space:]')" - [ "$cur" = "$target" ] && return 0 - sleep 3 - done - return 1 -} - -# card_profiles — DISCOVER the SMALL(size-1)/MID(size-4,3g)/FULL(size-8,7g) profiles + per-card counts + -# the partitioned-card count N and the group's TOTAL card count NTOT, from the cards' own capability. -# The physical-slice geometry is CAPABILITY, so it lives in Devices.spec (the per-card runtime ledger is -# in .status). NTOT is what tells the case whether the WHOLE pool moved family (N == NTOT) or only part -# of it, which decides whether the logical keys may legitimately still be present. Emits: -# SMALL SMALL_CNT MID MID_CNT FULL FULL_CNT N NTOT. -card_profiles() { - kubectl get devices "$GPU_NODE" -o json 2>/dev/null | python3 -c " -import json,sys -d=json.load(sys.stdin) -small=mid=full=None; n=0; ntot=0 -for g in d.get('spec',{}).get('groups',[]): - if g.get('manufacturer')!='nvidia': continue - for a in g.get('accelerators',[]): - ntot+=1 - profs=a.get('status',{}).get('physicalSliced',{}).get('profiles',[]) or [] - if profs: n+=1 - for p in profs: - ms=p.get('memorySlices',0); cs=p.get('computeSlices',0) - if ms==1 and (small is None or p.get('memoryMib',0)/dev/null -} - restore() { echo echo "[case-23] cleanup: deleting test Pods and restoring MIG mode to '${INITIAL_MODE}'" - for p in "${TESTPODS[@]}"; do kubectl -n default delete pod "$p" --ignore-not-found --force --grace-period=0 >/dev/null 2>&1 || true; done - local now; now="$(node_ssh sudo nvidia-smi -i "$GPU_INDEX" --query-gpu=mig.mode.current --format=csv,noheader 2>/dev/null | tr -d '[:space:]')" + delete_test_pods + local now; now="$(mig_mode "$GPU_INDEX")" if [ "$now" != "$INITIAL_MODE" ]; then # A disable needs an idle card; wait for any just-deleted workload's instance to reclaim first. [ "$INITIAL_MODE" = Enabled ] || wait_card_idle || true - [ "$INITIAL_MODE" = Enabled ] && set_mig_mode 1 || set_mig_mode 0 + if [ "$INITIAL_MODE" = Enabled ]; then + set_mig_mode "$GPU_INDEX" 1 || set_mig_mode "$GPU_INDEX" 0 + else + set_mig_mode "$GPU_INDEX" 0 + fi refresh_dm fi node_ssh_close } trap restore EXIT -# mkpod — a Pod on the GPU node, through the pool's LocalQueue. -mkpod() { - local name="$1" reslines="$2"; TESTPODS+=("$name") - cat </dev/null -apiVersion: v1 -kind: Pod -metadata: { name: ${name}, namespace: default, labels: { kueue.x-k8s.io/queue-name: ${LQ} } } -spec: - schedulerName: default-scheduler - ${RTC_LINE} - restartPolicy: Never - nodeSelector: { kubernetes.io/hostname: ${GPU_NODE} } - containers: - - name: main - image: ubuntu:24.04 - command: ["sleep", "86400"] - resources: - limits: -${reslines} - requests: -${reslines} -EOF -} -delpod() { kubectl -n default delete pod "$1" --ignore-not-found --force --grace-period=0 >/dev/null 2>&1 || true; } - # =================================================================================================== # Phase L — Logical slicing, only when the card started with MIG OFF. # =================================================================================================== @@ -530,7 +141,7 @@ echo echo "[case-23] === Phase E: enable MIG + Device Manager re-detect ===" CUR="$(node_ssh sudo nvidia-smi -i "$GPU_INDEX" --query-gpu=mig.mode.current --format=csv,noheader 2>/dev/null | tr -d '[:space:]')" if [ "$CUR" != Enabled ]; then - if ! set_mig_mode 1; then + if ! set_mig_mode "$GPU_INDEX" 1; then record FAIL "enable MIG mode" "nvidia-smi -mig 1 did not converge to Enabled (a pending GPU reset on Ampere, a busy card, or a loaded nvidia_drm blocks it) — see Prerequisites" echo; echo "== CASE 23 — cannot proceed without MIG enabled =="; exit 1 fi @@ -781,7 +392,7 @@ echo "[case-23] === Phase D: disable MIG → logical-slice capability returns == # A mode switch needs an idle card — wait for the whole-card FULL from Phase S (just deleted) to # reclaim before disabling, else nvidia-smi -mig 0 cannot converge (the disable prerequisite). wait_card_idle || echo "[case-23] warning: card still reports live instances before disable" -if set_mig_mode 0; then +if set_mig_mode "$GPU_INDEX" 0; then refresh_dm logicalpct=""; partleft="x"; punits_left="x" for _ in $(seq 1 30); do diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-24.sh b/.agents/skills/gpustack-operator-e2e/cases/case-24.sh index 80629f697..751938563 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-24.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-24.sh @@ -21,7 +21,7 @@ # That is deliberate -- placement is what is under test here, and starting a round inside # the reclaim window would make it fail for a reason that has nothing to do with the card # populations. It does mean repetition here buys sampling, not extra coverage: reuse of a -# just-freed slot belongs to case-31, and this case stayed green throughout a defect that +# just-freed slot belongs to case-29, and this case stayed green throughout a defect that # destroyed every such reuse. # Environment: A reachable cluster whose active context is the GPU cluster, a node with at least TWO # nvidia cards of one model where at least one card can be put into a partitioning mode, @@ -51,8 +51,8 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: MIG_NODE_SSH= case-24.sh }" CASE_ID=24 diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-25.sh b/.agents/skills/gpustack-operator-e2e/cases/case-25.sh index c2b7436bb..e598289d8 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-25.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-25.sh @@ -1,45 +1,57 @@ #!/usr/bin/env bash # -# CASE 25 — Per-profile capacity is derived from the live ledger, not from a static ceiling +# CASE 25 — Partition capacity and token health are read from the live ledger, not from a static ceiling # (MUTATING, self-recovering; AUTO-SKIPS without a partition-capable node AND a node-SSH address) # # MIG_NODE_SSH= case-25.sh # -# Goal: A node's per-profile partition key reports what its own cards can still host, because -# the profiles compete for the same physical slices. Carving one 4-memory-slice instance -# consumes half a card, so the whole-card profile stops fitting on that card and its key -# falls — on a node whose only partitioned card holds one such instance, to zero — while -# the same-size profile's key is unchanged, because the key publishes -# allocated + remaining and the scheduler subtracts the running Pod's own request from it. -# A static ceiling would keep advertising the whole-card profile and attract a Pod that -# cannot be placed. The case also MEASURES what this costs: the ledger-derived key -# re-patches node capacity on every partition allocate and release, so the Node-object -# write volume is counted over an idle window and over a carve/free window and PRINTED for -# the record. Finally it exercises the residual the key shape does not close: the -# per-profile keys are independently subtracted scalars, so two Pods naming mutually -# exclusive profiles can both be admitted to a node that can host only one of them. The -# case FAILS if both reach Running on the one card, since that is a double-booked +# Goal: A node's partition keys answer one question: how many more instances can this node host. +# A per-profile key publishes what its own cards can STILL host, because the profiles +# compete for the same physical slices: carving one 4-memory-slice instance consumes half +# a card, so the whole-card profile stops fitting on that card and its key falls — on a +# node whose only partitioned card holds one such instance, to zero — while the same-size +# profile's key is unchanged, because the key publishes allocated + remaining and the +# scheduler subtracts the running Pod's own request from it. The node-level token count +# is a fungible count rather than a card selection: the node advertises allocated + +# remaining healthy tokens, so a saturated node still advertises its live instance count +# while the scheduler's FREE view is zero, and freeing one raises the count again with no +# restart of anything. A static ceiling would keep advertising the whole-card profile and +# attract a Pod that cannot be placed. The case also MEASURES what this costs: the +# ledger-derived key re-patches node capacity on every partition allocate and release, so +# the Node-object write volume is counted over an idle window and over a carve/free window +# and PRINTED for the record. Finally it exercises the residual the key shape does not +# close: the per-profile keys are independently subtracted scalars, so two Pods naming +# mutually exclusive profiles can both be admitted to a node that can host only one of +# them. The case FAILS if both reach Running on the one card, since that is a double-booked # placement; how the loser is held is recorded, not asserted. # Environment: A reachable cluster whose active context is the GPU cluster, an nvidia node with at # least one card that can be put into a hardware partitioning mode, AND SSH to that node # (sudo nvidia-smi) supplied via MIG_NODE_SSH=. It EXITS 2 (input required) # when MIG_NODE_SSH is unset, and AUTO-SKIPS (exit 0) when the card reports no -# partitioning mode or the mode switch does not converge. The strict "reads zero" and the -# residual observation need exactly ONE partitioned card; with more, the arithmetic is -# asserted relative to the partitioned card count and the held-request sub-check records -# SKIPPED. +# partitioning mode, when the mode switch does not converge, or when the cards advertise +# no per-profile key this case needs. The strict "reads zero" and the residual observation +# need exactly ONE partitioned card; with more, the arithmetic is asserted relative to the +# partitioned card count and the held-request sub-check records SKIPPED. # Inputs: All real, nothing mocked — the node's own keys and the live geometry are the # verification. The case partitions card MIG_GPU_INDEX if no card is partitioned yet, -# carves one instance of the discovered 4-memory-slice profile through the pool's entrance -# LocalQueue, and reads the node's allocatable keys around it. Profiles are DISCOVERED -# from the cards' own capability, never composed. +# carves one instance of the discovered 4-memory-slice profile, then saturates every +# partitioned card with one whole-card-profile instance, and reads the node's allocatable +# keys around both. Profiles are DISCOVERED from the cards' own capability, never composed. # Expected: - on an idle partitioned card set, the whole-card profile key equals the partitioned -# card count and the 4-slice profile key equals twice it; +# card count, the 4-slice profile key equals twice it, and the node-level token count +# equals the cards' ceiling; # - after ONE 4-slice instance is carved, the whole-card profile key drops by exactly one # card's worth (zero when there is a single partitioned card), while the 4-slice key is # unchanged at allocated + remaining; # - with a single partitioned card so occupied, a whole-card-profile Pod is HELD rather # than admitted onto a node that cannot place it; +# - once every partitioned card holds a whole-card instance, allocatable equals the live +# instance count (one per card) — not zero, and not the ceiling — and the capacity entry +# survives the saturation; +# - a further partition Pod is HELD, and every already-running partition Pod is still +# Running and still holds exactly one hardware instance; +# - freeing one instance raises the count again with no restart, and a fresh partition +# request admits; # - the Node-object write counts over both windows are recorded and printed; # - of two Pods naming mutually exclusive profiles on one card, at most one runs; how the # other is held is recorded. @@ -51,15 +63,18 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: MIG_NODE_SSH= case-25.sh }" CASE_ID=25 # shellcheck source=/dev/null . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_partition-lib.sh" -PODPFX=gpustack-e2e-ledgercap +# Two pod families, one per half of the case: the ledger half carves and frees, the saturation half +# fills every card. The prefixes keep the two sets apart, so neither half can read the other's Pods. +LEDGERPFX=gpustack-e2e-ledgercap +SATURATEPFX=gpustack-e2e-saturate IDLE_WINDOW="${MIG_WRITE_IDLE_WINDOW:-60}" part_require_node_ssh "case-25.sh" @@ -78,26 +93,28 @@ restore() { trap restore EXIT # --------------------------------------------------------------------------------------------------- -# Make sure the node has a partitioned card, then discover the geometry from the cards themselves. +# One partitioned card, one geometry, one discovery — both halves of this case read it. # --------------------------------------------------------------------------------------------------- part_ensure_partitioned_card read -r SMALL SMALL_CNT MID MID_CNT FULL FULL_CNT NPART NTOT <<<"$(card_profiles)" MIDKEY="$(profile_key "$MID")" FULLKEY="$(profile_key "$FULL")" -echo "[case-25] discovered profiles: SMALL=${SMALL}(${SMALL_CNT}/card) MID=${MID}(${MID_CNT}/card) FULL=${FULL}(${FULL_CNT}/card); partitioned cards=${NPART:-0} of ${NTOT:-0}" -if [ -z "$MIDKEY" ] || [ -z "$FULLKEY" ] || [ "${NPART:-0}" -lt 1 ]; then +SMALLKEY="$(profile_key "$SMALL")" +CEILING="$(partition_ceiling)" +echo "[case-25] discovered profiles: SMALL=${SMALL}(${SMALL_CNT}/card) MID=${MID}(${MID_CNT}/card) FULL=${FULL}(${FULL_CNT}/card); partitioned cards=${NPART:-0} of ${NTOT:-0}, partition ceiling=${CEILING:-0}" +if [ -z "$MIDKEY" ] || [ -z "$FULLKEY" ] || [ -z "$SMALLKEY" ] || [ "${NPART:-0}" -lt 1 ]; then part_skip \ - "The node advertises no usable per-profile pair (MID key='${MIDKEY:-}', FULL key='${FULLKEY:-}'," \ - "partitioned cards=${NPART:-0}). This case needs a partitioned card offering both a 4-memory-slice" \ - "and a whole-card profile." + "The node advertises no usable per-profile keys (MID='${MIDKEY:-}', FULL='${FULLKEY:-}'," \ + "SMALL='${SMALLKEY:-}') over ${NPART:-0} partitioned card(s). This case needs a partitioned card" \ + "offering a 4-memory-slice profile, a whole-card profile and the smallest profile." fi -# The card must start empty, or the baseline arithmetic below is not the empty-card one. -wait_card_idle || echo "[case-25] warning: the card already holds live instances — the baseline may not be the empty-card one" +# The card must start empty, or the baselines below are not the empty-card ones. +wait_card_idle || echo "[case-25] warning: the card already holds live instances — the baselines may not be the empty-card ones" # --------------------------------------------------------------------------------------------------- -# Baseline: an idle partitioned card set advertises its full geometry. +# Half 1 — the per-profile keys are derived from the live ledger, not from a static ceiling. # --------------------------------------------------------------------------------------------------- echo echo "[case-25] === baseline: idle partitioned card(s) ===" @@ -130,67 +147,79 @@ echo echo "[case-25] === carving one ${MID} instance ===" start_node_write_probe CARVE_T0=$(date +%s) -A="${PODPFX}-mid" +A="${LEDGERPFX}-mid" mkpod "$A" "$(partition_reslines "$MIDKEY")" +LEDGER_OK=1 if ! wait_running "$A"; then record FAIL "a mid-size partition admits" "${A} did not reach Running (held: $(held_reason "$A")) — cannot set up the occupancy the rest of the case reads" stop_node_write_probe - part_results "Per-profile capacity is derived from the live ledger, not from a static ceiling" - exit 1 -fi -record PASS "a mid-size partition admits" "${A} Running on card(s) $(wait_pod_cards "$A"), profile recorded '$(pod_profiles "$A")'" - -exp_full=$((want_full - FULL_CNT)) -cur_full=""; cur_mid="" -for _ in $(seq 1 30); do - cur_full="$(node_key "$FULLKEY")" - cur_mid="$(node_key "$MIDKEY")" - [ "${cur_full:-x}" = "$exp_full" ] && break - sleep 3 -done -if [ "${cur_full:-x}" = "$exp_full" ]; then - record PASS "the whole-card profile key falls to what the cards can still host" "${FULLKEY}: ${base_full} → ${cur_full} (want ${exp_full}$([ "$exp_full" = 0 ] && echo ', the single partitioned card is half occupied')) after one ${MID} was carved" -else - record FAIL "the whole-card profile key falls to what the cards can still host" "${FULLKEY}='${cur_full:-}', want ${exp_full} — the key is still reporting a static ceiling the geometry cannot serve" -fi -if [ "${cur_mid:-x}" = "$want_mid" ]; then - record PASS "the same-size profile key still reads allocated + remaining" "${MIDKEY}=${cur_mid} unchanged (1 allocated + $((MID_CNT - 1)) remaining on the carved card) — publishing bare remaining would subtract the running instance twice" + LEDGER_OK=0 else - record FAIL "the same-size profile key still reads allocated + remaining" "${MIDKEY}='${cur_mid:-}', want ${want_mid} — the allocated term is missing, so the scheduler will double-subtract the live instance" + record PASS "a mid-size partition admits" "${A} Running on card(s) $(wait_pod_cards "$A"), profile recorded '$(pod_profiles "$A")'" fi -# With one partitioned card, a whole-card-profile Pod must not be admitted onto this node at all. -if [ "${NPART:-0}" = 1 ]; then - B="${PODPFX}-full" - mkpod "$B" "$(partition_reslines "$FULLKEY")" - hr="$(assert_held "$B")" - if [ -n "$hr" ]; then - record PASS "a whole-card-profile Pod is not placed on the occupied node" "${B} held [${hr}] — the zero-valued key kept the scheduler off this node" +# The rows below read the occupancy this Pod created, so they run only when it admitted. A failed +# carve must not swallow the second half of the case, which starts from its own baseline. +if [ "$LEDGER_OK" = 1 ]; then + exp_full=$((want_full - FULL_CNT)) + cur_full=""; cur_mid="" + for _ in $(seq 1 30); do + cur_full="$(node_key "$FULLKEY")" + cur_mid="$(node_key "$MIDKEY")" + [ "${cur_full:-x}" = "$exp_full" ] && break + sleep 3 + done + if [ "${cur_full:-x}" = "$exp_full" ]; then + record PASS "the whole-card profile key falls to what the cards can still host" "${FULLKEY}: ${base_full} → ${cur_full} (want ${exp_full}$([ "$exp_full" = 0 ] && echo ', the single partitioned card is half occupied')) after one ${MID} was carved" else - record FAIL "a whole-card-profile Pod is not placed on the occupied node" "${B} shows no concrete held signal (phase=$(phase "$B")) — a whole-card profile cannot fit a card that already holds a 4-slice instance" + record FAIL "the whole-card profile key falls to what the cards can still host" "${FULLKEY}='${cur_full:-}', want ${exp_full} — the key is still reporting a static ceiling the geometry cannot serve" + fi + if [ "${cur_mid:-x}" = "$want_mid" ]; then + record PASS "the same-size profile key still reads allocated + remaining" "${MIDKEY}=${cur_mid} unchanged (1 allocated + $((MID_CNT - 1)) remaining on the carved card) — publishing bare remaining would subtract the running instance twice" + else + record FAIL "the same-size profile key still reads allocated + remaining" "${MIDKEY}='${cur_mid:-}', want ${want_mid} — the allocated term is missing, so the scheduler will double-subtract the live instance" fi - delpod "$B" -else - record SKIP "a whole-card-profile Pod is not placed on the occupied node" "${NPART} partitioned cards — a sibling card can legitimately host the whole-card profile, so the node-level refusal is not assertable here" -fi -# --------------------------------------------------------------------------------------------------- -# Free the instance again and close the write window. -# --------------------------------------------------------------------------------------------------- -delpod "$A" -for _ in $(seq 1 30); do - [ "$(node_key "$FULLKEY")" = "$want_full" ] && break - sleep 3 -done -CARVE_T1=$(date +%s) -stop_node_write_probe -CARVE_WRITES="$(node_write_count)" -rm -f "$NODE_WRITE_LOG" -CARVE_SECS=$((CARVE_T1 - CARVE_T0)) -record PASS "OBSERVED: node-status write volume of the ledger-derived key" "idle ${IDLE_WRITES} write(s)/${IDLE_WINDOW}s; one carve+free cycle ${CARVE_WRITES} write(s)/${CARVE_SECS}s (sampled at 1 Hz on the Node object's resourceVersion)" + # With one partitioned card, a whole-card-profile Pod must not be admitted onto this node at all. + if [ "${NPART:-0}" = 1 ]; then + B="${LEDGERPFX}-full" + mkpod "$B" "$(partition_reslines "$FULLKEY")" + hr="$(assert_held "$B")" + if [ -n "$hr" ]; then + record PASS "a whole-card-profile Pod is not placed on the occupied node" "${B} held [${hr}] — the zero-valued key kept the scheduler off this node" + else + record FAIL "a whole-card-profile Pod is not placed on the occupied node" "${B} shows no concrete held signal (phase=$(phase "$B")) — a whole-card profile cannot fit a card that already holds a 4-slice instance" + fi + delpod "$B" + else + record SKIP "a whole-card-profile Pod is not placed on the occupied node" "${NPART} partitioned cards — a sibling card can legitimately host the whole-card profile, so the node-level refusal is not assertable here" + fi + + # ------------------------------------------------------------------------------------------------- + # Free the instance again and close the write window. + # ------------------------------------------------------------------------------------------------- + delpod "$A" + for _ in $(seq 1 30); do + [ "$(node_key "$FULLKEY")" = "$want_full" ] && break + sleep 3 + done + CARVE_T1=$(date +%s) + stop_node_write_probe + CARVE_WRITES="$(node_write_count)" + rm -f "$NODE_WRITE_LOG" + CARVE_SECS=$((CARVE_T1 - CARVE_T0)) + record PASS "OBSERVED: node-status write volume of the ledger-derived key" "idle ${IDLE_WRITES} write(s)/${IDLE_WINDOW}s; one carve+free cycle ${CARVE_WRITES} write(s)/${CARVE_SECS}s (sampled at 1 Hz on the Node object's resourceVersion)" + + echo + echo "---- fold back into the spec: node-status write volume ----" + echo "idle window : ${IDLE_WRITES} Node write(s) over ${IDLE_WINDOW}s with no partition activity" + echo "carve+free window : ${CARVE_WRITES} Node write(s) over ${CARVE_SECS}s covering one allocate and one release" + echo "method : 1 Hz poll of node/${GPU_NODE} .metadata.resourceVersion, counting transitions" + echo "-----------------------------------------------------------" +fi # --------------------------------------------------------------------------------------------------- -# The residual this key shape does not close, DEMONSTRATED rather than asserted: the per-profile keys +# The residual the key shape does not close, DEMONSTRATED rather than asserted: the per-profile keys # are independently subtracted scalars, so two Pods naming mutually exclusive profiles can both be # admitted to a node that can host only one. The containment is that Allocate fails closed and the # admission check retries; the case records what actually happened. @@ -199,7 +228,7 @@ echo echo "[case-25] === observation: mutually exclusive profiles on one card ===" wait_card_idle || true if [ "${NPART:-0}" = 1 ]; then - X="${PODPFX}-x-full"; Y="${PODPFX}-x-mid" + X="${LEDGERPFX}-x-full"; Y="${LEDGERPFX}-x-mid" mkpod "$X" "$(partition_reslines "$FULLKEY")" mkpod "$Y" "$(partition_reslines "$MIDKEY")" xr=1; wait_running "$X" || xr=0 @@ -217,22 +246,131 @@ else record SKIP "OBSERVED: mutually exclusive profiles do not both run" "${NPART} partitioned cards — the two profiles can legitimately land on different cards, so the node-level window is not observable here" fi -part_results "Per-profile capacity is derived from the live ledger, not from a static ceiling" +# --------------------------------------------------------------------------------------------------- +# Half 2 — the token count is a node-level number: allocated + remaining. +# --------------------------------------------------------------------------------------------------- +wait_card_idle || echo "[case-25] warning: the card still holds live instances — the ceiling baseline may be off" + +echo +echo "[case-25] === baseline: idle partitioned card(s) advertise the full ceiling ===" +base_tok="" +for _ in $(seq 1 20); do + base_tok="$(node_key "$PARTITIONED")" + [ "${base_tok:-x}" = "${CEILING}" ] && break + sleep 3 +done +if [ "${base_tok:-x}" = "${CEILING}" ]; then + record PASS "idle node advertises the full partition ceiling" "${PARTITIONED}=${base_tok} = Σ per-card ceiling over ${NPART} partitioned card(s)" +else + record FAIL "idle node advertises the full partition ceiling" "${PARTITIONED}='${base_tok:-}', want ${CEILING} — an empty partitioned card must advertise every slot it could host as Healthy" +fi +# --------------------------------------------------------------------------------------------------- +# Saturate: one whole-card instance per partitioned card leaves no room for any profile. +# --------------------------------------------------------------------------------------------------- echo -echo "---- fold back into the spec: node-status write volume ----" -echo "idle window : ${IDLE_WRITES} Node write(s) over ${IDLE_WINDOW}s with no partition activity" -echo "carve+free window : ${CARVE_WRITES} Node write(s) over ${CARVE_SECS}s covering one allocate and one release" -echo "method : 1 Hz poll of node/${GPU_NODE} .metadata.resourceVersion, counting transitions" -echo "-----------------------------------------------------------" +echo "[case-25] === saturating ${NPART} partitioned card(s) with one ${FULL} each ===" +FILLED=() +SAT_OK=1 +for i in $(seq 1 "$NPART"); do + p="${SATURATEPFX}-full-${i}" + mkpod "$p" "$(partition_reslines "$FULLKEY")" + if ! wait_running "$p"; then + record FAIL "every partitioned card is saturated" "${p} did not reach Running (held: $(held_reason "$p")) — cannot saturate the node" + SAT_OK=0 + break + fi + FILLED+=("$p") +done + +# The rows below read the saturation this loop created, so they run only when every card filled. +if [ "$SAT_OK" = 1 ]; then + record PASS "every partitioned card is saturated" "${#FILLED[@]}× ${FULL} Running, node reports $(node_gi_count) live hardware instance(s)" + + sat_tok="" + for _ in $(seq 1 30); do + sat_tok="$(node_key "$PARTITIONED")" + [ "${sat_tok:-x}" = "${#FILLED[@]}" ] && break + sleep 3 + done + if [ "${sat_tok:-x}" = "${#FILLED[@]}" ]; then + record PASS "a saturated node advertises exactly its live instance count" "${PARTITIONED}=${sat_tok} = ${#FILLED[@]} allocated + 0 remaining; the scheduler subtracts the running Pods' requests, so its free view is zero" + else + record FAIL "a saturated node advertises exactly its live instance count" "${PARTITIONED}='${sat_tok:-}', want ${#FILLED[@]} — allocatable must keep the allocated term, or the scheduler double-subtracts the live instances (too low) / keeps attracting Pods (too high)" + fi + # Capacity is the operator-owned side and must not have collapsed with the free view. + sat_cap="$(node_key_cap "$PARTITIONED")" + if [ -n "$sat_cap" ]; then + record PASS "capacity survives saturation" "${PARTITIONED} capacity=${sat_cap} while allocatable=${sat_tok:-?} — the family is saturated, not absent" + else + record FAIL "capacity survives saturation" "${PARTITIONED} has no capacity entry — a saturated family must keep its keys, or they would be deleted and re-added on every release" + fi + + # ----------------------------------------------------------------------------------------------- + # A further partition Pod is held, and the running ones are untouched. + # ----------------------------------------------------------------------------------------------- + EXTRA="${SATURATEPFX}-extra" + mkpod "$EXTRA" "$(partition_reslines "$SMALLKEY")" + hr="$(assert_held "$EXTRA")" + if [ -n "$hr" ]; then + record PASS "a further partition request is held on a full node" "${EXTRA} held [${hr}] — no room for even the smallest profile" + else + record FAIL "a further partition request is held on a full node" "${EXTRA} shows no concrete held signal (phase=$(phase "$EXTRA")) — the node has no room, so the request must not be admitted onto it" + fi + + stillok=1 + for p in "${FILLED[@]}"; do + running "$p" || stillok=0 + [ "$(pod_mig_devices "$p")" = 1 ] || stillok=0 + done + if [ "$stillok" = 1 ]; then + record PASS "running partitions are unaffected by saturation" "${#FILLED[@]}× ${FULL} still Running, each still holding exactly one hardware instance" + else + record FAIL "running partitions are unaffected by saturation" "at least one saturating Pod is no longer Running or no longer sees exactly one instance — a health recount must never disturb a live allocation" + fi + + # ----------------------------------------------------------------------------------------------- + # Freeing one instance raises the count again, with no restart of anything. + # ----------------------------------------------------------------------------------------------- + echo + echo "[case-25] === freeing one instance restores the count ===" + delpod "$EXTRA" + FREED="${FILLED[0]}" + delpod "$FREED" + freed_tok="" + for _ in $(seq 1 40); do + freed_tok="$(node_key "$PARTITIONED")" + [ -n "$freed_tok" ] && [ "${freed_tok}" -gt "${sat_tok:-0}" ] && break + sleep 3 + done + if [ -n "$freed_tok" ] && [ "${freed_tok}" -gt "${sat_tok:-0}" ]; then + record PASS "freeing an instance raises the healthy count" "${PARTITIONED}: ${sat_tok} → ${freed_tok} after ${FREED} was deleted, with no plugin or kubelet restart" + else + record FAIL "freeing an instance raises the healthy count" "${PARTITIONED} still '${freed_tok:-}' after freeing an instance (was ${sat_tok:-?}) — released room must reappear on the next ListAndWatch cycle" + fi + + REFILL="${SATURATEPFX}-refill" + mkpod "$REFILL" "$(partition_reslines "$SMALLKEY")" + if wait_running "$REFILL"; then + record PASS "the freed room is usable again" "${REFILL} (${SMALL}) Running on the room ${FREED} released" + else + record FAIL "the freed room is usable again" "${REFILL} did not reach Running (held: $(held_reason "$REFILL")) — a released slot must be allocatable without a restart" + fi +fi + +part_results "Per-profile capacity and partition token health are read from the live ledger" if [ "$FAILS" -ne 0 ]; then echo echo "FAILED ${FAILS} check(s). Each per-profile key must publish Σ (allocated + remaining) over the node's" - echo "partitioned cards, so the scheduler's free view is the room the geometry actually has. Diagnose:" + echo "partitioned cards, and the node-level token count must publish allocated + remaining as well —" + echo "never bare remaining and never zero while instances live. Diagnose:" echo " kubectl get devices ${GPU_NODE} -o json | jq '.status.groups[].accelerators[] | {id, allocatedProfiles, remainingProfiles}'" + echo " kubectl get node ${GPU_NODE} -o json | jq '{cap:.status.capacity, alloc:.status.allocatable} | map_values(with_entries(select(.key|test(\"partitioned\"))))'" echo " kubectl get node ${GPU_NODE} -o json | jq '.status.allocatable | with_entries(select(.key|test(\"partitioned\")))'" echo " kubectl -n ${NS} logs deploy/gpustack-operator-worker --tail=200 | grep -i 'node capacity'" + echo " ${MIG_NODE_SSH} sudo nvidia-smi mig -lgi" + echo " kubectl -n ${NS} logs ds/${DM_DS} --tail=200" exit 1 fi echo "CASE 25 PASS" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-26.sh b/.agents/skills/gpustack-operator-e2e/cases/case-26.sh deleted file mode 100755 index 914437d9e..000000000 --- a/.agents/skills/gpustack-operator-e2e/cases/case-26.sh +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/env bash -# -# CASE 26 — Partition token health is a node-level count: allocated + remaining -# (MUTATING, self-recovering; AUTO-SKIPS without a partition-capable node AND a node-SSH address) -# -# MIG_NODE_SSH= case-26.sh -# -# Goal: The partition family's tokens are a fungible count, not a card selection, so their -# health answers only one question: how many more instances can this node host. The node -# advertises allocated + remaining healthy tokens. The allocated term is load-bearing — -# the scheduler subtracts the requests of the Pods already on the node, so publishing -# bare remaining would lose one slot per live instance. The consequence this case proves -# is the one an operator sees: on a node with no room left, allocatable still equals the -# live instance count while the scheduler's FREE view is zero, so a further partition Pod -# is held; the running partitions are untouched; and freeing one raises the count again -# with no restart of anything. -# Environment: A reachable cluster whose active context is the GPU cluster, an nvidia node with at -# least one card that can be put into a hardware partitioning mode, AND SSH to that node -# (sudo nvidia-smi) supplied via MIG_NODE_SSH=. It EXITS 2 (input required) -# when MIG_NODE_SSH is unset, and AUTO-SKIPS (exit 0) when the card reports no -# partitioning mode, when the mode switch does not converge, or when the cards offer no -# whole-card profile to saturate them with. -# Inputs: All real, nothing mocked — the node's token pool and the live geometry are the -# verification. The case partitions card MIG_GPU_INDEX if no card is partitioned yet, then -# saturates every partitioned card with ONE whole-card-profile instance each (the cheapest -# way to leave a card with no room for any profile), submitted sequentially through the -# pool's entrance LocalQueue. Profiles are DISCOVERED from the cards' own capability. -# Survival across a kubelet restart is deliberately NOT exercised here — the interesting -# path is the one taken when the container is STOPPED at restart time, which has its own -# case. -# Expected: - an idle partitioned card set advertises its full partition ceiling healthy; -# - once every partitioned card holds a whole-card instance, allocatable equals the live -# instance count (one per card) — not zero, and not the ceiling; -# - a further partition Pod is HELD, because the scheduler's free view is zero; -# - every already-running partition Pod is still Running and still holds exactly one -# hardware instance; -# - freeing one instance raises allocatable again with no restart, and a fresh partition -# request admits. -# Cleanup: Trap deletes every test Pod, waits for the live instances to reclaim, and restores the -# partitioning mode of the card this case toggled (a card found already partitioned is -# left as found). Idempotent; runs on pass AND fail. -set -uo pipefail - -# Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail -# on transport alone, and a check that takes such a failure for an answer reports a verdict -# about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" - -NS="${1:?usage: MIG_NODE_SSH= case-26.sh }" -CASE_ID=26 -# shellcheck source=/dev/null -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_partition-lib.sh" - -PODPFX=gpustack-e2e-saturate - -part_require_node_ssh "case-26.sh" -part_require_mig_capable -part_discover - -restore() { - echo - echo "[case-26] cleanup: deleting test Pods and restoring card(s) '${TOGGLED:-}'" - delete_test_pods - part_restore_toggled -} -trap restore EXIT - -part_ensure_partitioned_card - -read -r SMALL SMALL_CNT MID MID_CNT FULL FULL_CNT NPART NTOT <<<"$(card_profiles)" -FULLKEY="$(profile_key "$FULL")" -SMALLKEY="$(profile_key "$SMALL")" -CEILING="$(partition_ceiling)" -echo "[case-26] discovered profiles: SMALL=${SMALL}(${SMALL_CNT}/card) FULL=${FULL}(${FULL_CNT}/card); partitioned cards=${NPART:-0}, partition ceiling=${CEILING:-0}" -if [ -z "$FULLKEY" ] || [ -z "$SMALLKEY" ] || [ "${NPART:-0}" -lt 1 ]; then - part_skip \ - "The node advertises no whole-card profile key ('${FULLKEY:-}') or no smallest-profile key" \ - "('${SMALLKEY:-}') over ${NPART:-0} partitioned card(s). This case saturates each card with one" \ - "whole-card instance, so it needs both." -fi - -wait_card_idle || echo "[case-26] warning: the card already holds live instances — the ceiling baseline may be off" - -# --------------------------------------------------------------------------------------------------- -# Baseline — an empty node advertises its whole partition ceiling healthy. -# --------------------------------------------------------------------------------------------------- -echo -echo "[case-26] === baseline: idle partitioned card(s) advertise the full ceiling ===" -base_tok="" -for _ in $(seq 1 20); do - base_tok="$(node_key "$PARTITIONED")" - [ "${base_tok:-x}" = "${CEILING}" ] && break - sleep 3 -done -if [ "${base_tok:-x}" = "${CEILING}" ]; then - record PASS "idle node advertises the full partition ceiling" "${PARTITIONED}=${base_tok} = Σ per-card ceiling over ${NPART} partitioned card(s)" -else - record FAIL "idle node advertises the full partition ceiling" "${PARTITIONED}='${base_tok:-}', want ${CEILING} — an empty partitioned card must advertise every slot it could host as Healthy" -fi - -# --------------------------------------------------------------------------------------------------- -# Saturate: one whole-card instance per partitioned card leaves no room for any profile. -# --------------------------------------------------------------------------------------------------- -echo -echo "[case-26] === saturating ${NPART} partitioned card(s) with one ${FULL} each ===" -FILLED=() -for i in $(seq 1 "$NPART"); do - p="${PODPFX}-full-${i}" - mkpod "$p" "$(partition_reslines "$FULLKEY")" - if ! wait_running "$p"; then - record FAIL "every partitioned card is saturated" "${p} did not reach Running (held: $(held_reason "$p")) — cannot saturate the node" - part_results "Partition token health is a node-level count: allocated + remaining" - exit 1 - fi - FILLED+=("$p") -done -record PASS "every partitioned card is saturated" "${#FILLED[@]}× ${FULL} Running, node reports $(node_gi_count) live hardware instance(s)" - -sat_tok="" -for _ in $(seq 1 30); do - sat_tok="$(node_key "$PARTITIONED")" - [ "${sat_tok:-x}" = "${#FILLED[@]}" ] && break - sleep 3 -done -if [ "${sat_tok:-x}" = "${#FILLED[@]}" ]; then - record PASS "a saturated node advertises exactly its live instance count" "${PARTITIONED}=${sat_tok} = ${#FILLED[@]} allocated + 0 remaining; the scheduler subtracts the running Pods' requests, so its free view is zero" -else - record FAIL "a saturated node advertises exactly its live instance count" "${PARTITIONED}='${sat_tok:-}', want ${#FILLED[@]} — allocatable must keep the allocated term, or the scheduler double-subtracts the live instances (too low) / keeps attracting Pods (too high)" -fi -# Capacity is the operator-owned side and must not have collapsed with the free view. -sat_cap="$(node_key_cap "$PARTITIONED")" -[ -n "$sat_cap" ] \ - && record PASS "capacity survives saturation" "${PARTITIONED} capacity=${sat_cap} while allocatable=${sat_tok:-?} — the family is saturated, not absent" \ - || record FAIL "capacity survives saturation" "${PARTITIONED} has no capacity entry — a saturated family must keep its keys, or they would be deleted and re-added on every release" - -# --------------------------------------------------------------------------------------------------- -# A further partition Pod is held, and the running ones are untouched. -# --------------------------------------------------------------------------------------------------- -EXTRA="${PODPFX}-extra" -mkpod "$EXTRA" "$(partition_reslines "$SMALLKEY")" -hr="$(assert_held "$EXTRA")" -[ -n "$hr" ] \ - && record PASS "a further partition request is held on a full node" "${EXTRA} held [${hr}] — no room for even the smallest profile" \ - || record FAIL "a further partition request is held on a full node" "${EXTRA} shows no concrete held signal (phase=$(phase "$EXTRA")) — the node has no room, so the request must not be admitted onto it" - -stillok=1 -for p in "${FILLED[@]}"; do - running "$p" || stillok=0 - [ "$(pod_mig_devices "$p")" = 1 ] || stillok=0 -done -[ "$stillok" = 1 ] \ - && record PASS "running partitions are unaffected by saturation" "${#FILLED[@]}× ${FULL} still Running, each still holding exactly one hardware instance" \ - || record FAIL "running partitions are unaffected by saturation" "at least one saturating Pod is no longer Running or no longer sees exactly one instance — a health recount must never disturb a live allocation" - -# --------------------------------------------------------------------------------------------------- -# Freeing one instance raises the count again, with no restart of anything. -# --------------------------------------------------------------------------------------------------- -echo -echo "[case-26] === freeing one instance restores the count ===" -delpod "$EXTRA" -FREED="${FILLED[0]}" -delpod "$FREED" -freed_tok="" -for _ in $(seq 1 40); do - freed_tok="$(node_key "$PARTITIONED")" - [ -n "$freed_tok" ] && [ "${freed_tok}" -gt "${sat_tok:-0}" ] && break - sleep 3 -done -if [ -n "$freed_tok" ] && [ "${freed_tok}" -gt "${sat_tok:-0}" ]; then - record PASS "freeing an instance raises the healthy count" "${PARTITIONED}: ${sat_tok} → ${freed_tok} after ${FREED} was deleted, with no plugin or kubelet restart" -else - record FAIL "freeing an instance raises the healthy count" "${PARTITIONED} still '${freed_tok:-}' after freeing an instance (was ${sat_tok:-?}) — released room must reappear on the next ListAndWatch cycle" -fi - -REFILL="${PODPFX}-refill" -mkpod "$REFILL" "$(partition_reslines "$SMALLKEY")" -if wait_running "$REFILL"; then - record PASS "the freed room is usable again" "${REFILL} (${SMALL}) Running on the room ${FREED} released" -else - record FAIL "the freed room is usable again" "${REFILL} did not reach Running (held: $(held_reason "$REFILL")) — a released slot must be allocatable without a restart" -fi - -part_results "Partition token health is a node-level count: allocated + remaining" - -if [ "$FAILS" -ne 0 ]; then - echo - echo "FAILED ${FAILS} check(s). A node advertises Σ (allocated + remaining) healthy partition tokens, never" - echo "bare remaining and never zero while instances live. Diagnose:" - echo " kubectl get node ${GPU_NODE} -o json | jq '{cap:.status.capacity, alloc:.status.allocatable} | map_values(with_entries(select(.key|test(\"partitioned\"))))'" - echo " kubectl get devices ${GPU_NODE} -o json | jq '.status.groups[].accelerators[] | {id, allocatedProfiles, remainingProfiles}'" - echo " ${MIG_NODE_SSH} sudo nvidia-smi mig -lgi" - echo " kubectl -n ${NS} logs ds/${DM_DS} --tail=200" - exit 1 -fi -echo "CASE 26 PASS" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-27.sh b/.agents/skills/gpustack-operator-e2e/cases/case-27.sh index 9b8f610ea..2af9ef258 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-27.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-27.sh @@ -39,8 +39,8 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: MIG_NODE_SSH= case-27.sh }" CASE_ID=27 diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-28.sh b/.agents/skills/gpustack-operator-e2e/cases/case-28.sh index 974bae81b..dd6b4dac9 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-28.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-28.sh @@ -49,13 +49,15 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: MIG_NODE_SSH= case-28.sh }" CASE_ID=28 # shellcheck source=/dev/null . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_partition-lib.sh" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_ssh-lib.sh" INST=gpustack-e2e-part-ssh SECRET=gpustack-e2e-part-ssh-key @@ -86,9 +88,7 @@ echo "[case-28] requesting partition profile ${MID} on ${IT}" # --------------------------------------------------------------------------------------------------- # One SSH-enabled Instance whose accelerator request is a hardware partition. # --------------------------------------------------------------------------------------------------- -ssh-keygen -t ed25519 -f "$KEYDIR/id" -N "" -q -kubectl -n default delete secret "$SECRET" --ignore-not-found >/dev/null 2>&1 -kubectl -n default create secret generic "$SECRET" --from-file=authorized_keys="$KEYDIR/id.pub" >/dev/null +ssh_setup_key_secret default "$SECRET" "$KEYDIR/id" cat </dev/null apiVersion: worker.gpustack.ai/v1alpha1 kind: Instance diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-29.sh b/.agents/skills/gpustack-operator-e2e/cases/case-29.sh index 6589a2227..ef07205ed 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-29.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-29.sh @@ -1,55 +1,122 @@ #!/usr/bin/env bash # -# CASE 29 — Two concurrent requests for different profiles each get their own instance -# (MUTATING, self-recovering; AUTO-SKIPS without a partition-capable node AND a node-SSH address) +# CASE 29 — Two partition shapes at once, a replacement inside the reclaim window, and a terminated +# container's charge (MUTATING, self-recovering; AUTO-SKIPS without a partition-capable +# node AND a node-SSH address) # # MIG_NODE_SSH= case-29.sh # -# Goal: The device-plugin allocation RPC carries a resource name and a device count and nothing -# else — no profile, no Pod identity — so when two Pods are admitted together, both -# carrying the same bare partition key and differing only in the profile they name, the -# plugin must still actuate each one's OWN profile. Getting this wrong is silent rather -# than loud: the wrong hardware shape is carved and both Pods start, so nothing fails -# until a tenant notices the instance is not the size they asked for. This case submits -# the two requests simultaneously and checks the shape each Pod actually received, from -# two independent sources: the hardware itself, through the device list inside the -# container, and the durable per-Pod allocation record the plugin wrote. It also checks -# that neither Pod's record was overwritten with the other's — the record is keyed by -# container, and a second allocation must accumulate rather than erase. +# Goal: Three contracts of the hardware-partition path, read on one card in one run, in the +# order they can be read on. +# +# ONE: TWO REQUESTS THAT DIFFER ONLY IN THE PROFILE THEY NAME. The device-plugin +# allocation RPC carries a resource name and a device count and nothing else -- no +# profile, no Pod identity -- so when two Pods are admitted together, both carrying +# the same bare partition key and differing only in the profile they name, the plugin +# must still actuate each one's OWN profile. Getting this wrong is silent rather than +# loud: the wrong hardware shape is carved and both Pods start, so nothing fails until +# a tenant notices the instance is not the size they asked for. The shape each Pod +# actually received is read from two independent sources -- the hardware itself, +# through the device list inside the container, and the durable per-Pod allocation +# record the plugin wrote -- and neither Pod's record may be overwritten with the +# other's, because the record is keyed by container and a second allocation must +# accumulate rather than erase. +# +# TWO: THE GAP BETWEEN THE ACCOUNTING FREEING A PARTITION AND THE HARDWARE RELEASING +# IT, MEASURED RATHER THAN ASSUMED AWAY. Node-level occupancy is rebuilt from the Pod +# annotations, so a deleted Pod's slot reappears in the per-profile key the moment the +# Pod is gone -- while the reclaimer only destroys the instance after several +# consecutive absent sightings on its resync cadence. A replacement scheduled inside +# that window meets an instance that still exists, and what it then gets is MEASURED +# here rather than assumed: the allocation may be refused, or it may be granted that +# very instance which the reclaimer then destroys underneath the starting container. +# Both cost a retry, they are told apart by WHERE the attempt died, and only the first +# raises an UnexpectedAdmissionError -- the second leaves that count at zero. So the +# shape of every failed attempt is recorded, not just how many there were. The +# containment is that the request is retried and the window closes on its own; the +# failure mode worth catching is a request that NEVER converges. +# +# THREE: A TERMINATED INIT CONTAINER STILL CHARGES THE CARD ITS INSTANCE OCCUPIES. A +# device is scoped to the Pod's life, not the container's. The kubelet keeps a finished +# init container's devices in its record for as long as the Pod exists, and the +# operator's reclaimer destroys a hardware partition on Pod deletion -- not on +# container termination. So a Pod whose init container took a partition and then +# exited, with the Pod still running, is STILL occupying that hardware. The +# accounting has to say so: the node's per-profile capacity must keep charging the +# instance, or the node advertises room it does not have and the next tenant is +# admitted onto a card that cannot take it. Only hardware shows this -- a ledger +# fixture passes whether or not the entry survives the container -- so the live +# instance is read from the card, the node key is read, and a further Pod is admitted +# against the room the node claims is left, requiring it to actually start. +# # Environment: A reachable cluster whose active context is the GPU cluster, an nvidia node with at # least one card that can be put into a hardware partitioning mode, AND SSH to that node # (sudo nvidia-smi) supplied via MIG_NODE_SSH=. It EXITS 2 (input required) # when MIG_NODE_SSH is unset, and AUTO-SKIPS (exit 0) when the card reports no -# partitioning mode, the mode switch does not converge, or the cards offer fewer than two -# distinct profiles that can coexist. -# Inputs: All real, nothing mocked — the actuated hardware shapes are the verification. Two Pods -# are created back to back with no wait between them, one naming the DISCOVERED -# 4-memory-slice profile and one the smallest profile, both through the pool's entrance -# LocalQueue. The two shapes are chosen so they fit side by side on one card, which is -# what makes the resolution — rather than the geometry — the thing under test. -# Expected: - both Pods reach Running; -# - the device list inside each Pod names the profile that Pod requested, and exactly one -# instance; -# - each Pod's durable allocation record names its own profile and not the other's; -# - neither Pod records an UnexpectedAdmissionError. -# Cleanup: Trap deletes every test Pod, waits for the instances to reclaim, and restores the -# partitioning mode of the card this case toggled (a card found already partitioned is -# left as found). Idempotent; runs on pass AND fail. +# partitioning mode, the mode switch does not converge, or the node advertises no +# 4-memory-slice profile key -- the shape every section below places. +# +# The two remaining sections carry their own shape requirement and are SKIPPED ON +# THEIR OWN when the node does not advertise it, loudly and in their own banner, so +# that one missing shape never costs the other sections their rows: the two-shape +# section needs a smallest-profile key distinct from the 4-memory-slice one, and the +# terminated-container section needs a whole-card profile key to read the charge +# against. +# +# Inputs: All real, nothing mocked -- the actuated hardware shapes are the verification. Every +# Pod is created through the pool's entrance LocalQueue and pinned to the accelerator +# node: two submitted back to back with no wait between them, one naming the +# DISCOVERED 4-memory-slice profile and one the smallest profile, chosen so they fit +# side by side on one card, which is what makes the resolution rather than the +# geometry the thing under test; then one partition Pod that is deleted and +# immediately re-requested, the replacement retried the way a controller would retry +# it; then one Pod whose INIT container requests the 4-memory-slice partition and +# exits after a few seconds while its app container holds no accelerator at all (the +# request rules allow accelerator claims in exactly one container group, and this Pod +# puts them in the init group), followed by a second, ordinary partition Pod of the +# same profile, to consume the room the node still reports. +# +# Expected: - both of the first two Pods reach Running, each device list names the profile that +# Pod requested and shows exactly one instance, and each Pod's durable allocation +# record names its own profile and not the other's; +# - neither of those Pods records an UnexpectedAdmissionError; +# - the first partition of the reclaim section runs and is deleted, and a replacement +# eventually runs within MIG_RECLAIM_BOUND seconds (default 300) -- that is the +# containment claim, and the only hard assertion in that section; +# - the attempt count, the elapsed seconds and any terminal allocation failures along +# the way are RECORDED as the measurement of the window, not asserted against a +# threshold; +# - the init-container Pod reaches Running with its init container Terminated, the +# card still holds a live hardware instance, the whole-card profile key still reads +# one card's worth less, and the follow-up partition Pod, admitted against the +# remaining reported room, reaches Running without a single +# UnexpectedAdmissionError. +# +# Cleanup: One trap deletes every Pod this case created, waits for the instances to reclaim, +# and restores the partitioning mode of the card this case toggled (a card found +# already partitioned is left as found). Idempotent; runs on pass AND fail. set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: MIG_NODE_SSH= case-29.sh }" CASE_ID=29 # shellcheck source=/dev/null . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_partition-lib.sh" +# One prefix per section, so every Pod carries the name it carried in the case it came from. PODPFX=gpustack-e2e-concprof +# How long the reclaim window is given to close on its own. +BOUND="${MIG_RECLAIM_BOUND:-300}" + +# The sections a cluster did not give the shape for, named for the footer. +SKIPPED="" + part_require_node_ssh "case-29.sh" part_require_mig_capable part_discover @@ -64,77 +131,333 @@ trap restore EXIT part_ensure_partitioned_card +# ONE discovery for the whole case: the card geometry and the three profile keys every section +# below places against. read -r SMALL SMALL_CNT MID MID_CNT FULL FULL_CNT NPART NTOT <<<"$(card_profiles)" MIDKEY="$(profile_key "$MID")" SMALLKEY="$(profile_key "$SMALL")" -echo "[case-29] discovered profiles: SMALL=${SMALL} MID=${MID}; partitioned cards=${NPART:-0}" -if [ -z "$MIDKEY" ] || [ -z "$SMALLKEY" ] || [ "$SMALL" = "$MID" ]; then +FULLKEY="$(profile_key "$FULL")" +echo "[case-29] discovered profiles: SMALL=${SMALL}(${SMALL_CNT}/card) MID=${MID}(${MID_CNT}/card) FULL=${FULL}(${FULL_CNT}/card); partitioned cards=${NPART:-0}" + +# The 4-memory-slice shape is the one every section places, so its absence skips the whole case. +if [ -z "$MIDKEY" ]; then part_skip \ - "The node does not advertise two distinct coexisting profile keys (SMALL='${SMALLKEY:-}'," \ - "MID='${MIDKEY:-}'). This case needs two different shapes that fit one card side by side." + "The node advertises no 4-memory-slice profile key ('${MIDKEY:-}') — nothing to partition with." \ + "This case needs a card that offers the 4-memory-slice hardware partition profile." fi + wait_card_idle || echo "[case-29] warning: the card already holds live instances" # --------------------------------------------------------------------------------------------------- -# Submit both at once — no wait between them, so the two allocations are in flight together. +# Section 1 — two simultaneous requests for DIFFERENT profiles. +# +# The allocation RPC carries no profile and no Pod identity, so two Pods that differ only in the +# profile they name must still each be given their own. Each shape is read from the hardware and +# from the durable per-Pod record. # --------------------------------------------------------------------------------------------------- echo -echo "[case-29] === two simultaneous requests: ${MID} and ${SMALL} ===" +echo "[case-29] === section 1: two simultaneous requests: ${MID} and ${SMALL} ===" PM="${PODPFX}-mid" PS="${PODPFX}-small" -mkpod "$PM" "$(partition_reslines "$MIDKEY")" -mkpod "$PS" "$(partition_reslines "$SMALLKEY")" - -mrun=1; wait_running "$PM" || mrun=0 -srun=1; wait_running "$PS" || srun=0 -if [ "$mrun" = 1 ] && [ "$srun" = 1 ]; then - record PASS "both distinct-profile requests run" "${PM} (${MID}) and ${PS} (${SMALL}) both Running; node reports $(node_gi_count) live instance(s)" +if [ -z "$SMALLKEY" ] || [ "$SMALL" = "$MID" ]; then + SKIPPED="${SKIPPED}the two-shape section; " + echo + echo "== SECTION 1 — SKIPPED ==" + echo "The node does not advertise two distinct coexisting profile keys (SMALL='${SMALLKEY:-}'," \ + "MID='${MIDKEY:-}'). This section needs two different shapes that fit one card side by side." else - record FAIL "both distinct-profile requests run" "${PM} running=${mrun} (held: $(held_reason "$PM")), ${PS} running=${srun} (held: $(held_reason "$PS")) — both shapes fit one card, so both must be placed" + mkpod "$PM" "$(partition_reslines "$MIDKEY")" + mkpod "$PS" "$(partition_reslines "$SMALLKEY")" + + mrun=1; wait_running "$PM" || mrun=0 + srun=1; wait_running "$PS" || srun=0 + if [ "$mrun" = 1 ] && [ "$srun" = 1 ]; then + record PASS "both distinct-profile requests run" "${PM} (${MID}) and ${PS} (${SMALL}) both Running; node reports $(node_gi_count) live instance(s)" + else + record FAIL "both distinct-profile requests run" "${PM} running=${mrun} (held: $(held_reason "$PM")), ${PS} running=${srun} (held: $(held_reason "$PS")) — both shapes fit one card, so both must be placed" + fi + + # What each Pod actually got, from the hardware and from the durable record. + if [ "$mrun" = 1 ]; then + smi="$(kubectl -n default exec "$PM" -- nvidia-smi -L 2>/dev/null)" + devs="$(printf '%s\n' "$smi" | grep -c 'MIG')" + rec="$(pod_profiles "$PM")" + if printf '%s\n' "$smi" | grep -q "$MID" && [ "${devs:-0}" = 1 ]; then + record PASS "${PM} was given the ${MID} it asked for" "device list names ${MID}, exactly ${devs} instance visible" + else + record FAIL "${PM} was given the ${MID} it asked for" "device list = '$(printf '%s' "$smi" | tr '\n' ';')' (want one ${MID} instance) — a request was resolved to the wrong container's profile" + fi + if [ "$rec" = "$MID" ]; then + record PASS "${PM}'s durable record names its own profile" "recorded profile '${rec}', with no trace of ${SMALL}" + else + record FAIL "${PM}'s durable record names its own profile" "recorded profile '${rec:-}', want exactly '${MID}' — the per-container allocation record was cross-patched or lost" + fi + fi + if [ "$srun" = 1 ]; then + smi="$(kubectl -n default exec "$PS" -- nvidia-smi -L 2>/dev/null)" + devs="$(printf '%s\n' "$smi" | grep -c 'MIG')" + rec="$(pod_profiles "$PS")" + if printf '%s\n' "$smi" | grep -q "$SMALL" && [ "${devs:-0}" = 1 ]; then + record PASS "${PS} was given the ${SMALL} it asked for" "device list names ${SMALL}, exactly ${devs} instance visible" + else + record FAIL "${PS} was given the ${SMALL} it asked for" "device list = '$(printf '%s' "$smi" | tr '\n' ';')' (want one ${SMALL} instance) — a request was resolved to the wrong container's profile" + fi + if [ "$rec" = "$SMALL" ]; then + record PASS "${PS}'s durable record names its own profile" "recorded profile '${rec}', with no trace of ${MID}" + else + record FAIL "${PS}'s durable record names its own profile" "recorded profile '${rec:-}', want exactly '${SMALL}' — the per-container allocation record was cross-patched or lost" + fi + fi + + uae=0 + unreadable=0 + for p in "$PM" "$PS"; do + n="$(pod_unexpected_admission "$p")" + case "$n" in + '') ;; + # "?" is the answer when the events cannot be read. It is not a count, so it is not added to + # one -- an unreadable answer is no admission failure seen, and the row below says so. + *[!0-9]*) unreadable=$((unreadable + 1)) ;; + *) uae=$((uae + n)) ;; + esac + done + if [ "$uae" = 0 ] && [ "$unreadable" = 0 ]; then + record PASS "no terminal admission failure" "0 UnexpectedAdmissionError events across both Pods" + else + record FAIL "no terminal admission failure" "${uae} UnexpectedAdmissionError event(s)${unreadable:+, ${unreadable} Pod(s) whose events could not be read} — a profile the card could not host was attempted" + fi + + # The two instances go before the next section places its own: the card is the same one, and + # the reclaim window the next section measures starts from a card that is idle. + delpod "$PM" + delpod "$PS" + wait_card_idle || echo "[case-29] warning: the card still holds live instances after the two-shape section" fi # --------------------------------------------------------------------------------------------------- -# What each Pod actually got, from the hardware and from the durable record. +# Section 2 — the reclaim window: a same-profile replacement scheduled right after its predecessor +# was deleted. +# +# The accounting frees the partition on Pod deletion; the hardware releases it a few resync passes +# later. A replacement inside that window is retried, and the window must close on its own. # --------------------------------------------------------------------------------------------------- -check_one() { - local pod="$1" want="$2" other="$3" smi devs rec - smi="$(kubectl -n default exec "$pod" -- nvidia-smi -L 2>/dev/null)" - devs="$(printf '%s\n' "$smi" | grep -c 'MIG')" - rec="$(pod_profiles "$pod")" - if printf '%s\n' "$smi" | grep -q "$want" && [ "${devs:-0}" = 1 ]; then - record PASS "${pod} was given the ${want} it asked for" "device list names ${want}, exactly ${devs} instance visible" +echo +echo "[case-29] === section 2: one ${MID} partition, deleted, then re-requested at once ===" +PODPFX=gpustack-e2e-reclaimwin +A="${PODPFX}-first" +mkpod "$A" "$(partition_reslines "$MIDKEY")" +arun=0 +if wait_running "$A"; then + arun=1 + GI_BEFORE="$(node_gi_count)" + record PASS "the first partition runs" "${A} Running on card(s) $(pod_cards "$A"); ${GI_BEFORE} live hardware instance(s)" +else + record FAIL "the first partition runs" "${A} did not reach Running (held: $(held_reason "$A")) — cannot set up the reclaim window" + record SKIP "the replacement converges without intervention" "not attempted: the predecessor never reached Running" + record SKIP "OBSERVED: cost of the reclaim window" "not attempted: the predecessor never reached Running" +fi + +if [ "$arun" = 1 ]; then + delpod "$A" + T0="$(date +%s)" + ATTEMPTS=0 + UAE=0 + # Where the failed attempts died, tallied separately from UAE. A grant that is destroyed under the + # starting container raises no admission event, so UAE stays 0 through exactly the failure this window + # is most likely to produce; without this tally the run reads as clean. + SHAPE_ADMISSION=0 + SHAPE_START=0 + SHAPE_OTHER=0 + OK=0 + WINNER="" + while [ "$(( $(date +%s) - T0 ))" -lt "$BOUND" ]; do + ATTEMPTS=$((ATTEMPTS + 1)) + R="${PODPFX}-repl-${ATTEMPTS}" + mkpod "$R" "$(partition_reslines "$MIDKEY")" + for _ in $(seq 1 20); do + ph="$(phase "$R")" + case "$ph" in + Running) OK=1; WINNER="$R"; break ;; + Failed) break ;; + esac + sleep 3 + done + n="$(pod_unexpected_admission "$R")" + case "$n" in + '') ;; + # "?" is the answer when the events cannot be read. It is not a count, so it is not added to + # one -- the tally below then reports the events it could not read as neither. + *[!0-9]*) ;; + *) UAE=$((UAE + n)) ;; + esac + [ "$OK" = 1 ] && break + # Read the shape ONCE: the Pod is live and a second read can answer differently. + shape="$(pod_failure_shape "$R")" + case "$shape" in + admission) SHAPE_ADMISSION=$((SHAPE_ADMISSION + 1)) ;; + start) SHAPE_START=$((SHAPE_START + 1)) ;; + *) SHAPE_OTHER=$((SHAPE_OTHER + 1)) ;; + esac + echo "[case-29] attempt ${ATTEMPTS} did not run (phase=$(phase "$R"), where=${shape}, held: $(held_reason "$R")) — retrying" + delpod "$R" + sleep 5 + done + T1="$(date +%s)" + ELAPSED=$((T1 - T0)) + + if [ "$OK" = 1 ]; then + record PASS "the replacement converges without intervention" "${WINNER} Running ${ELAPSED}s after the predecessor was deleted, on attempt ${ATTEMPTS} of at most ${BOUND}s" else - record FAIL "${pod} was given the ${want} it asked for" "device list = '$(printf '%s' "$smi" | tr '\n' ';')' (want one ${want} instance) — a request was resolved to the wrong container's profile" + record FAIL "the replacement converges without intervention" "no same-profile replacement reached Running within ${BOUND}s over ${ATTEMPTS} attempt(s) — the window is supposed to close on its own" fi - if [ "$rec" = "$want" ]; then - record PASS "${pod}'s durable record names its own profile" "recorded profile '${rec}', with no trace of ${other}" + record PASS "OBSERVED: cost of the reclaim window" "attempts=${ATTEMPTS}, elapsed=${ELAPSED}s, refused before actuating=${SHAPE_ADMISSION} (UnexpectedAdmissionError events=${UAE}), granted then destroyed under the container=${SHAPE_START}, other=${SHAPE_OTHER} (a first attempt that runs immediately means the window was already closed on this cadence; a non-zero 'granted then destroyed' means the window handed out a partition it went on to reclaim, which no admission counter records)" + + echo + echo "---- fold back into the spec: reclaim-window replacement ----" + echo "profile : ${MID}" + echo "attempts : ${ATTEMPTS}" + echo "elapsed : ${ELAPSED}s (bound ${BOUND}s)" + echo "refused : ${SHAPE_ADMISSION} attempt(s) never actuated (${UAE} UnexpectedAdmissionError event(s))" + echo "granted then lost: ${SHAPE_START} attempt(s) were given a partition no container could then start on" + if [ "$OK" = 1 ]; then + echo "converged : yes" else - record FAIL "${pod}'s durable record names its own profile" "recorded profile '${rec:-}', want exactly '${want}' — the per-container allocation record was cross-patched or lost" + echo "converged : NO" fi -} -[ "$mrun" = 1 ] && check_one "$PM" "$MID" "$SMALL" -[ "$srun" = 1 ] && check_one "$PS" "$SMALL" "$MID" + echo "-------------------------------------------------------------" -uae=0 -for p in "$PM" "$PS"; do - n="$(pod_unexpected_admission "$p")" - [ -n "$n" ] && uae=$((uae + n)) -done -[ "$uae" = 0 ] \ - && record PASS "no terminal admission failure" "0 UnexpectedAdmissionError events across both Pods" \ - || record FAIL "no terminal admission failure" "${uae} UnexpectedAdmissionError event(s) — a profile the card could not host was attempted" + # The replacement holds its instance until it is released, and the section below reads its + # baseline off an idle card. + if [ -n "$WINNER" ]; then + delpod "$WINNER" + fi + wait_card_idle || echo "[case-29] warning: the card still holds live instances after the reclaim-window section" +fi -part_results "Two concurrent requests for different profiles each get their own instance" +# --------------------------------------------------------------------------------------------------- +# Section 3 — a terminated init container still charges the card its instance occupies. +# +# The device is scoped to the Pod's life, not the container's, so the per-profile capacity must +# keep charging an instance whose container is long gone -- and the room the node still reports +# must be room the card really has. +# --------------------------------------------------------------------------------------------------- +echo +echo "[case-29] === section 3: init container takes a ${MID} partition, then terminates ===" +PODPFX=gpustack-e2e-initterm +P="${PODPFX}-pod" +if [ -z "$FULLKEY" ]; then + SKIPPED="${SKIPPED}the terminated-container section; " + echo + echo "== SECTION 3 — SKIPPED ==" + echo "The node advertises no 4-memory-slice key ('${MIDKEY:-}') or no whole-card key" \ + "('${FULLKEY:-}'). This section reads the whole-card key to see the init container's charge." +else + wait_card_idle || echo "[case-29] warning: the card already holds live instances" + BASE_FULL="$(node_key "$FULLKEY")" + echo "[case-29] baseline ${FULLKEY}=${BASE_FULL:-}" + + TESTPODS+=("$P") + cat </dev/null +apiVersion: v1 +kind: Pod +metadata: { name: ${P}, namespace: default, labels: { kueue.x-k8s.io/queue-name: ${LQ} } } +spec: + schedulerName: default-scheduler + ${RTC_LINE} + restartPolicy: Never + nodeSelector: { kubernetes.io/hostname: ${GPU_NODE} } + initContainers: + - name: warmup + image: ${IMAGE} + command: ["sh", "-c", "nvidia-smi -L || true; sleep 15"] + resources: + limits: +$(partition_reslines "$MIDKEY") + requests: +$(partition_reslines "$MIDKEY") + containers: + - name: main + image: ${IMAGE} + command: ["sleep", "86400"] + resources: + limits: { cpu: "200m", memory: 256Mi } + requests: { cpu: "200m", memory: 256Mi } +EOF + + prun=0 + if wait_running "$P" 60; then + prun=1 + initstate="$(kubectl -n default get pod "$P" -o jsonpath='{.status.initContainerStatuses[0].state.terminated.reason}' 2>/dev/null)" + if [ -n "$initstate" ]; then + record PASS "the Pod runs on past its init container" "${P} Running with init container Terminated/${initstate}, card(s) $(pod_cards "$P"), profile '$(pod_profiles "$P")'" + else + record FAIL "the Pod runs on past its init container" "${P} is Running but its init container reports no terminated state — cannot exercise the terminated-container charge" + fi + else + record FAIL "the Pod runs on past its init container" "${P} did not reach Running (phase=$(phase "$P"), held: $(held_reason "$P")) — an init-group accelerator claim must be admitted" + record SKIP "the hardware instance outlives the container" "not attempted: the init-group Pod never reached Running" + record SKIP "the terminated container's instance is still charged" "not attempted: the init-group Pod never reached Running" + record SKIP "the reported free room is real" "not attempted: the init-group Pod never reached Running" + fi + + # The hardware is still occupied, and the accounting says so. + if [ "$prun" = 1 ]; then + gi="$(node_gi_count)" + if [ "${gi:-0}" -ge 1 ]; then + record PASS "the hardware instance outlives the container" "${gi} live instance(s) on the card while the init container is long gone — the reclaimer destroys on Pod deletion, not container exit" + else + record FAIL "the hardware instance outlives the container" "the card reports ${gi:-0} live instance(s) — the instance was destroyed while its Pod is still alive" + fi + + exp_full=$((BASE_FULL - FULL_CNT)) + cur_full="" + for _ in $(seq 1 30); do + cur_full="$(node_key "$FULLKEY")" + [ "${cur_full:-x}" = "$exp_full" ] && break + sleep 3 + done + if [ "${cur_full:-x}" = "$exp_full" ]; then + record PASS "the terminated container's instance is still charged" "${FULLKEY}: ${BASE_FULL} → ${cur_full} (want ${exp_full}) — the occupied card can no longer host a whole-card profile" + else + record FAIL "the terminated container's instance is still charged" "${FULLKEY}='${cur_full:-}', want ${exp_full} — the ledger dropped a terminated container's entry and is advertising room the card does not have" + fi + + # Admit against the room the node still claims: it must be real room. + echo + echo "[case-29] === section 3: a further request against the reported free room ===" + Q="${PODPFX}-next" + mkpod "$Q" "$(partition_reslines "$MIDKEY")" + if wait_running "$Q"; then + uae="$(pod_unexpected_admission "$Q")" + if [ -z "$uae" ]; then + record PASS "the reported free room is real" "${Q} (${MID}) Running beside the init container's instance, 0 UnexpectedAdmissionError" + else + record FAIL "the reported free room is real" "${Q} Running but recorded ${uae} UnexpectedAdmissionError event(s) — the node advertised room the card could not place" + fi + else + record FAIL "the reported free room is real" "${Q} did not reach Running (held: $(held_reason "$Q")) — the node's remaining per-profile room must be placeable" + fi + fi +fi + +part_results "Two partition shapes at once, a replacement inside the reclaim window, and a terminated container's charge" if [ "$FAILS" -ne 0 ]; then echo - echo "FAILED ${FAILS} check(s). Two Pods that differ only in the profile they name must each have their own" - echo "shape actuated: the plugin narrows the candidate containers by what the offered cards can serve, and" - echo "records each container's allocation under its own key. Diagnose:" + echo "FAILED ${FAILS} check(s). Three contracts are read here, and the failure text above names which" + echo "one broke. A shape resolved to the wrong container's profile is a resolution bug; a replacement" + echo "that never converges is a reclaim-window bug; a terminated container whose instance is no longer" + echo "charged is an accounting bug. Diagnose:" echo " kubectl -n default get pod ${PM} -o jsonpath='{.metadata.annotations.${ANNO}}'; echo" echo " kubectl -n default get pod ${PS} -o jsonpath='{.metadata.annotations.${ANNO}}'; echo" + echo " kubectl -n default get pod ${P} -o jsonpath='{.metadata.annotations.${ANNO}}'; echo" + echo " kubectl get devices ${GPU_NODE} -o json | jq '.status.groups[].accelerators[] | {id, allocatedProfiles, remainingProfiles}'" echo " ${MIG_NODE_SSH} sudo nvidia-smi mig -lgi" - echo " kubectl -n ${NS} logs ds/${DM_DS} --tail=200" + echo " kubectl -n ${NS} logs ds/${DM_DS} --tail=300 | grep -i reclaim" exit 1 fi +if [ -n "$SKIPPED" ]; then + echo + echo "SKIPPED section(s): ${SKIPPED}the run above does not cover them." +fi echo "CASE 29 PASS" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-3.sh b/.agents/skills/gpustack-operator-e2e/cases/case-3.sh deleted file mode 100755 index ba71c8a2d..000000000 --- a/.agents/skills/gpustack-operator-e2e/cases/case-3.sh +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env bash -# -# CASE 3 — Managed-toggle scopes node onboarding (MUTATING, self-recovering) -# -# case-3.sh -# -# Goal: De-managing every node feeding a pool DELETES its ResourceFlavor but keeps the -# InstanceType alive with its backing ClusterQueue's resource groups emptied -# (drain-when-no-flavors), and restoring the nodes refills the queue and reactivates -# the type. Under the queue-ownership split, NodeFlavorReconciler drops the flavor and -# NodeQueueReconciler empties the quota — the InstanceType is never deleted for lack of -# flavors. An idle pool has no reservations to drain, yet its queue is still put on -# HoldAndDrain before the quota is emptied and stays held while empty; the returning -# flavor is written into the held queue before its StopPolicy is restored, and identity -# is kept. -# Environment: A real cluster with a materialized general pool and a CONTINUOUSLY RUNNING operator -# (must NOT be restarted between toggle and assertion). No GPU. Targets the general -# (CPU) pool fed by every managed node — same on a 1-node or an N-node cluster. -# Inputs: All real, nothing mocked — toggles gpustack.ai/managed=false, then back to true, on -# every -gpustack-worker NodeFeature (toggle via the NodeFeature, not the node -# directly: NFD reverts a direct node label). -# Expected: - A — the pool's general ResourceFlavor is DELETED (no drain tombstone); -# - B — the InstanceType SURVIVES and its ClusterQueue's spec.resourceGroups is emptied -# (length 0) while the queue stays held (spec.stopPolicy is not None); -# - C — restoring the nodes refills resourceGroups and the InstanceType returns to Active. -# Cleanup: Trap restores gpustack.ai/managed=true on all nodes and waits for the InstanceType to -# return to Active so a following case still finds a healthy chain. -set -uo pipefail - -# Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail -# on transport alone, and a check that takes such a failure for an answer reports a verdict -# about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" - -NS="${1:?usage: case-3.sh }" -# Target the general (CPU) pool: non-accelerated (no GPU needed) and fed by every managed node, so -# the case behaves the same on a 1-node local cluster and an N-node real one. -IT=$(kubectl get instancetypes.worker.gpustack.ai \ - -o jsonpath='{.items[?(@.spec.acceleratable==false)].metadata.name}' 2>/dev/null | tr ' ' '\n' | grep -m1 'gpustack-') -# One of the pool's CPU ResourceFlavors (named gpustack--${gKey}-${os}-${arch}-${count}c, plus a -# -fnv64-<16 hex> topology profile on a TAS cluster — keyed by -# the node's real CPU, so its name is independent of the collapsed InstanceType name), kept as a -# full type/name path so `kubectl get "$RF"` works — watched so we see it deleted on de-manage. -RF=$(kubectl get resourceflavors.kueue.x-k8s.io -o name 2>/dev/null | grep -E '/gpustack--.*-[0-9]+c(-fnv64-[0-9a-f]{16})?$' | head -1) -[ -n "$RF" ] && [ -n "$IT" ] || { echo "no general RF/InstanceType found — run case-1 first to materialize the chain"; exit 1; } -# Every -gpustack-worker NodeFeature: de-managing ALL of them removes a pool that spans more -# than one node (a single node's toggle leaves the others' flavors behind). -WORKER_NFS=$(kubectl -n "$NS" get nodefeatures -o name 2>/dev/null | grep -E -- '-gpustack-worker$') -[ -n "$WORKER_NFS" ] || { echo "no -gpustack-worker NodeFeatures found"; exit 1; } -echo "general InstanceType: ${IT} ResourceFlavor: ${RF#*/} draining $(echo "$WORKER_NFS" | grep -c .) node(s)" - -restore() { - echo - echo "[case-3] restoring gpustack.ai/managed=true on all nodes and waiting for the chain to rebuild" - echo "$WORKER_NFS" | xargs -r -I{} kubectl -n "$NS" patch {} --type=merge \ - -p '{"spec":{"labels":{"gpustack.ai/managed":"true"}}}' 2>/dev/null || true - for _ in $(seq 1 90); do - p=$(kubectl get instancetypes.worker.gpustack.ai "$IT" -o jsonpath='{.status.phase}' 2>/dev/null) - [ "$p" = "Active" ] && break - sleep 3 - done -} -trap restore EXIT - -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } - -# Toggle out of management via the NodeFeature (NFD would revert a direct node label). -echo "[case-3] toggling gpustack.ai/managed=false on all worker NodeFeatures" -echo "$WORKER_NFS" | xargs -r -I{} kubectl -n "$NS" patch {} --type=merge \ - -p '{"spec":{"labels":{"gpustack.ai/managed":"false"}}}' - -# --- Assertion A: the node's general ResourceFlavor is DELETED (no tombstone). --- -# NFD can take a few minutes to propagate a NodeFeature label change to the Node object on a busy -# cluster, and the flavor deletion only follows once the Node is seen unmanaged — wait generously. -gone="" -for _ in $(seq 1 90); do - kubectl get "$RF" >/dev/null 2>&1 || { gone=1; break; } - sleep 3 -done -[ -n "$gone" ] && record PASS "flavor deleted on de-manage" "${RF#*/} gone (no drain tombstone)" \ - || record FAIL "flavor deleted on de-manage" "${RF#*/} still present — NodeFlavorReconciler did not drop the unmanaged node" - -# --- Assertion B: the InstanceType SURVIVES (not deleted); its backing queue's quota is emptied. --- -# The emptied queue must still be held: restoring its StopPolicy before a flavor returns reopens a -# queue with no quota. Both fields come from one read, so they describe the same instant; an absent -# stopPolicy is Kueue's default, None. -survived="" -for _ in $(seq 1 60); do - exists=$(kubectl get instancetypes.worker.gpustack.ai "$IT" -o name 2>/dev/null) - cqexists=$(kubectl get clusterqueue "$IT" -o name 2>/dev/null) - read -r rg stop < <(kubectl get clusterqueue "$IT" -o json 2>/dev/null | python3 -c "import json,sys -try: - spec = json.load(sys.stdin).get('spec', {}) - print(len(spec.get('resourceGroups') or []), spec.get('stopPolicy') or 'None') -except Exception: print(-1, '?')" 2>/dev/null) - if [ -n "$exists" ] && [ -n "$cqexists" ] && [ "$rg" = "0" ] && [ "$stop" != "None" ]; then survived=1; break; fi - sleep 3 -done -[ -n "$survived" ] && record PASS "InstanceType survives flavor loss; queue emptied and held" "${IT} kept, resourceGroups=0, stopPolicy=${stop} (drain-when-no-flavors, idle)" \ - || record FAIL "InstanceType survives flavor loss; queue emptied and held" "exists='${exists:-gone}' cqexists='${cqexists:-gone}' resourceGroups='${rg}' stopPolicy='${stop}' — the type must survive with its queue emptied and held, not tear down or reopen" - -# --- Assertion C: restoring the nodes refills the queue and the InstanceType reactivates. --- -echo "[case-3] restoring gpustack.ai/managed=true; expecting the pool to refill and reactivate" -echo "$WORKER_NFS" | xargs -r -I{} kubectl -n "$NS" patch {} --type=merge \ - -p '{"spec":{"labels":{"gpustack.ai/managed":"true"}}}' >/dev/null 2>&1 -reactivated="" -for _ in $(seq 1 90); do - p=$(kubectl get instancetypes.worker.gpustack.ai "$IT" -o jsonpath='{.status.phase}' 2>/dev/null) - rg=$(kubectl get clusterqueue "$IT" -o json 2>/dev/null | python3 -c "import json,sys -try: print(len(json.load(sys.stdin).get('spec',{}).get('resourceGroups') or [])) -except Exception: print(0)" 2>/dev/null) - if [ "$p" = "Active" ] && [ "${rg:-0}" -gt 0 ]; then reactivated=1; break; fi - sleep 3 -done -[ -n "$reactivated" ] && record PASS "pool reactivates when nodes return" "${IT} Active, resourceGroups refilled" \ - || record FAIL "pool reactivates when nodes return" "phase='${p:-?}' resourceGroups='${rg:-0}' — the pool must refill and reactivate" - -echo -echo "== CASE 3 — Managed-toggle scopes node onboarding ==" -{ - echo "STATUS|CHECK|OBJECT" - printf '%s\n' "${ROWS[@]}" -} | column -t -s '|' - -if [ "$FAILS" -ne 0 ]; then - echo - echo "FAILED ${FAILS} check(s). Confirm the operator was NOT restarted between toggle and assertion." - echo "Under the queue-ownership split the flavor is deleted but the InstanceType survives — the" - echo "NodeQueueReconciler empties/drains the queue (never deletes the type). See references/drain-recycle.md." - exit 1 -fi -echo "CASE 3 PASS" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-30.sh b/.agents/skills/gpustack-operator-e2e/cases/case-30.sh deleted file mode 100755 index 631c1ff12..000000000 --- a/.agents/skills/gpustack-operator-e2e/cases/case-30.sh +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env bash -# -# CASE 30 — A terminated init container still charges the card its instance occupies -# (MUTATING, self-recovering; AUTO-SKIPS without a partition-capable node AND a node-SSH address) -# -# MIG_NODE_SSH= case-30.sh -# -# Goal: A device is scoped to the Pod's life, not the container's. The kubelet keeps a finished -# init container's devices in its record for as long as the Pod exists, and the operator's -# reclaimer destroys a hardware partition on Pod deletion — not on container termination. -# So a Pod whose init container took a partition and then exited, with the Pod still -# running, is STILL occupying that hardware. The accounting has to say so: the node's -# per-profile capacity must keep charging the instance, or the node advertises room it -# does not have and the next tenant is admitted onto a card that cannot take it. Only -# hardware shows this — a ledger fixture passes whether or not the entry survives the -# container. The case therefore reads the live instance from the card, reads the node key, -# and then admits a further Pod against the room the node claims is left, requiring it to -# actually start. -# Environment: A reachable cluster whose active context is the GPU cluster, an nvidia node with at -# least one card that can be put into a hardware partitioning mode, AND SSH to that node -# (sudo nvidia-smi) supplied via MIG_NODE_SSH=. It EXITS 2 (input required) -# when MIG_NODE_SSH is unset, and AUTO-SKIPS (exit 0) when the card reports no -# partitioning mode, the mode switch does not converge, or the cards offer no -# 4-memory-slice plus whole-card profile pair. -# Inputs: All real, nothing mocked. One Pod whose INIT container requests the DISCOVERED -# 4-memory-slice partition and exits after a few seconds, and whose app container holds no -# accelerator at all — the request rules allow accelerator claims in exactly one container -# group, and this Pod puts them in the init group. Then a second, ordinary partition Pod -# of the same profile, to consume the room the node still reports. -# Expected: - the Pod reaches Running with its init container Terminated; -# - the card still holds a live hardware instance; -# - the whole-card profile key still reads one card's worth less, i.e. the terminated -# container's instance is still charged; -# - the follow-up partition Pod, admitted against the remaining reported room, reaches -# Running without a single UnexpectedAdmissionError. -# Cleanup: Trap deletes every test Pod, waits for the instances to reclaim, and restores the -# partitioning mode of the card this case toggled (a card found already partitioned is -# left as found). Idempotent; runs on pass AND fail. -set -uo pipefail - -# Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail -# on transport alone, and a check that takes such a failure for an answer reports a verdict -# about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" - -NS="${1:?usage: MIG_NODE_SSH= case-30.sh }" -CASE_ID=30 -# shellcheck source=/dev/null -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_partition-lib.sh" - -PODPFX=gpustack-e2e-initterm - -part_require_node_ssh "case-30.sh" -part_require_mig_capable -part_discover - -restore() { - echo - echo "[case-30] cleanup: deleting test Pods and restoring card(s) '${TOGGLED:-}'" - delete_test_pods - part_restore_toggled -} -trap restore EXIT - -part_ensure_partitioned_card - -read -r SMALL SMALL_CNT MID MID_CNT FULL FULL_CNT NPART NTOT <<<"$(card_profiles)" -MIDKEY="$(profile_key "$MID")" -FULLKEY="$(profile_key "$FULL")" -echo "[case-30] discovered profiles: MID=${MID}(${MID_CNT}/card) FULL=${FULL}(${FULL_CNT}/card); partitioned cards=${NPART:-0}" -if [ -z "$MIDKEY" ] || [ -z "$FULLKEY" ]; then - part_skip \ - "The node advertises no 4-memory-slice key ('${MIDKEY:-}') or no whole-card key" \ - "('${FULLKEY:-}'). This case reads the whole-card key to see the init container's charge." -fi -wait_card_idle || echo "[case-30] warning: the card already holds live instances" -BASE_FULL="$(node_key "$FULLKEY")" -echo "[case-30] baseline ${FULLKEY}=${BASE_FULL:-}" - -# --------------------------------------------------------------------------------------------------- -# A Pod whose INIT container holds the partition and then exits, while the Pod keeps running. -# --------------------------------------------------------------------------------------------------- -echo -echo "[case-30] === init container takes a ${MID} partition, then terminates ===" -P="${PODPFX}-pod" -TESTPODS+=("$P") -cat </dev/null -apiVersion: v1 -kind: Pod -metadata: { name: ${P}, namespace: default, labels: { kueue.x-k8s.io/queue-name: ${LQ} } } -spec: - schedulerName: default-scheduler - ${RTC_LINE} - restartPolicy: Never - nodeSelector: { kubernetes.io/hostname: ${GPU_NODE} } - initContainers: - - name: warmup - image: ${IMAGE} - command: ["sh", "-c", "nvidia-smi -L || true; sleep 15"] - resources: - limits: -$(partition_reslines "$MIDKEY") - requests: -$(partition_reslines "$MIDKEY") - containers: - - name: main - image: ${IMAGE} - command: ["sleep", "86400"] - resources: - limits: { cpu: "200m", memory: 256Mi } - requests: { cpu: "200m", memory: 256Mi } -EOF - -if ! wait_running "$P" 60; then - record FAIL "the Pod runs on past its init container" "${P} did not reach Running (phase=$(phase "$P"), held: $(held_reason "$P")) — an init-group accelerator claim must be admitted" - part_results "A terminated init container still charges the card its instance occupies" - exit 1 -fi -initstate="$(kubectl -n default get pod "$P" -o jsonpath='{.status.initContainerStatuses[0].state.terminated.reason}' 2>/dev/null)" -if [ -n "$initstate" ]; then - record PASS "the Pod runs on past its init container" "${P} Running with init container Terminated/${initstate}, card(s) $(pod_cards "$P"), profile '$(pod_profiles "$P")'" -else - record FAIL "the Pod runs on past its init container" "${P} is Running but its init container reports no terminated state — cannot exercise the terminated-container charge" -fi - -# --------------------------------------------------------------------------------------------------- -# The hardware is still occupied, and the accounting says so. -# --------------------------------------------------------------------------------------------------- -gi="$(node_gi_count)" -[ "${gi:-0}" -ge 1 ] \ - && record PASS "the hardware instance outlives the container" "${gi} live instance(s) on the card while the init container is long gone — the reclaimer destroys on Pod deletion, not container exit" \ - || record FAIL "the hardware instance outlives the container" "the card reports ${gi:-0} live instance(s) — the instance was destroyed while its Pod is still alive" - -exp_full=$((BASE_FULL - FULL_CNT)) -cur_full="" -for _ in $(seq 1 30); do - cur_full="$(node_key "$FULLKEY")" - [ "${cur_full:-x}" = "$exp_full" ] && break - sleep 3 -done -if [ "${cur_full:-x}" = "$exp_full" ]; then - record PASS "the terminated container's instance is still charged" "${FULLKEY}: ${BASE_FULL} → ${cur_full} (want ${exp_full}) — the occupied card can no longer host a whole-card profile" -else - record FAIL "the terminated container's instance is still charged" "${FULLKEY}='${cur_full:-}', want ${exp_full} — the ledger dropped a terminated container's entry and is advertising room the card does not have" -fi - -# --------------------------------------------------------------------------------------------------- -# Admit against the room the node still claims: it must be real room. -# --------------------------------------------------------------------------------------------------- -echo -echo "[case-30] === a further request against the reported free room ===" -Q="${PODPFX}-next" -mkpod "$Q" "$(partition_reslines "$MIDKEY")" -if wait_running "$Q"; then - uae="$(pod_unexpected_admission "$Q")" - [ -z "$uae" ] \ - && record PASS "the reported free room is real" "${Q} (${MID}) Running beside the init container's instance, 0 UnexpectedAdmissionError" \ - || record FAIL "the reported free room is real" "${Q} Running but recorded ${uae} UnexpectedAdmissionError event(s) — the node advertised room the card could not place" -else - record FAIL "the reported free room is real" "${Q} did not reach Running (held: $(held_reason "$Q")) — the node's remaining per-profile room must be placeable" -fi - -part_results "A terminated init container still charges the card its instance occupies" - -if [ "$FAILS" -ne 0 ]; then - echo - echo "FAILED ${FAILS} check(s). An allocation charges its card until its POD is gone, never until its" - echo "container exits — a liveness filter would report a card free while its instance still occupies" - echo "memory slices. Diagnose:" - echo " kubectl -n default get pod ${P} -o jsonpath='{.metadata.annotations.${ANNO}}'; echo" - echo " kubectl get devices ${GPU_NODE} -o json | jq '.status.groups[].accelerators[] | {id, allocatedProfiles, remainingProfiles}'" - echo " ${MIG_NODE_SSH} sudo nvidia-smi mig -lgi" - exit 1 -fi -echo "CASE 30 PASS" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-31.sh b/.agents/skills/gpustack-operator-e2e/cases/case-31.sh deleted file mode 100755 index f42640826..000000000 --- a/.agents/skills/gpustack-operator-e2e/cases/case-31.sh +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env bash -# -# CASE 31 — A same-profile replacement scheduled inside the reclaim window -# (MUTATING, self-recovering; AUTO-SKIPS without a partition-capable node AND a node-SSH address) -# -# MIG_NODE_SSH= case-31.sh -# -# Goal: MEASURE, rather than assume away, the gap between the accounting freeing a partition and -# the hardware actually releasing it. Node-level occupancy is rebuilt from the Pod -# annotations, so a deleted Pod's slot reappears in the per-profile key and in the healthy -# token count the moment the Pod is gone — while the reclaimer only destroys the instance -# after several consecutive absent sightings on its resync cadence. A replacement -# scheduled inside that window meets an instance that still exists, and what it then gets -# is MEASURED here rather than assumed: the allocation may be refused, or it may be -# granted that very instance which the reclaimer then destroys underneath the starting -# container. Both cost a retry, they are told apart by WHERE the attempt died, and only -# the first raises an UnexpectedAdmissionError -- the second leaves that count at zero. -# So this case records the shape of every failed attempt, not just how many there were. -# The containment is that the request is retried and the window closes on its own; the -# failure mode worth catching is a request that NEVER converges. So this case deletes a -# partition Pod, asks for the same profile again immediately, retries the way a controller -# would, and records how long convergence took, how many attempts it cost, and where each -# failed attempt died. -# Environment: A reachable cluster whose active context is the GPU cluster, an nvidia node with at -# least one card that can be put into a hardware partitioning mode, AND SSH to that node -# (sudo nvidia-smi) supplied via MIG_NODE_SSH=. It EXITS 2 (input required) -# when MIG_NODE_SSH is unset, and AUTO-SKIPS (exit 0) when the card reports no -# partitioning mode, the mode switch does not converge, or no 4-memory-slice profile is -# offered. -# Inputs: All real, nothing mocked — the reclaim cadence and the allocation outcome are the -# measurement. One partition Pod of the DISCOVERED 4-memory-slice profile is created, -# deleted, and then re-requested in a retry loop bounded by MIG_RECLAIM_BOUND seconds -# (default 300), all through the pool's entrance LocalQueue. A bare Pod is not -# re-created by anything, so the loop plays the part the controller plays in production. -# Expected: - the first Pod runs and is deleted; -# - the replacement eventually runs within the bound — that is the containment claim, and -# the only hard assertion here; -# - the attempt count, the elapsed seconds and any terminal allocation failures along the -# way are RECORDED as the measurement of the window, not asserted against a threshold. -# Cleanup: Trap deletes every test Pod, waits for the instances to reclaim, and restores the -# partitioning mode of the card this case toggled (a card found already partitioned is -# left as found). Idempotent; runs on pass AND fail. -set -uo pipefail - -# Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail -# on transport alone, and a check that takes such a failure for an answer reports a verdict -# about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" - -NS="${1:?usage: MIG_NODE_SSH= case-31.sh }" -CASE_ID=31 -# shellcheck source=/dev/null -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_partition-lib.sh" - -PODPFX=gpustack-e2e-reclaimwin -BOUND="${MIG_RECLAIM_BOUND:-300}" - -part_require_node_ssh "case-31.sh" -part_require_mig_capable -part_discover - -restore() { - echo - echo "[case-31] cleanup: deleting test Pods and restoring card(s) '${TOGGLED:-}'" - delete_test_pods - part_restore_toggled -} -trap restore EXIT - -part_ensure_partitioned_card - -read -r SMALL SMALL_CNT MID MID_CNT FULL FULL_CNT NPART NTOT <<<"$(card_profiles)" -MIDKEY="$(profile_key "$MID")" -[ -n "$MIDKEY" ] || part_skip "The node advertises no 4-memory-slice profile key — nothing to replace." -echo "[case-31] profile under test: ${MID} (${MIDKEY}); partitioned cards=${NPART:-0}" -wait_card_idle || echo "[case-31] warning: the card already holds live instances" - -# --------------------------------------------------------------------------------------------------- -# Occupy, then release, then immediately ask for the same shape again. -# --------------------------------------------------------------------------------------------------- -echo -echo "[case-31] === one ${MID} partition, deleted, then re-requested at once ===" -A="${PODPFX}-first" -mkpod "$A" "$(partition_reslines "$MIDKEY")" -if ! wait_running "$A"; then - record FAIL "the first partition runs" "${A} did not reach Running (held: $(held_reason "$A")) — cannot set up the reclaim window" - part_results "A same-profile replacement scheduled inside the reclaim window" - exit 1 -fi -GI_BEFORE="$(node_gi_count)" -record PASS "the first partition runs" "${A} Running on card(s) $(pod_cards "$A"); ${GI_BEFORE} live hardware instance(s)" - -delpod "$A" -T0="$(date +%s)" -ATTEMPTS=0 -UAE=0 -# Where the failed attempts died, tallied separately from UAE. A grant that is destroyed under the -# starting container raises no admission event, so UAE stays 0 through exactly the failure this window -# is most likely to produce; without this tally the run reads as clean. -SHAPE_ADMISSION=0 -SHAPE_START=0 -SHAPE_OTHER=0 -OK=0 -WINNER="" -while [ "$(( $(date +%s) - T0 ))" -lt "$BOUND" ]; do - ATTEMPTS=$((ATTEMPTS + 1)) - R="${PODPFX}-repl-${ATTEMPTS}" - mkpod "$R" "$(partition_reslines "$MIDKEY")" - for _ in $(seq 1 20); do - ph="$(phase "$R")" - case "$ph" in - Running) OK=1; WINNER="$R"; break ;; - Failed) break ;; - esac - sleep 3 - done - n="$(pod_unexpected_admission "$R")" - [ -n "$n" ] && UAE=$((UAE + n)) - [ "$OK" = 1 ] && break - # Read the shape ONCE: the Pod is live and a second read can answer differently. - shape="$(pod_failure_shape "$R")" - case "$shape" in - admission) SHAPE_ADMISSION=$((SHAPE_ADMISSION + 1)) ;; - start) SHAPE_START=$((SHAPE_START + 1)) ;; - *) SHAPE_OTHER=$((SHAPE_OTHER + 1)) ;; - esac - echo "[case-31] attempt ${ATTEMPTS} did not run (phase=$(phase "$R"), where=${shape}, held: $(held_reason "$R")) — retrying" - delpod "$R" - sleep 5 -done -T1="$(date +%s)" -ELAPSED=$((T1 - T0)) - -if [ "$OK" = 1 ]; then - record PASS "the replacement converges without intervention" "${WINNER} Running ${ELAPSED}s after the predecessor was deleted, on attempt ${ATTEMPTS} of at most ${BOUND}s" -else - record FAIL "the replacement converges without intervention" "no same-profile replacement reached Running within ${BOUND}s over ${ATTEMPTS} attempt(s) — the window is supposed to close on its own" -fi -record PASS "OBSERVED: cost of the reclaim window" "attempts=${ATTEMPTS}, elapsed=${ELAPSED}s, refused before actuating=${SHAPE_ADMISSION} (UnexpectedAdmissionError events=${UAE}), granted then destroyed under the container=${SHAPE_START}, other=${SHAPE_OTHER} (a first attempt that runs immediately means the window was already closed on this cadence; a non-zero 'granted then destroyed' means the window handed out a partition it went on to reclaim, which no admission counter records)" - -part_results "A same-profile replacement scheduled inside the reclaim window" - -echo -echo "---- fold back into the spec: reclaim-window replacement ----" -echo "profile : ${MID}" -echo "attempts : ${ATTEMPTS}" -echo "elapsed : ${ELAPSED}s (bound ${BOUND}s)" -echo "refused : ${SHAPE_ADMISSION} attempt(s) never actuated (${UAE} UnexpectedAdmissionError event(s))" -echo "granted then lost: ${SHAPE_START} attempt(s) were given a partition no container could then start on" -echo "converged : $([ "$OK" = 1 ] && echo yes || echo NO)" -echo "-------------------------------------------------------------" - -if [ "$FAILS" -ne 0 ]; then - echo - echo "FAILED ${FAILS} check(s). The accounting frees a partition on Pod deletion while the reclaimer destroys" - echo "the instance a few resync passes later; a replacement inside that window is retried, and the window must" - echo "close on its own. Read the shape line above before diagnosing: an attempt refused before actuating and an" - echo "attempt granted a partition that was then destroyed under it are different faults in different code." - echo "Diagnose:" - echo " ${MIG_NODE_SSH} sudo nvidia-smi mig -lgi" - echo " kubectl -n ${NS} logs ds/${DM_DS} --tail=300 | grep -i reclaim" - echo " kubectl get devices ${GPU_NODE} -o json | jq '.status.groups[].accelerators[] | {id, allocatedProfiles, remainingProfiles}'" - exit 1 -fi -echo "CASE 31 PASS" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-32.sh b/.agents/skills/gpustack-operator-e2e/cases/case-32.sh index d3af7b909..e5a8ae9e4 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-32.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-32.sh @@ -45,8 +45,8 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: MIG_NODE_SSH= case-32.sh }" CASE_ID=32 diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-34.sh b/.agents/skills/gpustack-operator-e2e/cases/case-34.sh index 2f2a66197..056ccd8b6 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-34.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-34.sh @@ -47,8 +47,8 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: MIG_NODE_SSH= case-34.sh }" CASE_ID=34 diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-35.sh b/.agents/skills/gpustack-operator-e2e/cases/case-35.sh index 7814fe3c3..e389c26b1 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-35.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-35.sh @@ -69,8 +69,11 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" + +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_pool-lib.sh" NS="${1:?usage: case-35.sh }" # Ascend's resource family. Hardcoded rather than derived: the case is gated on the ascend @@ -160,26 +163,7 @@ echo "[case-35] group card order (spec): ${SPEC_ORDER}; unhealthy excluded: ${UN # The accelerated InstanceType whose POOL this node belongs to, and its entrance LocalQueue. This must # match the target node, or the queue would route the claim to a different pool's nodes. Matched on the # pool's identity tuple — accelerator group plus os/arch — see the index's note on pool lookup. -read -r IT LQ <<<"$(kubectl get instancetypes.worker.gpustack.ai -o json 2>/dev/null | NODE_GID="$GROUPID" NODE_JSON="$(kubectl get node "$SLICE_NODE" -o json 2>/dev/null)" python3 -c " -import json,sys,os -gid=os.environ.get('NODE_GID','') -nl=(json.loads(os.environ.get('NODE_JSON') or '{}').get('metadata',{}) or {}).get('labels',{}) or {} -# A pool BACKS this node when every discriminator it carries is a label the node carries too. Those are -# the schedule labels the webhook stamps from PoolScheduleLabels, so this is the pool's whole identity — -# os/arch, the acceleratable boolean, the accelerator group and, under CPU-aware grouping, the general -# group. schedule.gpustack.ai/* is the pool's own bookkeeping and never a node label. -def backs(it): - d={k:v for k,v in (it['metadata'].get('labels') or {}).items() if not k.startswith('schedule.gpustack.ai/')} - return bool(d) and all(nl.get(k)==v for k,v in d.items()) -def in_group(s): - return bool(gid) and (s.get('acceleratorGroup') or '').endswith('-'+gid) -for it in json.load(sys.stdin).get('items',[]): - s=it.get('spec',{}); name=it['metadata']['name']; st=it.get('status',{}) - sd=(st.get('detail',{}) or {}).get('slicedDetail',{}) or {} - if s.get('acceleratable') and ((sd.get('logical',{}) or {}).get('count',0) or 0)>0 \ - and in_group(s) and backs(it): - print(name, st.get('entrance','')); break -")" +read -r IT LQ <<<"$(discover_node_pool "$SLICE_NODE" "$GROUPID" --sliceable name entrance)" [ -n "$IT" ] && [ -n "$LQ" ] || { echo "no logically sliceable ascend InstanceType with an entrance LocalQueue backs ${SLICE_NODE} in group '${GROUPID}'"; exit 1; } # The logical-slicing runtime needs the vendor runtimeClass to mount its driver-lib dependencies — a @@ -250,9 +234,6 @@ restore() { } trap restore EXIT -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } # verdict verdict() { [ "$1" -eq 0 ] && record PASS "$2" "$3" || record FAIL "$2" "$4"; } diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-36.sh b/.agents/skills/gpustack-operator-e2e/cases/case-36.sh index b87720d3b..52c469b0f 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-36.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-36.sh @@ -44,10 +44,12 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: case-36.sh }" +# shellcheck disable=SC2034 # Read by the sourced library, which shellcheck does not follow. +SYSTEM_NS="$NS" # the settings Secret lives in the namespace this case was given INST=gpustack-e2e-pinned INST_GATED=gpustack-e2e-gated INST_PLAIN=gpustack-e2e-plain # never holds an escape; the escalation-by-patch probe @@ -61,40 +63,32 @@ HOST_DIR=/tmp/gpustack-e2e-case36 LOCAL_PORT=22036 KEYDIR="$(mktemp -d)" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_ssh-lib.sh" + IT=$(kubectl get instancetypes.worker.gpustack.ai \ -o jsonpath='{.items[?(@.spec.acceleratable==false)].metadata.name}' 2>/dev/null | tr ' ' '\n' | grep -m1 'gpustack-') [ -n "$IT" ] || { echo "no general InstanceType found — run case-1 first to materialize the chain"; exit 1; } -# --- Setting helpers. The webhook resolves a Setting through a 30s cache, so a flip is not visible -# immediately; every gate assertion below RETRIES until the expected verdict appears rather than -# sleeping a fixed amount. --- -set_gate() { local b64; b64=$(printf '%s' "$2" | base64 | tr -d '\n') - kubectl -n "$NS" patch secret gpustack-settings --type=merge \ - -p "{\"data\":{\"$1\":\"${b64}\"}}" >/dev/null 2>&1; } -get_gate() { local v; v=$(kubectl -n "$NS" get secret gpustack-settings \ - -o jsonpath="{.data.$1}" 2>/dev/null | base64 -d 2>/dev/null); [ "$v" = "true" ] && echo true || echo false; } - -ORIG_PRIV=$(get_gate "$PRIV_KEY") -ORIG_HOST=$(get_gate "$HOST_KEY") +# --- The gates as the store carries them now, so cleanup puts both back as it found them. The +# webhook resolves a Setting through a 30s cache, so a flip is not visible immediately; every +# gate assertion below RETRIES until the expected verdict appears rather than sleeping a fixed +# amount. --- +settings_save "$PRIV_KEY" "$HOST_KEY" -PF_PID="" +# shellcheck disable=SC2317 # the trap keeps it reachable; the checker cannot see the EXIT restore() { echo echo "[case-36] cleanup: deleting test objects and restoring both gates" - [ -n "$PF_PID" ] && kill "$PF_PID" 2>/dev/null kubectl -n default delete instance "$INST" "$INST_GATED" "$INST_PLAIN" --ignore-not-found --wait=false 2>/dev/null || true kubectl -n default delete instancepersistentvolume "$IPV" --ignore-not-found --wait=false 2>/dev/null || true kubectl -n default delete configmap "$CM" --ignore-not-found 2>/dev/null || true kubectl -n default delete secret "$SEC" "$SSH_SECRET" --ignore-not-found 2>/dev/null || true - set_gate "$PRIV_KEY" "$ORIG_PRIV" - set_gate "$HOST_KEY" "$ORIG_HOST" + settings_restore rm -rf "$KEYDIR" 2>/dev/null || true } -trap restore EXIT +trap 'ssh_cleanup; restore' EXIT -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } print_and_exit() { echo echo "== CASE 36 — Node-pinned Instance with additional volumes, and the host-access gates ==" @@ -157,14 +151,12 @@ for _tool in ssh ssh-keygen; do command -v "$_tool" >/dev/null 2>&1 || HAS_SSH="" done if [ -n "$HAS_SSH" ]; then - ssh-keygen -t ed25519 -f "$KEYDIR/id" -N "" -q - kubectl -n default delete secret "$SSH_SECRET" --ignore-not-found >/dev/null 2>&1 - kubectl -n default create secret generic "$SSH_SECRET" --from-file=authorized_keys="$KEYDIR/id.pub" >/dev/null + ssh_setup_key_secret default "$SSH_SECRET" "$KEYDIR/id" fi # --- 1. The pinned Instance with four additional volumes. The hostPath one needs its gate on for the # CREATE, so turn that gate on here and assert its OFF behavior later with a separate Instance. --- -set_gate "$HOST_KEY" true +setting_set "$HOST_KEY" true echo "[case-36] creating pinned Instance ${INST} with four additional volumes" created="" for _ in $(seq 1 20); do @@ -343,15 +335,11 @@ ro=$(exec_main 'touch /mnt/host/probe >/dev/null 2>&1; echo $?') # --- 10. A mount is visible over SSH, with no sidecar change. --- if [ -n "$HAS_SSH" ]; then - kubectl -n default port-forward "pod/$POD" "${LOCAL_PORT}:22" >/dev/null 2>&1 & - PF_PID=$! - for _ in $(seq 1 20); do (exec 3<>"/dev/tcp/127.0.0.1/${LOCAL_PORT}") 2>/dev/null && { exec 3>&- 3<&-; break; }; sleep 1; done - SSH_OPTS=(-p "$LOCAL_PORT" -i "$KEYDIR/id" -o IdentitiesOnly=yes - -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=15) - over_ssh=$(ssh -T "${SSH_OPTS[@]}" root@127.0.0.1 'cat /mnt/cm/alpha' 2>/dev/null | tr -d '\r\n') + ssh_open_port_forward default "pod/$POD" 22 "$LOCAL_PORT" + over_ssh=$(ssh_exec "$KEYDIR/id" "$LOCAL_PORT" 'cat /mnt/cm/alpha' 2>/dev/null | tr -d '\r\n') [ "$over_ssh" = ALPHA_VALUE ] && record PASS "additional mount visible over SSH" "ssh 'cat /mnt/cm/alpha' == ALPHA_VALUE" \ || record FAIL "additional mount visible over SSH" "got '${over_ssh:-}' — the sidecar must see main's mounts" - kill "$PF_PID" 2>/dev/null; PF_PID="" + ssh_cleanup else record SKIP "additional mount visible over SSH" "no ssh client tools on the runner" fi @@ -404,8 +392,8 @@ await_verdict() { echo "$v" } -set_gate "$PRIV_KEY" false -set_gate "$HOST_KEY" false +setting_set "$PRIV_KEY" false +setting_set "$HOST_KEY" false v=$(await_verdict gated true false) case "$v" in gated:*) record PASS "privileged create rejected while gated off" "${PRIV_KEY}=false rejects spec.privileged" ;; *) record FAIL "privileged create rejected while gated off" "verdict '${v}' with ${PRIV_KEY}=false" ;; esac @@ -415,7 +403,7 @@ case "$v" in gated:*) record PASS "hostPath create rejected while gated off" "${ # Allowing only the hostPath gate must admit the hostPath one and still reject the privileged one — # the point of splitting the two Settings. -set_gate "$HOST_KEY" true +setting_set "$HOST_KEY" true v=$(await_verdict accepted false true) [ "$v" = accepted ] && record PASS "hostPath create admitted once allowed" "${HOST_KEY}=true admits it" \ || record FAIL "hostPath create admitted once allowed" "verdict '${v}'" @@ -426,8 +414,8 @@ case "$v" in gated:*) record PASS "each gate covers only its own field" "${HOST_ # --- 12. A gate judges the escape a change TAKES, not the one it carries. An Instance that already # holds an escape keeps it when the gate goes off; an Instance that never held one cannot acquire # it by patch. ${INST} was created with the hostPath gate on; turn it off and stop/restart it. --- -set_gate "$HOST_KEY" false -sleep 35 # outlive the Setting cache, so the updates below are judged with the gate genuinely off +setting_set "$HOST_KEY" false +settings_settle # outlive the Setting cache, so the updates below are judged with the gate genuinely off # A rejection here is only this contract's if it names a gate; report anything else as itself, so an # unrelated refusal can never read as "the gate leaked into UPDATE". diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-37.sh b/.agents/skills/gpustack-operator-e2e/cases/case-37.sh index 13bcc950f..785ca7fdd 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/case-37.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-37.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # -# CASE 37 — Instance metrics subresource serves the current instance-scoped utilization (MUTATING, self-recovering) +# CASE 37 — One Instance's utilization: served by the metrics subresource and published by the node's device manager +# (MUTATING, self-recovering; AUTO-SKIPS the exporter half without a device manager pod on the Instance's node) # # case-37.sh # @@ -8,15 +9,21 @@ # go. The Instance lives in `default`: the Instance webhook rejects a reserved namespace, so # creating it in would be denied outright. # -# Goal: The aggregated API's instances//metrics subresource returns the Instance's own -# current CPU/memory/storage utilization as Total/Used pairs, the used side read in -# real time from the node kubelet through the API-server node proxy — no Prometheus -# or metrics-server involved on this path. -# Environment: Any cluster with a materialized scheduling chain (run case-1 first). No GPU and no -# device manager required: this path reads the kubelet, and only the accelerator -# entries — which a CPU Instance has none of — come from a device manager. -# Inputs: All real, nothing mocked — reads the general InstanceType unit spec; a CPU Instance -# gpustack-e2e-metrics (alpine sleep + ephemeral volume) on the general pool. +# Goal: One Instance's own utilization is observable through both surfaces the operator +# offers: the aggregated API's instances//metrics subresource serves it straight +# from the node kubelet — no Prometheus or metrics-server on that path — and the +# device manager running on that node publishes the same Instance as Prometheus +# gauges labelled by the Instance's identity. +# Environment: Any cluster with a materialized scheduling chain (run case-1 first). The subresource +# half needs no GPU and no device manager: it reads the kubelet, and only the +# accelerator entries — which a CPU Instance has none of — come from a device manager. +# The exporter half AUTO-SKIPS when the node the Instance landed on runs no Ready +# device manager pod: a node with no accelerators runs no device manager, and then +# that surface has nothing to publish. +# Inputs: All real, nothing mocked — reads the general InstanceType unit spec; one CPU +# Instance gpustack-e2e-metrics (alpine sleep + ephemeral volume) on the general pool. +# Reads /metrics through the API server's pod proxy, so nothing has to be installed in +# the device manager image. # Expected: - GET .../instances//metrics returns one sample carrying timestamp and all # three pairs: cpuTotalMilliCores/cpuUsedMilliCores, memoryTotalMiB/memoryUsedMiB, # storageTotalMiB/storageUsedMiB; @@ -26,15 +33,26 @@ # - once stopped, and so holding no Pod at all, the same call still answers: the declared # totals with zero measurements rather than an error, since nothing has run; # - a caller without the instances/metrics grant is denied -# (kubectl auth can-i ... --as an unprivileged identity says no). -# Cleanup: Trap deletes the test Instance. +# (kubectl auth can-i ... --as an unprivileged identity says no); +# - exactly ONE device manager target on the node carries the pod-level families, so a +# query never double-counts an Instance; +# - every pod-level family carries a series for the Instance, labelled by exactly +# namespace/instance_name/instance_uid/node, and its totals agree with the +# subresource's sample; +# - no series collides with Prometheus's own instance label, which Prometheus would +# rename to exported_instance; +# - accelerator families, where present, carry id, manufacturer and mode beside the +# Instance's labels; +# - every device manager target reports on its own sources. +# Cleanup: Trap deletes the test Instance; idempotent, runs on pass AND fail. The InstanceType +# is only read. set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: case-37.sh }" INST=gpustack-e2e-metrics @@ -49,19 +67,14 @@ restore() { } trap restore EXIT -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } -# shellcheck source=/dev/null -. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" - # The derived general InstanceType is created with its unit spec, and that spec is immutable # afterwards, so there is nothing to set: read it as a precondition. The Instance webhook sizes the # Pod from it. unit_ram=$(kubectl get instancetypes.worker.gpustack.ai "$IT" -o jsonpath='{.spec.unitResources.ram}' 2>/dev/null) [ -n "$unit_ram" ] || { echo "no unit spec on ${IT} — the Instance webhook needs unitRAM to size the Pod"; exit 1; } -# 1. Create the Instance on the general pool. +# 1. Create the Instance on the general pool. Both halves of this case read the SAME Instance: the +# subresource through the kubelet, the device manager through its own exporter. echo "[case-37] creating Instance ${INST} of type ${IT}" cat </dev/null) +[ "$phase" = "Ready" ] || record FAIL "instance ready" "phase='${phase:-}'" + +INST_UID=$(kubectl -n default get instance "$INST" -o jsonpath='{.metadata.uid}' 2>/dev/null) +NODE=$(kubectl -n default get pod "$INST" -o jsonpath='{.spec.nodeName}' 2>/dev/null) + +# The device managers of THAT node, Ready ones only: a pod that is not Ready is not scraped, and +# is not in the running to be the node's exporter either. A read loop rather than mapfile, which +# bash 3.2 does not have. +DM_PODS=() +if [ -n "$NODE" ]; then + while IFS= read -r pod; do + [ -n "$pod" ] && DM_PODS+=("$pod") + done < <(kubectl -n "$NS" get pods \ + -l app.kubernetes.io/component=device-manager \ + --field-selector "spec.nodeName=${NODE},status.phase=Running" \ + -o jsonpath='{range .items[*]}{.metadata.name}{" "}{range .status.conditions[?(@.type=="Ready")]}{.status}{end}{"\n"}{end}' 2>/dev/null \ + | awk '$2=="True"{print $1}') +fi + +if [ -z "$NODE" ] || [ "${#DM_PODS[@]}" -eq 0 ]; then + echo "== CASE 37 — EXPORTER HALF SKIPPED ==" + echo "No Ready device manager pod runs on ${NODE:-}, the node the test Instance landed on." + echo "The exporter lives in the device manager, so on a node without one this surface has nothing to" + echo "publish. The metrics subresource half still answers for the Instance. Run this case on a" + echo "cluster whose general pool includes a node carrying accelerators." +else + echo "[case-37] node ${NODE} runs ${#DM_PODS[@]} Ready device manager pod(s): ${DM_PODS[*]}" + + # scrape — that pod's /metrics, through the API server's pod proxy so the device manager image + # needs no client tooling. The port is the container's own, not a Service port. + scrape() { + local pod=$1 port + port=$(kubectl -n "$NS" get pod "$pod" \ + -o jsonpath='{.spec.containers[0].ports[?(@.name=="https")].containerPort}' 2>/dev/null) + kubectl get --raw "/api/v1/namespaces/${NS}/pods/https:${pod}:${port:-32443}/proxy/metrics" 2>/dev/null + } + + # Which target carries the pod-level families for this Instance. The poller samples on the + # monitor period, so the first scrape after the Instance goes Ready can legitimately miss it. + POD_FAMILIES="gpustack_instance_cpu_total_millicores gpustack_instance_cpu_used_millicores \ +gpustack_instance_memory_total_mib gpustack_instance_memory_used_mib \ +gpustack_instance_storage_total_mib gpustack_instance_storage_used_mib" + + EXPORTING=() + SCRAPE="" + for _ in $(seq 1 12); do + EXPORTING=() + for pod in "${DM_PODS[@]}"; do + body=$(scrape "$pod") + [ -n "$body" ] || continue + if printf '%s' "$body" | grep -q "^gpustack_instance_cpu_total_millicores{.*instance_uid=\"${INST_UID}\""; then + EXPORTING+=("$pod") + SCRAPE=$body + fi + done + [ "${#EXPORTING[@]}" -ge 1 ] && break + sleep 5 + done + + if [ "${#EXPORTING[@]}" -eq 1 ]; then + record PASS "exactly one target carries the pod-level families" \ + "${EXPORTING[0]} of ${#DM_PODS[@]} device manager pod(s) on ${NODE}" + elif [ "${#EXPORTING[@]}" -eq 0 ]; then + record FAIL "exactly one target carries the pod-level families" \ + "no device manager pod on ${NODE} published instance_uid=${INST_UID} within 60s" + else + record FAIL "exactly one target carries the pod-level families" \ + "${#EXPORTING[@]} pods published it (${EXPORTING[*]}) — the node's Instances are double-counted" + fi + + # What the exporting target actually published for this Instance: every pod-level family + # present, labelled by exactly the four identifying labels with the Instance's own values, and + # carrying the same figures the subresource serves. One parse, three verdicts. + if [ -n "$SCRAPE" ]; then + sample=$(kubectl get --raw "$RAW" 2>/dev/null) + while IFS='|' read -r check state detail; do + [ -n "$check" ] && record "$state" "$check" "$detail" + done </dev/null) +VERDICTS + + # The reserved target label. `instance_name` exists precisely so the exposition never carries a + # bare `instance`, which Prometheus renames to exported_instance under honor_labels: false. + collide=$(printf '%s' "$SCRAPE" | grep -c "^gpustack_instance_.*[{,]instance=\"") + if [ "${collide:-1}" -eq 0 ]; then + record PASS "no series collides with Prometheus's own instance label" "no bare instance= on any gpustack_instance_ series" + else + record FAIL "no series collides with Prometheus's own instance label" "${collide} series carry a bare instance= label" + fi + + # Accelerator families, when the node runs an accelerated Instance. A CPU-only node has none, + # which is not a failure of anything — say so rather than pass vacuously. + acc=$(printf '%s' "$SCRAPE" | grep -m1 "^gpustack_instance_accelerator_memory_total_mib{") + # mode is part of the set: every accelerator family reports the Instance's own grant and usage + # whatever the allocation did, and mode is what a query groups or filters by rather than what it + # picks a metric name with. + ACC_WANT="id,index,instance_name,instance_uid,manufacturer,mode,namespace,node" + if [ -z "$acc" ]; then + record SKIP "accelerator families carry id, index, manufacturer and mode" \ + "no accelerator series on ${NODE}: no Instance here holds an accelerator, so there is none to label" + else + acc_labels=$(printf '%s' "$acc" | sed -e 's/^[^{]*{//' -e 's/}.*$//' \ + | tr ',' '\n' | cut -d= -f1 | sort | tr '\n' ',') + if [ "${acc_labels%,}" = "$ACC_WANT" ]; then + record PASS "accelerator families carry id, index, manufacturer and mode" "${acc_labels%,}" + else + record FAIL "accelerator families carry id, index, manufacturer and mode" \ + "label set is ${acc_labels%,}, expected ${ACC_WANT}" + fi + fi + fi + + # Every target reports on its own sources, whichever of them is the exporter: a device manager + # whose kubelet read is failing must say so here rather than only in its log. + for pod in "${DM_PODS[@]}"; do + body=$(scrape "$pod") + ok=$(printf '%s' "$body" | grep -m1 '^gpustack_instance_metrics_collector_success{source="kubelet"}' | awk '{print $2}') + dur=$(printf '%s' "$body" | grep -m1 '^gpustack_instance_metrics_collector_duration_seconds{source="kubelet"}' | awk '{print $2}') + snap=$(printf '%s' "$body" | grep -m1 '^gpustack_instance_metrics_collector_success{source="snapshot"}' | awk '{print $2}') + if [ "$ok" = "1" ] && [ -n "$dur" ] && [ -n "$snap" ]; then + record PASS "${pod} reports on its own sources" "kubelet success=1 duration=${dur}s, snapshot success=${snap}" + else + record FAIL "${pod} reports on its own sources" \ + "kubelet success='${ok:-}' duration='${dur:-}' snapshot success='${snap:-}'" + fi + done +fi + +# 6. An Instance that has started nothing is answered, not refused. Stopping it takes its Pod away, +# which is the same state as never-scheduled and as a previous incarnation's Pod: nothing has run, so +# nothing has been used. The surface must serve its DECLARED totals with zero measurements rather than +# an error — a console reading it should need no branch for an Instance that is merely stopped. +# The stop comes last: it takes the backing Pod away, and the exporter half above reads that Pod. echo "[case-37] stopping ${INST} to read the no-Pod answer" # The field is spec.stop. A merge patch of a field the CRD does not have is accepted, pruned and # only WARNED about, so a wrong name here does not fail — the Instance simply keeps running, its Pod @@ -250,7 +449,7 @@ print(("FAIL " + "; ".join(bad)) if bad else "PASS") esac fi -# 6. Authorization: an unprivileged identity must not hold get on instances/metrics. +# 7. Authorization: an unprivileged identity must not hold get on instances/metrics. deny=$(kubectl auth can-i get instances.worker.gpustack.ai/metrics -n default \ --as system:serviceaccount:default:default 2>/dev/null) if [ "$deny" = "no" ]; then diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-38.sh b/.agents/skills/gpustack-operator-e2e/cases/case-38.sh index 8cd1d1760..50aba967c 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-38.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-38.sh @@ -106,8 +106,11 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" + +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_pool-lib.sh" NS="${1:?usage: case-38.sh }" @@ -250,27 +253,7 @@ bdf_of() { # resources it sizes a whole-accelerator request by. Must match the target node, or the queue would route # the claim to another pool. Matched on the pool's identity tuple — accelerator group plus os/arch — see # the index's note on pool lookup. -read -r IT LQ UNIT_CPU UNIT_RAM <<<"$(kubectl get instancetypes.worker.gpustack.ai -o json 2>/dev/null | NODE_GID="$GROUPID" NODE_JSON="$(kubectl get node "$NODE" -o json 2>/dev/null)" python3 -c " -import json,sys,os -gid=os.environ.get('NODE_GID','') -nl=(json.loads(os.environ.get('NODE_JSON') or '{}').get('metadata',{}) or {}).get('labels',{}) or {} -# A pool BACKS this node when every discriminator it carries is a label the node carries too. Those are -# the schedule labels the webhook stamps from PoolScheduleLabels, so this is the pool's whole identity — -# os/arch, the acceleratable boolean, the accelerator group and, under CPU-aware grouping, the general -# group. schedule.gpustack.ai/* is the pool's own bookkeeping and never a node label. -def backs(it): - d={k:v for k,v in (it['metadata'].get('labels') or {}).items() if not k.startswith('schedule.gpustack.ai/')} - return bool(d) and all(nl.get(k)==v for k,v in d.items()) -def in_group(s): - return bool(gid) and (s.get('acceleratorGroup') or '').endswith('-'+gid) -for it in json.load(sys.stdin).get('items',[]): - s=it.get('spec',{}); name=it['metadata']['name']; st=it.get('status',{}) - sd=(st.get('detail',{}) or {}).get('slicedDetail',{}) or {} - if s.get('acceleratable') and ((sd.get('logical',{}) or {}).get('count',0) or 0)>0 \ - and in_group(s) and backs(it): - u=s.get('unitResources',{}) or {} - print(name, st.get('entrance',''), u.get('cpu','') or '-', u.get('ram','') or '-'); break -")" +read -r IT LQ UNIT_CPU UNIT_RAM <<<"$(discover_node_pool "$NODE" "$GROUPID" --sliceable name entrance unit-cpu unit-ram)" [ -n "${IT:-}" ] && [ -n "${LQ:-}" ] || { echo "== CASE 38 — FAILED (setup) ==" echo "No logically sliceable InstanceType with an entrance LocalQueue backs ${NODE} in group" @@ -386,9 +369,6 @@ restore() { } trap restore EXIT -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } # verdict verdict() { [ "$1" -eq 0 ] && record PASS "$2" "$3" || record FAIL "$2" "$4"; } diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-39.sh b/.agents/skills/gpustack-operator-e2e/cases/case-39.sh index 2eff00d6c..7d170ae5d 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-39.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-39.sh @@ -84,8 +84,11 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" + +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_pool-lib.sh" NS="${1:?usage: case-39.sh }" @@ -183,24 +186,7 @@ echo "[case-39] one accelerator = ${CARD_MEM_MIB}MiB VRAM" # the queue would route the claim to a different pool's nodes. The InstanceType qualifies the group with # the manufacturer where the per-node group id does not, so the manufacturer is supplied here and the # test is equality — see the index's note on pool lookup for why containment is not enough. -read -r IT LQ <<<"$(kubectl get instancetypes.v1alpha1.worker.gpustack.ai -o json 2>/dev/null | NODE_GID="$GROUPID" NODE_JSON="$(kubectl get node "$NODE" -o json 2>/dev/null)" python3 -c " -import json,sys,os -gid=os.environ.get('NODE_GID','') -nl=(json.loads(os.environ.get('NODE_JSON') or '{}').get('metadata',{}) or {}).get('labels',{}) or {} -# A pool BACKS this node when every discriminator it carries is a label the node carries too. Those are -# the schedule labels the webhook stamps from PoolScheduleLabels, so this is the pool's whole identity — -# os/arch, the acceleratable boolean, the accelerator group and, under CPU-aware grouping, the general -# group. schedule.gpustack.ai/* is the pool's own bookkeeping and never a node label. -def backs(it): - d={k:v for k,v in (it['metadata'].get('labels') or {}).items() if not k.startswith('schedule.gpustack.ai/')} - return bool(d) and all(nl.get(k)==v for k,v in d.items()) -def in_group(s): - return bool(gid) and (s.get('acceleratorGroup') or '').endswith('-'+gid) -for it in json.load(sys.stdin).get('items',[]): - s=it.get('spec',{}); st=it.get('status',{}) - if s.get('acceleratable') and in_group(s) and backs(it) and st.get('entrance'): - print(it['metadata']['name'], st['entrance']); break -")" +read -r IT LQ <<<"$(discover_node_pool "$NODE" "$GROUPID" --require-entrance name entrance)" [ -n "${IT:-}" ] && [ -n "${LQ:-}" ] || { echo "no T-Head InstanceType with an entrance LocalQueue backs ${NODE} in group '${GROUPID}'"; exit 1; } echo "[case-39] pool ${IT} via LocalQueue ${LQ}; slice pods carry no runtimeClass (this vendor needs none)" @@ -379,9 +365,6 @@ restore() { } trap restore EXIT -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } # verdict verdict() { [ "$1" -eq 0 ] && record PASS "$2" "$3" || record FAIL "$2" "$4"; } diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-4.sh b/.agents/skills/gpustack-operator-e2e/cases/case-4.sh index 9d2dfa64e..05e27e4f3 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-4.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-4.sh @@ -1,45 +1,63 @@ #!/usr/bin/env bash # -# CASE 4 — AdmissionCheck holds exclusive over-admit (MUTATING, self-recovering) +# CASE 4 — Mocked accelerator pool: over-admit held by the AdmissionCheck, four-view tracks the ledger (MUTATING, self-recovering) # # case-4.sh # -# Goal: When every card is already sliced, a request for whole exclusive cards passes the -# coarse Kueue credits gate but is HELD by the node-devices AdmissionCheck reading the -# per-card Devices ledger — so it never becomes an admitted-then-unschedulable Workload. -# The check returns Retry (transient, re-checked as capacity frees), not Rejected, and -# the same request is admitted once the cards are freed. +# Goal: Two contracts over one mocked accelerator pool. A request for whole exclusive cards the +# pool cannot satisfy passes the coarse Kueue credits gate but is HELD by the +# node-devices AdmissionCheck reading the per-card Devices ledger — never becoming an +# admitted-then-unschedulable Workload: the check returns Retry (transient, re-checked +# as capacity frees), not Rejected, and the same request is admitted once the cards are +# freed. The materialized InstanceType's four-view (.status.accelerator / +# acceleratorShared / acceleratorSliced / acceleratorPartitioned) tracks that same +# ledger through a six-step pooling sequence and moves live over a native watch; the +# mocked cards report no hardware partitioning capability, so the partition view stays +# 0 throughout; the unit spec is immutable after create and the rejected write touches +# no Node/NodeFeature. # Environment: Any cluster with one Ready, untainted, schedulable node that has no Devices ledger # and reports no nvidia.com/gpu, BY APPROXIMATION — no accelerator is needed, and the # fake product key nvidia-e2emock never collides with a real GPU's derived pool. # AUTO-SKIPS (exit 0, printing NOTHING WAS VERIFIED) when no such node exists. -# Inputs: - MOCKED: a fake accelerator NodeFeature (nvidia-e2emock, count=8, 24Gi/card) that -# drives real derivation of the accelerated ResourceFlavor → ClusterQueue → InstanceType; -# a Devices ledger named after the node where all 8 cards are 50%-sliced (no clean -# whole card). The check scopes a card to the node its ledger is named after, and a -# topology assignment names that node, so a ledger named after no node serves no -# demand and holds every request whatever its cards say; -# - MOCKED: the same occupancy as a Pod bound to the node whose allocation annotation, in -# the device plugin's own format, charges each card 50% in sliced mode. The check +# Inputs: - MOCKED: a fake accelerator NodeFeature (nvidia-e2emock, count=8, 24Gi A10G-like +# card) that drives real derivation of the accelerated ResourceFlavor → ClusterQueue +# → InstanceType; +# - MOCKED: a Devices ledger named after the node whose eight cards are 50%-sliced (no +# clean whole card), stepped through a six-state pooling sequence. The check scopes +# a card to the node its ledger is named after, so a ledger named after no node +# serves no demand and holds every request whatever its cards say; +# - MOCKED: the same occupancy as a Pod bound to the node whose allocation annotation, +# in the device plugin's own format, charges each card 50% in sliced mode: the check # rebuilds the ledger from the Pods bound to the node rather than reading its -# published Status, so a Status written without its Pods reads as free to the check. -# No device manager runs on the node, so nothing rewrites the published Status; -# - MOCKED: the Devices ledger is labelled with the accelerated flavor's own node -# selector (minus its node-batch pin), which is how the AdmissionCheck finds it; -# - MOCKED: the Node's nvidia.com/gpu capacity (8), advertised through the status -# subresource. Kueue's topology-aware scheduling fits a Pod against the Node's own -# allocatable, so on a node with no such resource the Workload never reserves quota -# and the AdmissionCheck is never consulted; +# published Status, and no device manager runs on the node, so nothing rewrites that +# Status; +# - MOCKED: the ledger's labels — the accelerated flavor's own node selector (minus its +# node-batch pin), which is how the AdmissionCheck finds it, and the queue's own +# schedule labels, which is how the four-view finds it; +# - MOCKED: the Node's nvidia.com/gpu capacity (8) through the status subresource. +# Kueue's topology-aware scheduling fits a Pod against the Node's own allocatable, so +# on a node with no such resource the Workload never reserves quota and the check is +# never consulted; # - real: a raw Pod requesting 5 exclusive nvidia.com/gpu cards on the pool's entrance -# LocalQueue (a raw Pod, not an Instance, keeps this independent of the Instance webhook); +# LocalQueue (a raw Pod, not an Instance, keeps this independent of the Instance +# webhook); +# - real probe: a patch editing the InstanceType's unit spec (expects rejection); # - NOT mocked (the verification): the CQ→AdmissionCheck wiring, the Workload quota -# reservation, and the per-card feasibility the reconciler computes over the ledger. +# reservation, the per-card feasibility over the ledger, the flavor/CQ/InstanceType +# derivation and the four-view bin-packing math. # Expected: - the gpustack-node-devices AdmissionCheck is Active; # - the derived accelerated InstanceType materializes; # - its backing ClusterQueue references the AdmissionCheck and carries the mocked flavor; # - the Workload's AdmissionCheck state is Retry and it is NOT Admitted (held, not rejected); # - positive, with the charging Pod deleted and the ledger freed: the same Workload's -# check turns Ready and it is Admitted. +# check turns Ready and it is Admitted; +# - the InstanceType's spec.os/spec.arch equal its ClusterQueue's kubernetes.io/os|arch +# schedule labels; +# - the four-view matches the oracle at each step of +# 8/80/800/0 → 6/60/600/0 → 4/58/400/0 → 2/38/360/0 → 2/38/356/0 → 1/28/256/0; +# - a native watch observes the exclusive count move 8 → 4 → 8; +# - the unit-spec edit is REJECTED (immutable) and the stored value is unchanged; +# - the worker NodeFeature labels are unchanged. # Cleanup: Trap deletes the test Pod, the charging Pod, the mocked Devices, and the injected # NodeFeature, removes the nvidia.com/gpu capacity from the Node when this case # advertised it, and deletes the derived InstanceType once the mocked flavor is gone. @@ -48,8 +66,8 @@ set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail # on transport alone, and a check that takes such a failure for an answer reports a verdict # about the network rather than about the operator. -E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" -[ -n "$E2E_SHIM_DIR" ] && PATH="$E2E_SHIM_DIR:$PATH" +# shellcheck source=/dev/null +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh" NS="${1:?usage: case-4.sh }" # A Ready, untainted, schedulable node without a Devices ledger of its own and without an @@ -82,12 +100,15 @@ AGID="${AKEY#*-}" COUNT=8 MEM_MIB=24576 D=1600000 +SLICES_PER_CARD=128 # declared logical-slice capability per card AC=gpustack-node-devices ACCEL_NF="${NODE}-gpustack-e2e-accel" +WORKER_NF="${NODE}-gpustack-worker" # the worker NodeFeature the unit-spec write must NOT touch MOCK_DEV="${NODE}" HOLDER=gpustack-e2e-overadmit-holds LABELPFX="acceleratable.feature.gpustack.ai/${AKEY}" -EXCL_RES="nvidia.com/gpu" # exclusive whole-card resource for nvidia +MANAGED_LABEL="gpustack.ai/managed" +EXCL_RES="nvidia.com/gpu" # exclusive whole-card resource for nvidia POD=gpustack-e2e-overadmit # Set to 1 BEFORE the Node is patched, so an interruption between the two still removes it; removing # a key that was never added fails harmlessly, while leaving an added one advertises cards that do not @@ -97,7 +118,7 @@ ITNAME="" restore() { echo - echo "[case-4] cleanup: deleting Pod, mocked Devices, injected NodeFeature" + echo "[case-4] cleanup: deleting Pod, charging Pod, mocked Devices, injected NodeFeature" kubectl -n default delete pod "$POD" "$HOLDER" --ignore-not-found --force --grace-period=0 2>/dev/null || true kubectl delete devices.worker.gpustack.ai "$MOCK_DEV" --ignore-not-found 2>/dev/null || true kubectl -n "$NS" delete nodefeature "$ACCEL_NF" --ignore-not-found 2>/dev/null || true @@ -123,18 +144,125 @@ restore() { } trap restore EXIT -FAILS=0 -ROWS=() -record() { ROWS+=("$1|$2|$3"); [ "$1" = FAIL ] && FAILS=$((FAILS + 1)); return 0; } +# print_table prints the rows collected so far. A precondition that ends the case prints it first: +# the rows that did hold are evidence, not noise. +print_table() { + { + echo "STATUS|CHECK|OBJECT" + if [ "${#ROWS[@]}" -gt 0 ]; then + printf '%s\n' "${ROWS[@]}" + fi + } | column -t -s '|' +} + +# set_ledger "<8 space-separated card tokens>" patches the mocked Devices ledger. +# Tokens: f=free, e=exclusive, s9=shared(9 shares free), l80/l78/l50=logically sliced(80/78/50% free). +# No token carries a hardware partitioning capability, so every mocked card is unpartitioned and the +# partition view (PT) stays 0 at every step. +set_ledger() { + local tokens="$1" + local groups + groups=$(D="$D" TOKENS="$tokens" AGID="$AGID" python3 - <<'PY' +import json, os +D = int(os.environ["D"]) +tok = { + "f": (0, D), # free whole card + "e": (1, 0), # exclusive: nothing left + "s9": (2, 9 * (D // 10)), # shared: 9 of 10 ownership shares free + "l80": (3, 80 * (D // 100)), # logically sliced: 80% VRAM free + "l78": (3, 78 * (D // 100)), # logically sliced: 78% VRAM free + "l50": (3, 50 * (D // 100)), # logically sliced: 50% VRAM free (the over-admit fixture) +} +accs = [] +for i, t in enumerate(os.environ["TOKENS"].split()): + mode, rem = tok[t] + accs.append({"id": "c%d" % i, "index": i, "mode": mode, "remaining": rem}) +print(json.dumps({"status": {"groups": [ + {"id": os.environ["AGID"], "manufacturer": "nvidia", "accelerators": accs}, +]}})) +PY +) + # Target the v1alpha1 CRD explicitly: the unversioned/v1 resource is the aggregated + # proxy, whose /status subresource write returns ServiceUnavailable — only the real + # v1alpha1 CRD serves the status subresource. + # + # Fail loudly. Discarding this write's outcome makes a lost patch indistinguishable from a view + # that never converged: the next assertion polls to its timeout and reports a four-view mismatch, + # which reads as a product defect while the ledger it was supposed to compare against was never + # written. On a remote cluster that is a live risk, not a hypothetical — the API endpoint drops a + # request often enough to have already sent one run down that path. + if ! kubectl patch devices.v1alpha1.worker.gpustack.ai "$MOCK_DEV" \ + --subresource=status --type=merge -p "$groups" >/dev/null; then + echo "[case-4] ledger patch failed for tokens '${tokens}' — the next assertion would blame the four-view for a write that never landed" + return 1 + fi + return 0 +} + +# abort_on_ledger ends the case when a ledger write fails: the four-view cannot be compared against +# a ledger that was never written. The rows collected so far are printed first, so the section that +# already ran keeps its evidence. +abort_on_ledger() { + echo "[case-4] FATAL: the ledger write never landed — the four-view rows that follow cannot be compared against it" + print_table + exit 1 +} + +# assert_view