diff --git a/.agents/skills/gpustack-operator-benchmark/SKILL.md b/.agents/skills/gpustack-operator-benchmark/SKILL.md index 6b6372d69..c16b7be83 100644 --- a/.agents/skills/gpustack-operator-benchmark/SKILL.md +++ b/.agents/skills/gpustack-operator-benchmark/SKILL.md @@ -1,17 +1,17 @@ --- name: gpustack-operator-benchmark -description: "Benchmark ModelDeployment serving topologies on a GPU cluster: warm NodeModelStore cache, drive in-cluster load across EPD, P/D, Elastic EP, and External DP, and collect stage and transfer evidence." +description: "Benchmark the operator on a GPU cluster — today the ModelDeployment serving topologies: warm NodeModelStore cache, drive in-cluster load across EPD, P/D, Elastic EP, and External DP, and collect stage and transfer evidence." disable-model-invocation: true allowed-tools: "Read, Agent, Bash(bash .claude/skills/_e2e-lib/scripts/preflight.sh*), Bash(bash .claude/skills/_e2e-lib/scripts/assert-core.sh*), Bash(bash .claude/skills/_e2e-lib/scripts/cluster-auth.sh*), Bash(bash .claude/skills/_e2e-lib/scripts/kube-context.sh*), Bash(bash .claude/skills/gpustack-operator-benchmark/cases/*), Bash(kubectl get*), Bash(kubectl describe*), Bash(kubectl logs*), Bash(kubectl cluster-info*), Bash(kubectl version*), Bash(kubectl config current-context), Bash(kubectl config get-contexts*), Bash(git diff*), Bash(git rev-parse*), Bash(command -v*), Bash(date*), Bash(mkdir -p .claude/reports/*), Bash(tee .claude/reports/*)" --- -# GPUStack Operator — ModelDeployment Serving Benchmark +# GPUStack Operator — Benchmark Use [the shared module map](../../../docs/README.md#module-map) to locate the relevant guides, owning specs, code entry points and related modules. [AGENTS.md](../../../AGENTS.md#finding-sources) owns the retrieval and evidence rules; this skill owns its execution procedure. -Measure and compare the four `ModelDeployment` serving topologies on a real multi-GPU cluster (8 × GPU baseline) and collect both end-to-end latency/throughput distributions (`TTFT`, `ITL`, `TPOT`, `E2E`, `output_tok/s`, `req/s`) and per-stage execution / KV-transfer / EC-transfer evidence: +Measure and compare the `ModelDeployment` serving scenarios on a real multi-GPU cluster (8 × GPU baseline) and collect both end-to-end latency/throughput distributions (`TTFT`, `ITL`, `TPOT`, `E2E`, `output_tok/s`, `req/s`) and per-stage execution / KV-transfer / EC-transfer evidence. The scenarios below are the suite's first five; a scenario outside serving joins this suite, not a new skill: - **Scenario 1 (`cases/case-1-epd.sh`)** — [Encode, Prefill and Decode](../../../docs/walkthroughs/model-deployment/encode-prefill-decode.md) ([benchmark](../../../docs/modules/model-deployment/encode-prefill-decode-benchmark.md)): 3-arm comparison (`Aggregated` vs `P/D` vs `E/P/D`) on a 3-GPU budget with `Qwen/Qwen2.5-VL-7B-Instruct` (`NIXL` encoder cache + `Mooncake` KV cache). - **Scenario 2 (`cases/case-2-prefill-decode.sh`)** — [Prefill and Decode](../../../docs/walkthroughs/model-deployment/prefill-decode.md) ([benchmark](../../../docs/modules/model-deployment/prefill-decode-benchmark.md)): 2-arm comparison (`Aggregated` 3 × `Server(TP=2)` vs `P/D` 1 × `Prefill(TP=2)` + 2 × `Decode(TP=2)`) on a 6-GPU budget with `Qwen/Qwen3-32B`. diff --git a/.agents/skills/gpustack-operator-e2e/SKILL.md b/.agents/skills/gpustack-operator-e2e/SKILL.md index b6c37c0b3..7c7a460b1 100644 --- a/.agents/skills/gpustack-operator-e2e/SKILL.md +++ b/.agents/skills/gpustack-operator-e2e/SKILL.md @@ -2,10 +2,10 @@ name: gpustack-operator-e2e description: "Run the operator's end-to-end suite on a Kubernetes cluster — the user's own, or one provisioned for the run and destroyed afterwards: deploy via the Helm chart, then assert the NFD → Worker → Kueue scheduling chain materializes." disable-model-invocation: true -allowed-tools: "Read, Agent, Bash(bash .claude/skills/_e2e-lib/scripts/preflight.sh*), Bash(bash .claude/skills/_e2e-lib/scripts/assert-core.sh*), Bash(bash .claude/skills/_e2e-lib/scripts/cluster-auth.sh*), Bash(bash .claude/skills/_e2e-lib/scripts/kube-context.sh*), Bash(bash .claude/skills/gpustack-operator-e2e/cases/case-1.sh*), Bash(kubectl get*), Bash(kubectl describe*), Bash(kubectl logs*), Bash(kubectl cluster-info*), Bash(kubectl version*), Bash(kubectl config current-context), Bash(kubectl config get-contexts*), Bash(git diff*), Bash(git rev-parse*), Bash(command -v*), Bash(date*), Bash(mkdir -p .claude/reports/*), Bash(tee .claude/reports/*)" +allowed-tools: "Read, Agent, Bash(bash .claude/skills/_e2e-lib/scripts/preflight.sh*), Bash(bash .claude/skills/_e2e-lib/scripts/assert-core.sh*), Bash(bash .claude/skills/_e2e-lib/scripts/cluster-auth.sh*), Bash(bash .claude/skills/_e2e-lib/scripts/kube-context.sh*), Bash(bash .claude/skills/_e2e-lib/scripts/build-load.sh*), Bash(bash .claude/skills/_e2e-lib/scripts/deploy.sh*), Bash(bash .claude/skills/_e2e-lib/scripts/provision.sh*), Bash(bash .claude/skills/_e2e-lib/scripts/teardown.sh*), Bash(bash .claude/skills/_e2e-lib/scripts/destroy.sh*), Bash(bash .claude/skills/gpustack-operator-e2e/cases/*), Bash(kubectl get*), Bash(kubectl describe*), Bash(kubectl logs*), Bash(kubectl cluster-info*), Bash(kubectl version*), Bash(kubectl config current-context), Bash(kubectl config get-contexts*), Bash(git diff*), Bash(git rev-parse*), Bash(command -v*), Bash(date*), Bash(mkdir -p .claude/reports/*), Bash(tee .claude/reports/*)" --- -# GPUStack Operator — local E2E verification +# GPUStack Operator — E2E verification Use [the shared module map](../../../docs/README.md#module-map) to locate the relevant guides, owning specs, code entry points and related modules. [AGENTS.md](../../../AGENTS.md#finding-sources) @@ -38,18 +38,19 @@ Run as a **test-orchestration lead** (main agent) coordinating read-only **domai - **Specialists are read-only** — the lead is the sole writer. **Layout:** +- The `cases/_*.sh` helpers are not cases: never run one directly. - `../_e2e-lib/scripts/` — `preflight.sh`, `build-load.sh `, `deploy.sh ` (**installs only** — it refuses an existing release; upgrade in place with `helm upgrade`, or `teardown.sh` first), `assert-core.sh `, `teardown.sh ` (custom-resource drain, test artifacts, release uninstall and a CRD-drain verdict; the drain and the cleanup are both delegated to the chart's `files/drain.sh` and `files/cleanup.sh`), plus the cluster-lifecycle trio `cluster-auth.sh ` (read-only), `provision.sh ` and `destroy.sh `, and `kube-context.sh ` (read-only; targets a context that is not the current one). - `cases/case-N.sh ` — one scenario each; ends in a `STATUS | CHECK | OBJECT` table, exits non-zero on any FAIL. -- `cases/_quota-lib.sh` — sourced by the cases that short a pool by occupying it (50, 95): `workload_charge` reads what Kueue charged one replica's Workload, and `pool_replica_bound` turns it into an upper bound on the replicas the queue's quota holds, summed over every flavor. Not a case; never run it directly. -- `cases/_rows-lib.sh` — `print_rows`, the printer for that table. It splits a row on its first two `|` only, so an OBJECT that carries `|` prints whole; source it rather than writing another loop. Not a case; never run it directly. -- `cases/_partition-lib.sh` — sourced by the hardware-partition cases (24–32, 34) for node correlation, the `MIG_NODE_SSH` gate, profile/key discovery and pod plumbing. Not a case; never run it directly. +- `cases/_quota-lib.sh` — sourced by the cases that short a pool by occupying it (50, 95, 118): `workload_charge` reads what Kueue charged one replica's Workload, and `pool_replica_bound` turns it into an upper bound on the replicas the queue's quota holds, summed over every flavor. +- `cases/_rows-lib.sh` — `print_rows`, the printer for that table. It splits a row on its first two `|` only, so an OBJECT that carries `|` prints whole; source it rather than writing another loop. +- `cases/_partition-lib.sh` — sourced by the hardware-partition cases (24–32, 34) for node correlation, the `MIG_NODE_SSH` gate, profile/key discovery and pod plumbing. - `cases/run-partition-block.sh [NS] [CASES...]` — runs that family in its required order, writing each case's raw log and exit code. **Drive 24–32/34 with it rather than by hand**: the ordering is a real constraint and the block outlives a context, so a step held only in conversation is one a compaction drops. -- `cases/_rdma-lib.sh` — sourced by the RDMA cases (80–85) for the precondition gate, the host-probe Pod and the ledger facts. Not a case; never run it directly. +- `cases/_rdma-lib.sh` — sourced by the RDMA cases (80–85) for the precondition gate, the host-probe Pod and the ledger facts. - `cases/run-rdma-block.sh [--report-only] [NS] [CASES...]` — `--report-only` prints, without touching the cluster, which of those readings this cluster can answer and what each unmet requirement is missing. A run prints the same matrix, then each case's exit code **and whether its output carries `NOTHING WAS VERIFIED`** — which is how a skip is told from a pass, since both exit 0. -- `cases/_topology-tas-lib.sh` — sourced by the TAS cases (87) for the zone-shape measurement, the apply and name-scoped teardown of a case-prefixed Topology/ResourceFlavor/ClusterQueue/LocalQueue set, the Workload admission readers, and the Workload→owner-Pod→ModelDeployment join. Not a case; never run it directly. -- `cases/_model-hub-lib.sh` — sourced by the node-delivery cases (101–103, 107, 108, 109): deploys `cases/_model-hub.py` as a test Hugging Face hub (revision, tree, whoami and ranged resolve endpoints, with throttle, corrupt and deny switches and a request record) or as a forward proxy on the stock python image, and holds the Setting, consumer-Pod and plugin readers, a raw-watch recorder of one NodeModelStore's writes with their threshold statistics, and an impersonated reader of an artifact's `progress`. Not a case; never run it directly. -- `cases/_serving-lib.sh` — sourced by the serving cases (90, 91, 94) and delegates to `../_e2e-lib/scripts/serving-lib.sh` (shared with `gpustack-operator-benchmark`): `serving_profile ` is the one place a vendor or engine difference is written — where a transfer is counted, how the leg is pinned, which routers render it — so aligning a vendor is an edit there, not in each case. Not a case; never run it directly. -- `references/` — `drain-recycle.md` (per-case rationale + mock recipes), `packaged-image-deploy.md` (image-ref ↔ chart-values contract), `rdma-host-shapes.md` (which host shape answers which RDMA reading, and the one nothing answers), `kind-two-zones.yaml` (a local cluster of four workers, two per zone, for cases 87 and 107); shared `../_e2e-lib/references/{orchestration,troubleshooting}.md`. +- `cases/_topology-tas-lib.sh` — sourced by the TAS cases (87) for the zone-shape measurement, the apply and name-scoped teardown of a case-prefixed Topology/ResourceFlavor/ClusterQueue/LocalQueue set, the Workload admission readers, and the Workload→owner-Pod→ModelDeployment join. +- `cases/_model-hub-lib.sh` — sourced by the node-delivery cases (101–104, 107–111, 113, 114): deploys `cases/_model-hub.py` as a test Hugging Face hub (revision, tree, whoami and ranged resolve endpoints, with throttle, corrupt and deny switches and a request record) or as a forward proxy on the stock python image, and holds the Setting, consumer-Pod and plugin readers, a raw-watch recorder of one NodeModelStore's writes with their threshold statistics, and an impersonated reader of an artifact's `progress`. +- `cases/_serving-lib.sh` — sourced by the serving cases (91, 94, 129) and delegates to `../_e2e-lib/scripts/serving-lib.sh` (shared with `gpustack-operator-benchmark`): `serving_profile ` is the one place a vendor or engine difference is written — where a transfer is counted, how the leg is pinned, which routers render it — so aligning a vendor is an edit there, not in each case. +- `references/` — `drain-recycle.md` (per-case rationale + mock recipes), `packaged-image-deploy.md` (image-ref ↔ chart-values contract), `rdma-host-shapes.md` (which host shape answers which RDMA reading, and the one nothing answers), `manual-ssh-verification.md` (verifying on a host the cases cannot reach through kubectl), `kind-two-zones.yaml` (a local cluster of four workers, two per zone, for cases 87 and 107); shared `../_e2e-lib/references/{orchestration,troubleshooting}.md`. ## Cases (locked titles) @@ -98,12 +99,12 @@ Each case is self-contained; its header (see **Case header contract**) states go | 42 | Hygon DCU partitions: what a grant carries, what one card holds at once, and what is refused | `pkg/devicemanager/allocator/hygon/**`, `pkg/deviceplugin/**`, `pkg/device/physical_placement.go` | yes (confirm) | REAL Hygon DCU hardware on one node ALREADY in Multi-Instance mode + SSH to that node (`MIG_NODE_SSH`); auto-skips otherwise | | 43 | Two namespaces share one KV cache pool: proportional quota, and both backend preconditions fail loudly | `pkg/worker/controllers/worker/kv_cache_pool.go`, `pkg/worker/webhooks/worker/kv_cache_pool{,_binding}.go`, `pkg/worker/kvcache/**`, `api/worker/v1alpha1/kv_cache_pool{,_binding}.go` | yes (confirm) | any (no GPU, no RDMA) + a registry the cluster can pull the Mooncake image from | | 44 | What an effective quota enforces: a refusal when nothing can be evicted, an eviction of the domain's own objects when something can | `pkg/worker/kvcache/mooncake/tenant_metrics.go`, `pkg/worker/controllers/worker/kv_cache_pool.go` (the Binding pass) | yes (confirm) | CASE 43 passing + a registry the cluster can pull the Mooncake image from | -| 45 | The ModelDeployment admission surface: every refusal fires, and each one from the layer that owns it (schema / webhook / controller) | `pkg/worker/webhooks/worker/model_deployment.go`, `pkg/worker/controllers/worker/model_deployment{,_binding}.go`, `api/worker/v1alpha1/model_deployment.go` | yes (confirm) | any (no GPU), and deliberately **no** KVCachePoolBinding — the case names one that does not exist. Optionally `E2E_MD_INSTANCE_TYPE` to pick the InstanceType | +| 45 | The ModelDeployment admission surface: every refusal fires, and each one from the layer that owns it (schema / webhook / controller) | `pkg/worker/webhooks/worker/model_deployment.go`, `pkg/worker/controllers/worker/model_deployment{,_binding}.go`, `api/worker/v1alpha1/model_deployment.go` | yes | any (no GPU), and deliberately **no** KVCachePoolBinding — the case names one that does not exist. Optionally `E2E_MD_INSTANCE_TYPE` to pick the InstanceType | | 47 | Two ModelDeployments on one Binding are rendered one reuse domain and a third on another Binding a different one, and the rendered value is what the store partitions on. Uses **sglang** because its tenant is one environment variable on the Pod spec, while the vLLM family carries the same domain as `tenant_id` inside a projected client config (case 55 reads both carriers) | `pkg/worker/controllers/worker/model_deployment_connector.go`, `pkg/worker/controllers/worker/model_deployment_binding.go`, `pkg/worker/kvcache/inject/**` | yes (confirm) | any (no GPU, no engine image) + a registry the cluster can pull the Mooncake image from; the replicas are never expected to start, every asserted value is on the Pod spec | | 48 | The failure side: a Binding whose pool cannot be reached renders no connector AT ALL — measured against a control on a Ready Binding — and a rendered connector is never reported as an attached cache. Uses **vllm** deliberately: its connector travels on five separate carriers, so "no connector" is five independent absences and a PARTIAL render is visible. `CacheAttached != True` is true of every deployment on a cluster with no engine image, so no row asserts it alone | `pkg/worker/controllers/worker/model_deployment_binding.go` (`resolveModelDeploymentConnection`), `pkg/worker/controllers/worker/model_deployment_cache_attached.go`, `pkg/worker/kvcache/inject/vllm.go` | yes (confirm) | any (no GPU, no engine image) + a registry the cluster can pull the Mooncake image from + an InstanceType (`E2E_MD_INSTANCE_TYPE` to pick one); the replicas are never expected to start, and one row SKIPs — which shape the engine takes when the connector cannot initialize needs an image that serves | | 49 | Every replica of a multi-role ModelDeployment becomes its own Kueue Workload, and deleting the deployment completes | `pkg/worker/controllers/worker/model_deployment{,_pod_group}.go`, `api/worker/v1alpha1/model_deployment.go` | yes (confirm) | any (no GPU, no engine image) + an InstanceType and, in ``, the pool's entrance LocalQueue. The replicas are placeholders that never serve, so the two serving rows SKIP; deleting one replica's Workload is asserted to leave its sibling's Pod and Workload admitted, and SKIPs only when the group never reaches two replicas on two Workloads. Optionally `E2E_MD_INSTANCE_TYPE` / `E2E_MD_IMAGE` | | 50 | A short pool starves a P/D deployment WHOLE: the role that would have fit alone waits too | `pkg/worker/controllers/worker/model_deployment_pod_group.go`, `pkg/worker/controllers/worker/model_deployment.go` | yes (confirm) | as CASE 49, on a pool of any number of flavors whose queue is in no cohort. The shortage is made by OCCUPYING the quota, never by editing the ClusterQueue: a probe measures how many replicas Kueue reserves across every flavor, the filler leaves one, and the case SKIPs, printing the numbers, when the probe would exceed `E2E_C50_MAX_REPLICAS` (default 128) or the subject does not show exactly one role holding quota | -| 51 | Every P/D refusal fires, from the layer that owns it, and a replicas change replaces nobody | `pkg/worker/webhooks/worker/model_deployment.go`, `pkg/worker/controllers/worker/model_deployment{,_pod_group}.go`, `api/worker/v1alpha1/model_deployment.go` | yes (confirm) | as CASE 49; the two-instanceType row SKIPs unless a second InstanceType exists that a deployment can name (not inactive, not terminating, and holding a whole card if accelerated) | +| 51 | Every P/D refusal fires, from the layer that owns it, and a replicas change replaces nobody | `pkg/worker/webhooks/worker/model_deployment.go`, `pkg/worker/controllers/worker/model_deployment{,_pod_group}.go`, `api/worker/v1alpha1/model_deployment.go` | yes | as CASE 49; the two-instanceType row SKIPs unless a second InstanceType exists that a deployment can name (not inactive, not terminating, and holding a whole card if accelerated) | | 52 | The NIC/RDMA inventory is published, is stable across passes, and the `rdma.*` labels agree with it in both directions. A detector restart is driven but its outcome is not claimed — an unchanged record is also what a replacement that never reported produces, so that check SKIPs | `pkg/devicemanager/detector/{network,link,detector}.go`, `pkg/devicemanager/detector/network_linux.go`, `pkg/nodefeature/rdma.go`, `pkg/device/topology_distance.go`, `api/worker/v1alpha1/devices.go`, `binding/helper*.go` | yes (confirm) | any node running a device manager; the RDMA-positive checks skip individually without RDMA hardware | | 53 | The headline: a plain Deployment reads and writes a KV cache pool with one label | `pkg/worker/webhooks/worker/pod_kv_cache*.go`, `pkg/worker/kvcache/inject/**` | yes (confirm) | any (no GPU, no RDMA) + a registry the cluster can pull the Mooncake image from | | 54 | No workload lock-in: an LWS Pod and a bare Pod get the identical injection | `pkg/worker/webhooks/worker/pod_kv_cache*.go`, `pkg/worker/kvcache/inject/**` | yes (confirm) | as CASE 53; the LWS half needs the leaderworkerset.x-k8s.io CRD, and self-skips without it | @@ -139,8 +140,8 @@ Each case is self-contained; its header (see **Case header contract**) states go | 87 | Topology sources drive TAS placement and joint ModelDeployment admission | `pkg/worker/controllers/worker/topology_source*.go`, `pkg/worker/controllers/worker/node_{topology,flavor,queue}.go`, `pkg/worker/controllers/worker/model_deployment{,_joint_admission,_pod_group}.go`, `testing/infra/clusters/eks/**`, `deploy/gpustack-operator/chart/values.yaml` (Kueue's TAS feature gates) | yes (confirm) | at least three schedulable Linux Nodes carrying `topology.kubernetes.io/region` and `topology.kubernetes.io/zone`, no zone holding three of them and one zone holding at least two — four split two per zone is the usual shape (a Node that is tainted NoSchedule/NoExecute or carries no zone label, such as a kind control-plane, is left out of the count); each placeholder Pod requests sixty percent of the smallest counted Node, so two never share one; the operator chart with Kueue TAS enabled; an InstanceType and its entrance LocalQueue in ``; pull access for the case's HTTPS webhook image; no accelerator or engine image required. A cluster with no zoned Node prints one SKIP row and exits 0 | | 88 | A topology-aware ModelDeployment passes per-card admission on the accelerator Node Kueue assigned | `pkg/worker/controllers/worker/node_devices.go`, `node_devices_admission.go`, `node_{topology,flavor,queue}.go` | yes (confirm) | one Ready schedulable Node with a free real accelerator, an Active accelerated InstanceType and its entrance LocalQueue in ``; Kueue TAS enabled; pull access to the pause image; the Node carries `topology.kubernetes.io/zone` — the fixture requires that level, so on a zone-less cluster TAS finds zero slices and the case can never pass | | 89 | A ModelDeployment role's rendered Pod requests the selected fabric device | `api/worker/v1alpha1/model_deployment.go`, `pkg/worker/controllers/worker/model_deployment_{render,fabric}.go`, `pkg/worker/kvcache/mooncake/member_workload.go` (`FabricDeviceResource`), `pkg/worker/webhooks/worker/model_deployment.go` | no | an already reconciled ModelDeployment fixture for each requested count and protocol; inspect Pods even when Pending | -| 90 | A ModelDeployment metrics snapshot reflects served traffic and Pod endpoints | `pkg/worker/extensionapis/worker/model_deployment.metrics.go`, `pkg/worker/controllers/worker/model_deployment_{render,router}.go`, `pkg/worker/kvcache/router/**` | yes (confirm) | a serving ModelDeployment under representative traffic, and a pullable curl probe image; the probe Pod is removed by a trap | -| 91 | A routed prefill/decode pair moves real KV bytes over its direct leg, and writes the store when it has one | `pkg/worker/kvcache/inject/{vllm,sglang,types}.go`, `pkg/worker/controllers/worker/model_deployment_render.go`, `cases/_serving-lib.sh` | yes (confirm) | a Ready vLLM or SGLang P/D ModelDeployment with a router on real accelerators, one Pod per half on two nodes, with or without a bound store; on NVIDIA a tcp leg, vLLM 0.29.0 (an older CUDA image embeds a client without the pin and exits 2), and no role setting `MC_FORCE_TCP` or an SGLang cache switch; a store on tcp and on the client's minor line, and for SGLang a prefill node with more than 10 GiB of host memory to spare beyond its host pool; plus `E2E_PD_PROBE_IMAGE` and `E2E_PD_MODEL`; the probe Pod is removed by a trap | +| 90 | A ModelDeployment metrics snapshot reflects served traffic and Pod endpoints | `pkg/worker/extensionapis/worker/model_deployment.metrics.go`, `pkg/worker/controllers/worker/model_deployment_{render,router}.go`, `pkg/worker/kvcache/router/**` | yes | a serving ModelDeployment under representative traffic, and a pullable curl probe image; the probe Pod is removed by a trap | +| 91 | A routed prefill/decode pair moves real KV bytes over its direct leg, and writes the store when it has one | `pkg/worker/kvcache/inject/{vllm,sglang,types}.go`, `pkg/worker/controllers/worker/model_deployment_render.go`, `cases/_serving-lib.sh` | no | a Ready vLLM or SGLang P/D ModelDeployment with a router on real accelerators, one Pod per half on two nodes, with or without a bound store; on NVIDIA a tcp leg, vLLM 0.29.0 (an older CUDA image embeds a client without the pin and exits 2), and no role setting `MC_FORCE_TCP` or an SGLang cache switch; a store on tcp and on the client's minor line, and for SGLang a prefill node with more than 10 GiB of host memory to spare beyond its host pool; plus `E2E_PD_PROBE_IMAGE` and `E2E_PD_MODEL`; the probe Pod is removed by a trap | | 93 | A replica shed as surplus while it runs admitted releases Kueue's finalizer and takes its Workload with it, the replica beside it keeps its Pod and its Workload, and a twin in the seated group is removed without touching either — against a baseline showing an admitted Pod nobody releases stays held | `pkg/worker/controllers/worker/model_deployment{,_pod_group}.go` | yes (confirm) | any (no GPU, no engine image) + an InstanceType and, in ``, the pool's entrance LocalQueue; room for three single-replica placeholder Pods. Optionally `E2E_MD_INSTANCE_TYPE` / `E2E_MD_IMAGE`, `E2E_C93_HOLD` / `E2E_C93_BOUND` | | 94 | Scaling the router or a role under continuous traffic drops no request and puts the new Pod to work | `pkg/worker/controllers/worker/model_deployment{,_router,_pod_group}.go`, `pkg/worker/kvcache/router/**`, `cases/_serving-lib.sh` | yes (confirm) | a Ready routed ModelDeployment on real accelerators whose target (`router`, `prefill`, `decode` or `server`) runs one replica, room for one more, `E2E_PD_PROBE_IMAGE` and `E2E_PD_MODEL`; optionally `E2E_SCALE_READY_BOUND`, `E2E_SCALE_SETTLE`, `E2E_SCALE_LOG_DIR`; the trap restores the replica count and removes the probe Pod | | 95 | A change on a waiting replica's Workload alone reaches the deployment's QuotaReserved condition within a bounded window, with no Pod of the deployment moving — against a control showing a Pod event makes the status quote it | `pkg/worker/controllers/worker/model_deployment.go` (the `Watches(&kueue.Workload{})` clause, `mapModelDeploymentWorkload`, `modelDeploymentWorkloadPredicate`), `pkg/worker/controllers/worker/model_deployment_status.go` | yes (confirm) | any (no GPU, no engine image) + an InstanceType and, in ``, the pool's entrance LocalQueue; the case sizes one role one replica past the quota of every flavor of the pool summed, so the pool must hold that many placeholder Pods' worth of objects, and SKIPs past `E2E_C95_MAX_REPLICAS` (default 128). Optionally `E2E_MD_INSTANCE_TYPE` / `E2E_MD_IMAGE`, `E2E_C95_WINDOW` | @@ -170,9 +171,9 @@ Each case is self-contained; its header (see **Case header contract**) states go | 119 | The shared claim mounts the same files in the encoder's and the prefiller's Pods | `pkg/worker/controllers/worker/model_deployment_render.go` (the additionalVolumes render), `pkg/worker/webhooks/worker/model_deployment_epd.go` (the shared-claim rules), `api/worker/v1alpha1/model_deployment.go` (the claim source) | yes (confirm) | as CASE 116's fixture plus `E2E_C119_WITNESS_IMAGE` (default busybox:1.36) pullable for the runtime half — that row skips naming the image when it cannot pull; the case creates its own hostPath PV and PVC (removed by the trap), so no StorageClass is needed | | 120 | The admitted network rows run: llm-d-router + vLLM on NVIDIA CUDA or AMD ROCm with the NIXL encoder-cache leg, and a multimodal request is answered through the router | `pkg/worker/kvcache/inject/vllm_ec.go`, `pkg/worker/controllers/worker/model_deployment_connector_epd.go`, `pkg/worker/webhooks/worker/model_deployment_epd.go` (the matrix rows) | yes (confirm) | a pool of 2+ NVIDIA CUDA or AMD ROCm accelerators, an InstanceType (`E2E_C120_INSTANCE_TYPE`) reporting nvidia or amd with `E2E_C120_CARDS` (default 3) free exclusive cards, weights at a hostPath, a VISION-LANGUAGE model (a text-only model never engages the encoder stage, so every row would pass with the leg dormant), the matching engine image (default the CUDA runner on nvidia and the ROCm runner on amd, `E2E_C120_IMAGE`) and the llm-d-router image pullable, `E2E_C120_PROBE_IMAGE` pullable, and the pool's entrance LocalQueue in ``; auto-skips on any other platform, on too few free cards, and on an unpullable engine or router image (naming it — the router image is watched during the endpoint wait, because the router's own Deployment is what publishes the endpoint). The endpoint is polled for `E2E_C120_READY_BOUND` seconds: it is a PASS when it appears, a FAIL when the bound is spent with all three role Pods AND the router Pod Ready (the controller writes it once the router's own Deployment reports a ready replica), and a SKIP naming the missing Pods when the bound is spent without them. Overrides `model-deployment-router-image` and `model-deployment-routing-sidecar-image` in the `gpustack-settings` Secret, with the trap armed before the first patch; the restore happens before the deployment is deleted and is waited for on the router Deployment's image (`E2E_C120_SETTLE_BOUND`, default 180) | | 121 | The admitted Ascend row runs: llm-d-router + vLLM with the shared-directory encoder-cache leg over one ReadWriteMany claim, and a multimodal request is answered through the router | `pkg/worker/kvcache/inject/vllm_ec.go`, `pkg/worker/controllers/worker/model_deployment_connector_epd.go`, `pkg/worker/webhooks/worker/model_deployment_epd.go` (the matrix row and the claim rules) | yes (confirm) | real Ascend, an InstanceType (`E2E_C121_INSTANCE_TYPE`) reporting ascend with `E2E_C121_CARDS` (default 3) free exclusive cards, weights at a hostPath, a VISION-LANGUAGE model, the vllm-ascend and llm-d-router images and `E2E_C121_PROBE_IMAGE` pullable, the pool's entrance LocalQueue in ``, and an existing ReadWriteMany claim in `` (`E2E_C121_CLAIM`, required — this row's two ends are two Pods, and a ReadWriteOnce claim binds one node's disk); auto-skips on any other platform, on too few free cards, and on an unpullable image. A claim that is not Bound or does not offer ReadWriteMany is a SKIP ROW, not a failure: it is a storage answer about the cluster and nothing about the operator was exercised. The endpoint follows CASE 120's contract. The claim is the CALLER's and is never deleted; the router and sidecar image settings are overridden and restored as in CASE 120 | -| 122 | The admitted SGLang rows are admitted and the pairing the matrix does not carry is refused: llm-d-router + SGLang and sglang-gateway + SGLang, mounting no claim, are ACCEPTED; sglang-gateway + vLLM is REFUSED naming the router AND the engine it is not paired with | `pkg/worker/controllers/worker/model_deployment_connector_epd.go` (the six SGLang rows), `pkg/worker/webhooks/worker/model_deployment_epd.go` (the matrix gap and the floor project) | yes (confirm) | any cluster with a materialized scheduling chain; if an active InstanceType already reports `E2E_C122_VENDOR` (default `Ascend`; also supports `nvidia` or `amd`) it is used directly without mutating node labels or settings, otherwise the case injects the CPU vendor label and flips CPU-manufacturer awareness on as in CASE 116. Every assertion is a server-side dry run, so nothing is stored. Auto-skips the platform rows when no InstanceType for the target manufacturer can be resolved | -| 123 | vllm-router admits vLLM encoder deployments on NVIDIA and AMD, refuses SGLang pairings, refuses user-supplied `--epd-config` as operator-owned, and admits plain prefill/decode without an encoder | `pkg/worker/controllers/worker/model_deployment_connector_epd.go` (the two vllm-router rows), `pkg/worker/webhooks/worker/{model_deployment,model_deployment_epd}.go` | yes (confirm) | any cluster with a materialized scheduling chain; if an active InstanceType already reports `nvidia` or `amd` it is used directly without mutating node labels or settings, otherwise the case injects `E2E_C123_VENDOR` (default `NVIDIA`) and flips CPU-manufacturer awareness on as in CASE 116. Every assertion is a server-side dry run | -| 124 | The matrix admits ascend, nvidia and amd only: the refusal names the platform beside the router and engine it accepted and the admitted rows, a second refusal names the engine instead, and two roles on two platforms are refused by the mixing rule | `pkg/worker/webhooks/worker/model_deployment_epd.go` (the matrix gap and the mixed-platform rule), `pkg/nodefeature/knowns.go` (the declared manufacturers), `pkg/worker/controllers/worker/model_deployment_connector_epd.go` (the admitted rows) | yes (confirm) | any cluster with a materialized scheduling chain and at least one usable InstanceType whose reported manufacturer is one this operator declares and the matrix does not admit — it iterates every declared accelerator manufacturer minus ascend, nvidia and amd (cambricon, hygon, iluvatar, metax, mthreads, thead) and takes the first usable type, so no supported accelerator is left out by a guess; the accepted baseline shape follows whichever admitted platform it finds, carrying the shared-claim mounts on ascend and none on nvidia or amd (the network rows forbid the shared mount); auto-skips the unadmitted rows naming the manufacturers it looked for and the types it saw on a cluster of admitted platforms only. MOCKS nothing — no injected label, no settings change, no cleanup | +| 122 | The admitted SGLang rows are admitted and the pairing the matrix does not carry is refused: llm-d-router + SGLang and sglang-gateway + SGLang, mounting no claim, are ACCEPTED; sglang-gateway + vLLM is REFUSED naming the router AND the engine it is not paired with | `pkg/worker/controllers/worker/model_deployment_connector_epd.go` (the six SGLang rows), `pkg/worker/webhooks/worker/model_deployment_epd.go` (the matrix gap and the floor project) | no | any cluster with a materialized scheduling chain; if an active InstanceType already reports `E2E_C122_VENDOR` (default `Ascend`; also supports `nvidia` or `amd`) it is used directly without mutating node labels or settings, otherwise the case injects the CPU vendor label and flips CPU-manufacturer awareness on as in CASE 116. Every assertion is a server-side dry run, so nothing is stored. Auto-skips the platform rows when no InstanceType for the target manufacturer can be resolved | +| 123 | vllm-router admits vLLM encoder deployments on NVIDIA and AMD, refuses SGLang pairings, refuses user-supplied `--epd-config` as operator-owned, and admits plain prefill/decode without an encoder | `pkg/worker/controllers/worker/model_deployment_connector_epd.go` (the two vllm-router rows), `pkg/worker/webhooks/worker/{model_deployment,model_deployment_epd}.go` | yes | any cluster with a materialized scheduling chain; if an active InstanceType already reports `nvidia` or `amd` it is used directly without mutating node labels or settings, otherwise the case injects `E2E_C123_VENDOR` (default `NVIDIA`) and flips CPU-manufacturer awareness on as in CASE 116. Every assertion is a server-side dry run | +| 124 | The matrix admits ascend, nvidia and amd only: the refusal names the platform beside the router and engine it accepted and the admitted rows, a second refusal names the engine instead, and two roles on two platforms are refused by the mixing rule | `pkg/worker/webhooks/worker/model_deployment_epd.go` (the matrix gap and the mixed-platform rule), `pkg/nodefeature/knowns.go` (the declared manufacturers), `pkg/worker/controllers/worker/model_deployment_connector_epd.go` (the admitted rows) | no | any cluster with a materialized scheduling chain and at least one usable InstanceType whose reported manufacturer is one this operator declares and the matrix does not admit — it iterates every declared accelerator manufacturer minus ascend, nvidia and amd (cambricon, hygon, iluvatar, metax, mthreads, thead) and takes the first usable type, so no supported accelerator is left out by a guess; the accepted baseline shape follows whichever admitted platform it finds, carrying the shared-claim mounts on ascend and none on nvidia or amd (the network rows forbid the shared mount); auto-skips the unadmitted rows naming the manufacturers it looked for and the types it saw on a cluster of admitted platforms only. MOCKS nothing — no injected label, no settings change, no cleanup | | 125 | The llm-d-router picker carries the encoder stage: an encode filter, an unscored encode profile, the multimodal decider, and an event producer whose `podLabelSelector` excludes both eventless kinds — read against a control deployment with no encoder | `pkg/worker/kvcache/router/router.go` (the encode filter, profile, decider and exclusion), `pkg/worker/controllers/worker/model_deployment_connector_epd.go` | yes (confirm) | as CASE 116's fixture plus the pool's entrance LocalQueue in `` (without it nothing is admitted and no router document is rendered), and `pyyaml` in the runner to read the router ConfigMap. The case creates its own hostPath PV and PVC (removed by the trap). Waits for each deployment's own role count of Pods — all three for the encoder, two for the control. An unpullable ROUTER image does not skip the rows — they read the ConfigMap, which the operator renders before any image is pulled | | 126 | A ModelDeployment without `engine.image` is refused by the API server's schema before any webhook runs, in either measured wording: "Required value" while the key is absent, "Invalid value" once the defaulting patch materializes it empty; the accepted baseline proves the refusal is about the missing field, and a role image does not substitute for it | `api/worker/v1alpha1/model_deployment.go` (the required engine image), `cases/case-126.sh` | no | any cluster with the ModelDeployment CRD installed; every assertion is a server-side dry run; auto-skips the accepted-baseline row when no usable InstanceType reports an admitted platform | | 127 | A managed role's Pod mounts its own subdirectory of the host compilation cache at one constant in-container path, is pointed at it by the engine's cache-root variable, and is given a startup gate scaled by the budget the role declares, while a replica still inside its startup window is held out of `Revoked`; schema validation bounds `startupTimeoutSeconds` to `[10, 86400]`, and admission allows a user `additionalVolumes` mount at the compilation cache path (yielding the default `HostPath` volume) | `pkg/worker/controllers/worker/model_deployment_compile_cache.go`, `pkg/worker/controllers/worker/model_deployment_render.go` (the cache mount, user volume yield, and startup-gate scaling), `pkg/worker/controllers/worker/model_deployment_health.go`, `api/worker/v1alpha1/model_deployment.go` (`startupTimeoutSeconds`), `cases/case-127.sh` | yes (confirm) | any cluster with the ModelDeployment CRD installed for the admission rows (server-side dry runs); a usable InstanceType and the pool's entrance LocalQueue in `` for the rendered-Pod rows, else they auto-skip; the warm-restart rows additionally `E2E_C127_ENGINE_IMAGE` and `E2E_C127_MODEL` on a real accelerator pool, plus a pullable `E2E_C127_PROBE_IMAGE` (default busybox) and a namespace whose PodSecurity admits a hostPath probe Pod, else they skip | @@ -327,7 +328,7 @@ Each note below is something the **lead** must act on before or around a run. Wh ## Case header contract -Every `cases/case-N.sh` opens with a header describing the case **on its own terms — no spec anchors** (no Story/Task numbers, F-codes, `specs/*.md` paths, or commit hashes). A reader must understand what the case does, needs, and asserts from the header alone. New/edited cases follow this six-field template: +Every `cases/case-N.sh` opens with a header describing the case **on its own terms — no spec anchors** (no Story/Task numbers, F-codes or `specs/*.md` paths; no commit hashes from repository history — an input artifact's own revision identifier, such as a weight revision under test, is an input, not an anchor). A reader must understand what the case does, needs, and asserts from the header alone. New/edited cases follow this six-field template: ``` # CASE N — () @@ -343,7 +344,7 @@ Every `cases/case-N.sh` opens with a header describing the case **on its own ter # pass AND fail, safe to re-run> ``` -- **Mutation posture** rides the title line: `READ-ONLY` (no trap) or `MUTATING, self-recovering` (writes, then a trap restores). Append `AUTO-SKIPS without ` when the case self-skips (real hardware, `ssh` client, ≥2 logically sliceable cards, …). +- **Mutation posture** rides the title line, and every case's title line carries one: `MUTATING, self-recovering` (writes, then a trap restores), `MUTATING, self-cleaning` (writes, then removes what it wrote), `NON-MUTATING` (creates nothing outside its own report tree) or `READ-ONLY` (no trap). Append `AUTO-SKIPS without ` when the case self-skips (real hardware, `ssh` client, ≥2 logically sliceable cards, …). - **Environment** always names the auto-skip condition so the run/skip decision is readable without executing the case. - **Inputs** distinguish **MOCKED** inputs (fake accelerator NodeFeature, mocked `Devices` ledger) from the **real** objects under test — the mock is a fixture, never the verification. - Describe behavior in **plain terms**, not spec reference; the case↔code mapping lives in the table's "Run when these change" column. @@ -352,7 +353,7 @@ Every `cases/case-N.sh` opens with a header describing the case **on its own ter ## Flow -Generic phase semantics (roles, rendezvous, report, fix-retest) live in [`orchestration.md`](../_e2e-lib/references/orchestration.md); below are the operator-e2e specifics. Let `RPT=.claude/reports/$(date +%F)-operator-e2e`. +Generic phase semantics live in [`orchestration.md`](../_e2e-lib/references/orchestration.md); below are the operator-e2e specifics. Let `RPT=.claude/reports/$(date +%F)-operator-e2e`. **Phase 0 — Environment (read-only; provisioning only on opt-in).** Preflight, offer the user's own contexts, confirm the target, record drift: @@ -379,7 +380,7 @@ bash .claude/skills/_e2e-lib/scripts/provision.sh -var='= # diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-116.sh b/.agents/skills/gpustack-operator-e2e/cases/case-116.sh index 0f2133929..15f80b89a 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/case-116.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-116.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # # CASE 116 — Every encoder-role admission refusal fires, each naming what its rule refuses -# (MUTATING for the fixture only: one settings key and one injected node label, both restored by +# (MUTATING, self-recovering: one settings key and one injected node label, both restored by # the trap; every verdict below is a server-side dry run that persists nothing) # AUTO-SKIPS its matrix rows without an InstanceType that reports an admitted accelerator # platform, which this case builds by injecting a CPU vendor label. diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-123.sh b/.agents/skills/gpustack-operator-e2e/cases/case-123.sh index d26bc2f6b..2f59cce03 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-123.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-123.sh @@ -2,7 +2,8 @@ # # CASE 123 — vllm-router admits vLLM encoder deployments on NVIDIA and AMD, refuses unadmitted # engines and user-supplied --epd-config -# (NON-MUTATING on GPU clusters; fixture-mutating on CPU-only clusters, restored by trap) +# (MUTATING, self-recovering: the platform fixture on CPU-only clusters, restored by +# the trap — GPU clusters need no fixture, and every verdict is a server-side dry run) # AUTO-SKIPS the platform rows without an InstanceType reporting nvidia or amd # # case-123.sh diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-18.sh b/.agents/skills/gpustack-operator-e2e/cases/case-18.sh index b2113cc57..24f440e47 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-18.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-18.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# CASE 18 — CPU-manufacturer awareness reshapes the catalog (MUTATING: flips an editable setting, self-restoring) +# CASE 18 — CPU-manufacturer awareness reshapes the catalog (MUTATING, self-recovering: flips an editable setting) # # case-18.sh # diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-42.sh b/.agents/skills/gpustack-operator-e2e/cases/case-42.sh index a48420f56..325e44198 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-42.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-42.sh @@ -51,6 +51,9 @@ # one-slice partitions requested together, then one two-accelerator request expected to # be refused. # Verdicts: One row per contract above, plus the setup gates. Any FAIL fails the case. +# Expected: Every contract in the Goal holds from both sides — the operator's records and the +# container or the vendor's own registry — and the table names both sides per row. +# Cleanup: An EXIT trap deletes the case's Pods and waits for their release. set -uo pipefail CASE_ID=42 diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-45.sh b/.agents/skills/gpustack-operator-e2e/cases/case-45.sh index 51456cbd3..366e547f1 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/case-45.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-45.sh @@ -2,8 +2,8 @@ # # CASE 45 — The ModelDeployment admission surface: every refusal fires, and each one fires from the # layer that is supposed to own it -# (NON-MUTATING for the refusals, which are server-side dry-runs; one short-lived object for the -# controller-level check) +# (MUTATING, self-cleaning: the refusals are server-side dry-runs; one short-lived object +# serves the controller-level check) # # case-45.sh # @@ -74,6 +74,9 @@ # the behaviour is missing, but that asserting it would not DISCRIMINATE. See the row # about what the unregistered domain costs, near the end. # +# Expected: Every refusal fires from the layer the Goal names, the field and message asserted +# per row; the created-object row observes creation before it judges. +# Cleanup: An EXIT trap deletes the one short-lived deployment (case45-nobind). set -o pipefail NS="${1:-}" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-47.sh b/.agents/skills/gpustack-operator-e2e/cases/case-47.sh index eb9d97b66..c6d4fe107 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/case-47.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-47.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # # CASE 47 — Two deployments on one Binding share a reuse domain; two on different Bindings do not -# (MUTATING: three ModelDeployments, one extra Binding and one probe Pod, all in the fixture's -# namespace, all removed by the shared teardown) +# (MUTATING, self-cleaning: three ModelDeployments, one extra Binding and one probe Pod, all +# removed by the shared teardown) # # case-47.sh # @@ -43,6 +43,12 @@ # image from. No GPU, no engine image. # Needs: a live Mooncake store (the shared fixture brings up backend, pool and Bindings) # +# Inputs: All real, nothing mocked: the fixture's pool and Bindings, one additional Binding in +# the same namespace, and the engine image the deployments render. +# Expected: Deployments on the same Binding render the same reuse domain and the third +# deployment a different one; the probe writes land under the rendered values. +# Cleanup: The EXIT teardown deletes the three deployments, the extra Binding and the probe +# Pod; the shared fixture's teardown covers the rest. set -uo pipefail NS="${1:?usage: case-47.sh }" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-48.sh b/.agents/skills/gpustack-operator-e2e/cases/case-48.sh index 03f3e999b..34f5185ca 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/case-48.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-48.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # # CASE 48 — The deliberate break: what a deployment looks like when the cache cannot work -# (MUTATING: one external KVCacheBackend and one KVCachePool, both CLUSTER-SCOPED, plus one -# KVCachePoolBinding and two ModelDeployments in the fixture's namespace — all removed here) +# (MUTATING, self-cleaning: one external KVCacheBackend and one KVCachePool, both +# cluster-scoped, plus one KVCachePoolBinding and two ModelDeployments — all removed here) # # case-48.sh # @@ -47,6 +47,13 @@ # Needs: a live Mooncake store (the shared fixture brings up backend, pool and Bindings) and an # InstanceType, for the role to reference # +# Inputs: All real, nothing mocked: the fixture's pool and Bindings beside the case's own +# unreachable pool, and the engine image the deployments render. +# Expected: The unreachable pool renders no connector onto its deployment while the reachable +# pool's image still gets its render; every CacheAttached row pairs the condition +# with a fact not true of everything. +# Cleanup: The EXIT teardown deletes the namespace objects and both cluster-scoped objects +# the case created. set -uo pipefail NS="${1:?usage: case-48.sh }" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-51.sh b/.agents/skills/gpustack-operator-e2e/cases/case-51.sh index 7313143a4..58c1aab5a 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-51.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-51.sh @@ -2,8 +2,8 @@ # # CASE 51 — Every P/D refusal fires, from the layer that owns it, and a replicas change replaces # nobody -# (NON-MUTATING for the refusals, which are server-side dry-runs; one short-lived deployment for -# the convergence rows) +# (MUTATING, self-cleaning: the refusals are server-side dry-runs; one short-lived deployment +# serves the convergence rows) # # case-51.sh # @@ -43,6 +43,9 @@ # and whether a short pool leaves both roles queued is case-50's. This file stops at # the operator's own writes, which is what it can assert without an engine. # +# Expected: Each refusal names its owning layer and message; the replicas change converges +# without replacing the replica already running. +# Cleanup: An EXIT trap deletes the short-lived deployment and its groups' Workloads. set -o pipefail NS="${1:-}" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-68.sh b/.agents/skills/gpustack-operator-e2e/cases/case-68.sh index 92c033e93..195ca165d 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-68.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-68.sh @@ -83,6 +83,8 @@ # against one object would be asserting one of them against a state that cannot produce it. Phase E # needs a THIRD object for the same reason: Phase C deactivates the deployment it runs on, and a # deactivated workload is never admitted again whatever the cluster does. +# Cleanup: An EXIT trap deletes the three ModelDeployments and the held InstanceType the case +# created. set -uo pipefail # Route every kubectl through the retrying shim. Against a remote API endpoint a read can fail on diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-70.sh b/.agents/skills/gpustack-operator-e2e/cases/case-70.sh index f2daa8c7f..64ace7c81 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-70.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-70.sh @@ -6,6 +6,9 @@ # # case-70.sh # +# Goal: A routed deployment owns exactly one complete router object set through both +# router transitions, and its status reports every cache-event publication state +# observable from the cluster. # Environment: A single-node cluster with no accelerator, the operator deployed, the general # InstanceType materialized by CASE 1, and pull access to the Mooncake fixture image. # No serving engine image is required: the assertions read rendered objects, Pod @@ -18,6 +21,11 @@ # garbage collection remove all six. The deployment remains Starting with no ready # role replicas. KVEventsPublishing reaches NotApplicable, NoRouter, # PublisherDisabled, RoleUnmanaged and Publishing, with status and reason asserted. +# Inputs: All real: the operator's charts and images, the Mooncake fixture image and a Ready +# Binding. No serving engine image — the assertions read rendered objects, Pod +# specifications and status. +# Cleanup: The teardown releases any finalizer-held Pods it created and deletes the lifecycle +# objects. set -uo pipefail E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-71.sh b/.agents/skills/gpustack-operator-e2e/cases/case-71.sh index 1f0604796..896eba26f 100755 --- a/.agents/skills/gpustack-operator-e2e/cases/case-71.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-71.sh @@ -8,6 +8,16 @@ # This case is separate from CASE 70 because the upstream router images may be unavailable. An image # pull failure is a stated SKIP. Any other failure to become Ready is a FAIL. No accelerator or # serving engine is needed: the router runs outside Kueue and discovers the role Pods dynamically. +# Goal: The router endpoint publishes only once the router Deployment has a ready replica — +# the readiness contract status.endpoint carries. +# Environment: A single-node cluster with no accelerator, the operator deployed, the general +# InstanceType from CASE 1, and pull access to a router image. +# Inputs: All real: the router image is pulled from its registry; no accelerator and no +# serving engine image is needed. +# Expected: With the image pullable, the deployment publishes its endpoint once the router +# replica is Ready and RouterReady reads True/Ready. An image pull failure is the +# stated SKIP; any other failure to become Ready is a FAIL. +# Cleanup: An EXIT trap deletes the deployment and any Pods left under it. set -uo pipefail E2E_SHIM_DIR="$(cd "$(dirname "$0")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)" diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-89.sh b/.agents/skills/gpustack-operator-e2e/cases/case-89.sh index 642356160..3589dd4b8 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/case-89.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-89.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # # CASE 89 — A ModelDeployment role's rendered Pod requests the selected fabric device. +# (NON-MUTATING; E2E_REJECT_MANIFEST, when set, is a server-side dry run) # # case-89.sh # diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-90.sh b/.agents/skills/gpustack-operator-e2e/cases/case-90.sh index 3e9c46252..c57d082d5 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/case-90.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-90.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # # CASE 90 — A ModelDeployment metrics snapshot reflects served traffic and Pod endpoints. +# (MUTATING, self-cleaning: one temporary probe Pod, deleted by the trap) # # case-90.sh # diff --git a/.agents/skills/gpustack-operator-e2e/cases/case-91.sh b/.agents/skills/gpustack-operator-e2e/cases/case-91.sh index 7d68f80e7..6fe833e68 100644 --- a/.agents/skills/gpustack-operator-e2e/cases/case-91.sh +++ b/.agents/skills/gpustack-operator-e2e/cases/case-91.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # # CASE 91 — A routed prefill/decode pair moves real KV bytes over its direct leg, and writes the -# store when it has one. +# store when it has one. (NON-MUTATING) # # case-91.sh # diff --git a/docs/modules/model-deployment/routing.md b/docs/modules/model-deployment/routing.md index 324260b87..6ed5d85ad 100644 --- a/docs/modules/model-deployment/routing.md +++ b/docs/modules/model-deployment/routing.md @@ -34,7 +34,7 @@ pairing: | Mode | `llm-d-router` | `vllm-router` | `sglang-gateway` | |---|---|---|---| | Multi-replica *Server* | One fixed scoring profile — prefix match, queue depth, KV utilization — with no policy switch; measured 187/170 on vLLM and 375/0, every request on one replica, on SGLang | `cache_aware` by default; `--policy round_robin` spreads evenly, measured 60 per replica | The same policies as `vllm-router`; `round_robin` measured 60 per replica | -| *PD* pair | The disaggregated pairing's own router: the token-counted prefill/decode split, per-role connector documents, the decode sidecar | Its own *PD* mode over the pair | Native `--pd-disaggregation` | +| *PD* pair | The disaggregated pairing's own router: the token-counted prefill/decode split, per-role connector documents, the decode sidecar | Its own *PD* mode over the pair | Native `--pd-disaggregation`; combining it with `--dp-aware` produced an invalid *decode* destination in the recorded container run, so the operator never renders the two together | | *EPD* | An `encode` scheduling profile beside the pair; the decode sidecar coordinates the encoder transfer | Drives encode, prefill and decode itself through operator-owned `--epd-config`, no sidecar | Routes the pair natively; the prefiller reaches the encoder through `--encoder-urls` | | *Elastic EP* | The engine's own internal load balancing spreads work inside the collective, and the deployment publishes one endpoint, so a router is not how this mode balances | Same | Same | | *External DP* | Admitted, but no `--policy` to spread the ranks evenly and no recorded run — use the engine's own router | Every rank endpoint behind `--policy round_robin` (recorded) | Per-member endpoints; the registration boundary is the paragraph below | @@ -63,10 +63,10 @@ carries a rank suffix (`http://host:port@0`, `http://host:port@1`), and every su member's one physical serving listener — the suffix is a rank, not a port. That registration is the gateway patch this project ships: an image carrying it is -`gpustack/llm-router:v0.4.0` or later, while the default -[`model-deployment-router-image`](../../reference/settings.md) still names `v0.3.1`, which -predates the patch. The registration activates only where *`spec.router.image`* or the setting -names an image that carries the patch. +`gpustack/llm-router:v0.4.0` or later, which is also the default +[`model-deployment-router-image`](../../reference/settings.md), so a deployment that names no +image of its own registers per rank out of the box. An older default or a named image that +predates the patch runs the gateway without the fan-out. No runtime run has exercised the flag: the render gate and the patched fan-out are verified in source and unit tests on the recorded shape. diff --git a/docs/reference/settings.md b/docs/reference/settings.md index fb1e7e9dc..7347ba18b 100644 --- a/docs/reference/settings.md +++ b/docs/reference/settings.md @@ -167,7 +167,7 @@ request that reaches a webhook. Neither window needs a restart. | `model-artifact-no-proxy`
(`GPUSTACK_MODEL_ARTIFACT_NO_PROXY`) | Effect window: Next trigger
Default: *(blank)*
Usage: Comma-separated hosts that bypass `model-artifact-https-proxy`, for the node plugin too, given to engines as `NO_PROXY` on the same terms. | | `model-artifact-revalidate-interval`
(`GPUSTACK_MODEL_ARTIFACT_REVALIDATE_INTERVAL`) | Effect window: Next trigger
Default: `24h`
Usage: How often a resolved hub artifact's access is checked again, at least `1m`; a `Secret` change checks it at once. What a refusal does is under [Resolution and revalidation](../modules/model-delivery/artifact.md#resolution-and-revalidation). | | `model-deployment-kv-cache-dtype-owned`
(`GPUSTACK_MODEL_DEPLOYMENT_KV_CACHE_DTYPE_OWNED`) | Effect window: Next trigger
Default: `true`
Usage: Hand every `KVCachePoolBinding`'s *`spec.domain.dtype`* to the engines attached through it as `--kv-cache-dtype`, verbatim, and refuse a pool-attached role or injected `Pod` naming that flag itself, and a new Binding declaring `auto` — see [The dtype is handed to the engine](../modules/kv-cache/pool.md#engine-dtype). **It is the escape from a Binding whose spelling the engine rejects**, which makes every new `Pod` fail argument parsing: `false` renders and refuses nothing, as before the flag was owned. Flipping it recreates every pool-attached replica at its next reconcile. The recovery is under [Upgrading to an enforced Binding dtype](../operate/migration/kv-cache-dtype.md). | -| `model-deployment-router-image`
(`GPUSTACK_MODEL_DEPLOYMENT_ROUTER_IMAGE`) | Effect window: Next trigger
Default: `gpustack/llm-router:v0.3.1`
Usage: Image a managed router runs when the `ModelDeployment` does not name one in *`spec.router.image`*. **It carries every router this operator supports, and which binary runs is decided by the rendered command rather than by the image** — *`spec.router.name`* picks the binary, and this setting only says where the binaries come from. Its default is safe for every cluster at once in a way `kv-cache-backend-image`'s is not: it runs no model, so it links no accelerator runtime and cannot be paired with the wrong one. The default is this project's own build rather than an upstream tag, because the three routers are compiled from three separate sources and one of them carries a patch this repository ships, so no upstream image holds them together. An image named on the object is used verbatim and is never redirected to the cluster mirror, because it is the user's own reference. | +| `model-deployment-router-image`
(`GPUSTACK_MODEL_DEPLOYMENT_ROUTER_IMAGE`) | Effect window: Next trigger
Default: `gpustack/llm-router:v0.4.0`
Usage: Image a managed router runs when the `ModelDeployment` does not name one in *`spec.router.image`*. **It carries every router this operator supports, and which binary runs is decided by the rendered command rather than by the image** — *`spec.router.name`* picks the binary, and this setting only says where the binaries come from. Its default is safe for every cluster at once in a way `kv-cache-backend-image`'s is not: it runs no model, so it links no accelerator runtime and cannot be paired with the wrong one. The default is this project's own build rather than an upstream tag, because the three routers are compiled from three separate sources and one of them carries a patch this repository ships, so no upstream image holds them together. An image named on the object is used verbatim and is never redirected to the cluster mirror, because it is the user's own reference. | | `model-deployment-router-proxy-image`
(`GPUSTACK_MODEL_DEPLOYMENT_ROUTER_PROXY_IMAGE`) | Effect window: Next trigger
Default: `gpustack/mirrored-envoy:distroless-v1.33.2`
Usage: Proxy fronting a managed router's endpoint picker. It has **no field on the API** to override it: this operator renders the proxy's configuration against one proxy's configuration schema, so swapping the binary would mean swapping that configuration too. The setting exists for registry redirection and for pinning a release back, not for running a different proxy. | | `model-deployment-routing-sidecar-image`
(`GPUSTACK_MODEL_DEPLOYMENT_ROUTING_SIDECAR_IMAGE`) | Effect window: Next trigger
Default: `gpustack/mirrored-llm-d-router-disagg-sidecar:v0.11.0`
Usage: Sidecar a decoder runs to accept a remote *prefill* handoff. Same terms as the proxy above: this operator renders its arguments, so the setting is for redirection and pinning rather than for a different implementation. | | `model-deployment-tcp-tw-reuse`
(`GPUSTACK_MODEL_DEPLOYMENT_TCP_TW_REUSE`) | Effect window: Next trigger
Default: `false`
Usage: Render `net.ipv4.tcp_tw_reuse=1` on the *prefill* half of every SGLang *prefill*/*decode* pair, which otherwise [runs out of local ports](../modules/model-deployment/engine-versions.md#limitations) under sustained load. **Allow the sysctl on the kubelet of every node that can run such a `Pod` before turning it on**: without that the `Pod` fails with `SysctlForbidden`, and so does every replacement. The steps, which `Pod`s it reaches and how to confirm the refusal are under [SGLang TIME-WAIT reuse](#sglang-time-wait-reuse). | diff --git a/pkg/worker/controllers/worker/model_deployment_router_observation.go b/pkg/worker/controllers/worker/model_deployment_router_observation.go index e3602aeaa..610698c11 100644 --- a/pkg/worker/controllers/worker/model_deployment_router_observation.go +++ b/pkg/worker/controllers/worker/model_deployment_router_observation.go @@ -1169,6 +1169,21 @@ func modelDeploymentGroupFact( expected[member.UID] = true } + // THE RETENTION FACT: every answering member carries the eligibility label, and the eligibility + // annotation on those members names the group's CURRENT complete membership generation. It is + // proved by the fresh membership rather than by the view, so it is computed before the view is + // judged: a partial view is an observation loss about half the group, and the retention is + // exactly what carries an already-eligible group of an unchanged generation through one. + fact.Retained = true + for _, member := range answering { + carried, known := members[member.UID] + if !known || !carried.Eligible || carried.Generation != generation { + fact.Retained = false + + break + } + } + // THE ENDPOINTS THE ROUTER HOLDS FOR THIS GROUP, against the ones it is expected to hold. observed := make(map[types.UID]bool, len(view.Members)) endpoints := int32(0) @@ -1210,20 +1225,6 @@ func modelDeploymentGroupFact( "the router's view of the group is partial: it holds %d of the group's %d expected "+ "endpoints, so the group's serving cannot be confirmed", heldExpected, len(expected)) - - return fact - } - - // THE RETENTION FACT: every answering member carries the eligibility label, and the eligibility - // annotation on those members names the group's CURRENT complete membership generation. - fact.Retained = true - for _, member := range answering { - carried, known := members[member.UID] - if !known || !carried.Eligible || carried.Generation != generation { - fact.Retained = false - - break - } } return fact diff --git a/pkg/worker/controllers/worker/model_deployment_router_observation_test.go b/pkg/worker/controllers/worker/model_deployment_router_observation_test.go index 37501b290..af5c0e694 100644 --- a/pkg/worker/controllers/worker/model_deployment_router_observation_test.go +++ b/pkg/worker/controllers/worker/model_deployment_router_observation_test.go @@ -3527,6 +3527,65 @@ func TestModelDeploymentServingGroups(t *testing.T) { assert.False(t, fact.SnapshotValid, "a group seen half is not a group seen") assert.Equal(t, workercore.ModelDeploymentServingStateUnknown, fact.State) assert.Contains(t, fact.InvalidReason, "partial") + assert.False(t, fact.Retained, "no member carries the eligibility label yet") + }) + + // THE RECORDED WEDGE: a gateway that registers one rank of a two-rank group leaves the + // survivor's view permanently partial, and the partial branch once returned before the + // retention was computed, so an eligible group of an unchanged generation was held + // AwaitingObservation forever. The retention is proved by the fresh membership, so it is + // computed before the view is judged and carries such a group through the half view. + t.Run("a partial view of a labeled group retains its eligibility", func(t *testing.T) { + t.Parallel() + + generation := modelDeploymentMembershipGeneration("uid-a/uid-b") + labeled := func(uid types.UID, seat int) *corev1.Pod { + pod := servingGroupMember("regular", 0, seat, 2, uid, true) + pod.Labels[modelDeploymentLabelKeyEndpointEligible] = modelDeploymentEndpointEligibleValue + pod.Annotations[modelDeploymentAnnotationKeyEndpointEligibleGeneration] = string(generation) + + return pod + } + + groups, _ := servingGroupFacts(md, []*corev1.Pod{labeled("uid-a", 0), labeled("uid-b", 1)}, + []RouterServingObservation{servingObservation("uid-a")}) + + fact, found := groups[modelDeploymentGroupKey{ + Role: "regular", Ordinal: 0, Generation: generation, + }] + require.True(t, found, "a partial view still yields an entry") + assert.False(t, fact.SnapshotValid, "the view is still half, and that fact is published") + assert.Equal(t, workercore.ModelDeploymentServingStateUnknown, fact.State) + assert.Contains(t, fact.InvalidReason, "partial") + assert.True(t, fact.Retained, + "every answering member carries the label at the current generation, so the retention holds") + assert.True(t, modelDeploymentGroupRestores(true, true, fact), + "a retained group restores through a partial view instead of waiting on one forever") + }) + + t.Run("a partial view of a group whose generation moved does not retain", func(t *testing.T) { + t.Parallel() + + pod := servingGroupMember("regular", 0, 0, 2, "uid-a", true) + pod.Labels[modelDeploymentLabelKeyEndpointEligible] = modelDeploymentEndpointEligibleValue + pod.Annotations[modelDeploymentAnnotationKeyEndpointEligibleGeneration] = "uid-a/uid-old" + peer := servingGroupMember("regular", 0, 1, 2, "uid-b", true) + peer.Labels[modelDeploymentLabelKeyEndpointEligible] = modelDeploymentEndpointEligibleValue + peer.Annotations[modelDeploymentAnnotationKeyEndpointEligibleGeneration] = "uid-a/uid-old" + + groups, _ := servingGroupFacts(md, []*corev1.Pod{pod, peer}, + []RouterServingObservation{servingObservation("uid-a")}) + + fact, found := groups[modelDeploymentGroupKey{ + Role: "regular", + Ordinal: 0, + Generation: modelDeploymentMembershipGeneration("uid-a/uid-b"), + }] + require.True(t, found) + assert.False(t, fact.Retained, + "the members' annotation names an older generation, so the retention does not apply") + assert.False(t, modelDeploymentGroupRestores(true, true, fact), + "a group whose membership moved waits for a complete observation") }) t.Run("a group the router holds none of is valid and observed as empty", func(t *testing.T) { diff --git a/pkg/worker/controllers/worker/model_deployment_router_test.go b/pkg/worker/controllers/worker/model_deployment_router_test.go index e88e87dcc..9a0faa0b1 100644 --- a/pkg/worker/controllers/worker/model_deployment_router_test.go +++ b/pkg/worker/controllers/worker/model_deployment_router_test.go @@ -1589,7 +1589,7 @@ func TestRenderModelDeploymentRouterObjects_ImageSources(t *testing.T) { name: "picker defaults to the image that carries every router", md: routedModelDeployment(), container: "epp", - want: "gpustack/llm-router:v0.3.1", + want: "gpustack/llm-router:v0.4.0", why: "the default names a release of this project's own build, not a moving upstream tag", }, { @@ -1598,7 +1598,7 @@ func TestRenderModelDeploymentRouterObjects_ImageSources(t *testing.T) { md.Spec.Router.Name = workercore.ModelDeploymentRouterVLLM }), container: "router", - want: "gpustack/llm-router:v0.3.1", + want: "gpustack/llm-router:v0.4.0", why: "one setting says where the binaries come from; the command picks which one runs", }, { @@ -1645,12 +1645,14 @@ func TestRenderModelDeploymentRouterObjects_SerializedOutputIsPinnedToThePreSpli // The digests moved when the router HTTP port unified on 8000; they are pinned again here. // They moved again when a managed protected group began to name the eligible endpoints from // the first enable: every case whose roles are managed carries the term, and the sole - // single-member Server, which keeps its existing behavior, does not move at all. - "a prefill and decode pair": "9a449fcaaf867826dea0367a483e54e5388ddb395bd4afc8294553ba02b5bcb5", - "a sole server role": "093e7e521452bbc07a00de54ad477f1a58dfd0d155fb16bdac70d94fad06edb5", + // single-member Server, which keeps its existing behavior, does not move at all. They moved + // once more when the default router image advanced to v0.4.0: every case running the default + // moved together, and the case naming its own image did not. + "a prefill and decode pair": "2011724654ee75385a74533b0462fc2e31fff428ecfbe0e09ce2e5ba0f57b2ec", + "a sole server role": "2682f3e1bca1311a93f12f8416feeac60e854731e1e8d3169af2e4a9702badaf", "a router declaring its own image, policy, replicas and extra arguments": "79a13fe3afffc0896e22d1b01cff9ad87f7c9be22aa04356838731d18d41645f", - "a role whose command the user took over, so it publishes no events": "c9e93f6badf7d4ec334934ad2b5807bfe92ab16af110ac47b921ee67be001f17", - "an sglang engine, whose metrics contract differs": "475e475ecf7d41aa7eee26de36516cd582d70c955a3bd03c975acb525d58bc6d", + "a role whose command the user took over, so it publishes no events": "fa48c32b9e77475557cbb48bbc18463921f841d1824faecffe2d0d972fee0d38", + "an sglang engine, whose metrics contract differs": "f9523a56a596e838d45c1e89c7578d8c6d0943a3594d5ec73b10cf1f8344579e", } soleServerRole := func() *workercore.ModelDeployment { diff --git a/pkg/worker/settings/value.go b/pkg/worker/settings/value.go index bfbaafa4a..03eceecbb 100644 --- a/pkg/worker/settings/value.go +++ b/pkg/worker/settings/value.go @@ -434,7 +434,7 @@ var ( "Indicates the image a managed router runs, when the ModelDeployment does not name one. "+ "It carries every router this operator supports; the rendered command chooses which "+ "binary runs.", - setting.InitializeFromEnv("gpustack/llm-router:v0.3.1"), + setting.InitializeFromEnv("gpustack/llm-router:v0.4.0"), setting.AllowContainerImageReference(), ) diff --git a/pkg/worker/settings/value_test.go b/pkg/worker/settings/value_test.go index 96baf3243..b19fa7c39 100644 --- a/pkg/worker/settings/value_test.go +++ b/pkg/worker/settings/value_test.go @@ -93,7 +93,7 @@ func TestModelDeploymentImageSettings(t *testing.T) { ModelDeploymentRouterImage, "model-deployment-router-image", "GPUSTACK_MODEL_DEPLOYMENT_ROUTER_IMAGE", - "gpustack/llm-router:v0.3.1", + "gpustack/llm-router:v0.4.0", }, { ModelDeploymentRoutingSidecarImage,