feat(model-deployment): allow ordinary role configuration updates

.agents/skills/_e2e-lib/scripts/kubectl-shim/kubectl

@@ -119,7 +119,7 @@ transient() {
 }
 
 errf="$(mktemp)"
-trap 'rm -f "$errf" "${SPOOL:-}"' EXIT
+trap 'rm -f "$errf" ${SPOOL:+"$SPOOL"}' EXIT
 attempt=0
 while :; do
   if [ -n "$SPOOL" ]; then

.agents/skills/gpustack-operator-e2e/SKILL.md

@@ -163,6 +163,7 @@ Each case is self-contained; its header (see **Case header contract**) states go
 | 112 | Image source: a tag-only reference is refused naming the digest contract; an image artifact resolves claim-shaped (no manifest digest, no revision, no `status.nodes`); an Instance pinned to the node mounts the pinned image read-only through an image volume and reads the fixture byte for byte; a `ModelPrefetch` naming the artifact is refused | `pkg/worker/webhooks/worker/{model_artifact,model_prefetch}.go`, `pkg/worker/controllers/worker/{model_artifact_placement,model_deployment_artifact,instance,model_placement_preference}.go`, `pkg/kubediscovery/feature.go`, `cases/case-112.sh` | yes (confirm) | one worker whose kubelet/containerd serve image volumes (kubelet 1.35+, containerd 2.1+), docker on the runner, the stock `registry:2`, `python:3.12-slim` and `busybox:1.36` images pullable |
 | 113 | ModelScope source: a `modelScope` artifact resolves to the commit `git ls-remote` also names and materializes on a cold node hashing to the hub's own sha256; a second node pulls from the peer with its hub byte counter flat; a `modelscope` SDK at the floor downloads at the resolved commit; a well-formed `modelScope` source is admitted and a three-part repository refused | `pkg/modelartifact/modelscope.go`, `pkg/worker/controllers/worker/model_artifact.go`, `pkg/modelmanager/{driver/authorize.go,materialize/materialize.go,report/report.go}`, `pkg/worker/webhooks/worker/model_artifact.go`, `cases/case-113.sh` | yes (confirm) | two schedulable workers, the chart with `modelManager.enabled`, the stock python image pullable, and an internet path to www.modelscope.cn and pypi.org from the nodes and the runner (`E2E_C113_OFFLINE=1` skips) |
 | 114 | Expected digest: a match resolves with `digestSource Hub`, a foreign anchor is refused `DigestMismatch` naming both digests, a twice-unreachable hub writes the anchor as the identity (`digestSource Expected`, no revision) with the hub's request log flat, `Engine` delivery holds both anchored artifacts with `AnchorNeedsNodeDelivery` and no Pod, and an `Expected` identity mounts on a second node from the peer, hub log still flat | `api/worker/v1alpha1/model_artifact.go`, `pkg/worker/webhooks/worker/model_artifact.go`, `pkg/worker/controllers/worker/{model_artifact,model_artifact_placement,model_deployment}.go`, `pkg/modelmanager/materialize/materialize.go`, `pkg/modelmanager/peer/client.go`, `cases/case-114.sh` | yes (confirm) | one schedulable worker for the anchor legs, two for the peer leg (`E2E_C114_OFFLINE=1` skips that leg), the chart with `modelManager.enabled`, the stock python image pullable, one InstanceType for the engine-block rows; no internet — the hub is the case's own test hub |
+| 115 | Ordinary role updates replace whole groups: initially queued resources without a replacement slot, size growth and shrink, whole-card/slice resources, takeover command, optional cross-pool InstanceType, and P/D sibling identity | `pkg/worker/controllers/worker/model_deployment{,_initial_queue,_replacement_slot,_pod_group,_render,_service}.go`, `pkg/worker/webhooks/worker/model_deployment.go`, `cases/case-115.sh` | yes (confirm) | an idle NVIDIA pool with onceMaxRequest=1; two free whole cards for the size leg; logical slice capacity for slice edits; a different accelerator group for the cross-pool leg; a CPU type for P/D size and command checks; no engine inference |
 
 Each note below is something the **lead** must act on before or around a run. What a case *does* — its goal, environment, inputs, assertions and cleanup — lives in its own header, which the **Case header contract** below requires to be readable on its own; the index never restates it.
 

.agents/skills/gpustack-operator-e2e/cases/case-22.sh

@@ -85,6 +85,8 @@ best=None
 for n in json.load(sys.stdin).get('items',[]):
     a=n.get('status',{}).get('allocatable',{})
     for k,v in a.items():
+        # Allocation checks below read accelerator grants, so exclude RDMA interfaces.
+        if k == 'device.gpustack.ai/rdma': continue
         if not re.match(r'^[^/]+/[^.]+$', k): continue  # the bare exclusive whole-card resource
         try: cnt=int(v)
         except: continue
@@ -327,7 +329,7 @@ fill_exclusive() {
   local tag="$1" check="$2" count="$3" lq="$4" forbidden="$5"
   local tagl="${tag,,}" i p c cc seen=" "
   FILL_CARDS=""
-  for i in $(seq 1 "$count"); do
+  for ((i = 1; i <= count; i++)); do
     p="${PODPFX}-${tagl}-excl-${i}"; TESTPODS+=("$p")
     mkpod "$p" "$EXCL" "$lq"
     if [ "$(wait_settled "$p")" != "Running" ]; then

.agents/skills/gpustack-operator-e2e/cases/case-43.sh

@@ -149,7 +149,7 @@ restore() {
   for ns in "$NS_A" "$NS_B"; do
     kubectl -n "$ns" get kvcachepoolbindings.worker.gpustack.ai -o name 2>/dev/null \
       | while read -r b; do
-      kubectl -n "$ns" patch "$b" --subresource=status --type=merge \
+      kubectl -n "$ns" patch kvcachepoolbindings.v1alpha1.worker.gpustack.ai "${b##*/}" --subresource=status --type=merge \
         -p '{"status":{"usedBy":null}}' >/dev/null 2>&1 || true
     done
   done
@@ -630,8 +630,20 @@ echo "== 6. a binding a workload holds cannot be deleted =="
 # WRITTEN BY THIS CASE. The ModelDeployment controller fills a Binding's usedBy in production, and
 # this case runs no ModelDeployment. A case that waited for a writer would wait forever, and one
 # that skipped the write would assert a release nothing was holding.
-kubectl -n "$NS_A" patch kvcachepoolbindings.worker.gpustack.ai bind-a --subresource=status --type=merge \
-  -p '{"status":{"usedBy":[{"kind":"ModelDeployment","namespace":"","name":"qwen-e2e"}]}}' >/dev/null 2>&1
+# The public v1 proxy exposes no status writer. Use the storage version for this fixture write.
+if ! kubectl -n "$NS_A" patch kvcachepoolbindings.v1alpha1.worker.gpustack.ai bind-a --subresource=status --type=merge \
+  -p '{"status":{"usedBy":[{"kind":"ModelDeployment","namespace":"","name":"qwen-e2e"}]}}' >/dev/null; then
+  record FAIL "the holder fixture is written" "the storage-version status patch failed; no held deletion can be asserted"
+  print_rows
+  exit 1
+fi
+holder_kind="$(kubectl -n "$NS_A" get kvcachepoolbindings.v1alpha1.worker.gpustack.ai bind-a \
+  -o 'jsonpath={.status.usedBy[?(@.name=="qwen-e2e")].kind}')"
+if [ "$holder_kind" != ModelDeployment ]; then
+  record FAIL "the holder fixture is read back" "usedBy does not contain ModelDeployment/qwen-e2e"
+  print_rows
+  exit 1
+fi
 
 kubectl -n "$NS_A" delete kvcachepoolbindings.worker.gpustack.ai bind-a --wait=false >/dev/null 2>&1 || true
 sleep 20
@@ -650,8 +662,12 @@ fi
 
 # Released by hand, since nothing else will: this is the same patch a real consumer's controller
 # would make when it stops using the pool.
-kubectl -n "$NS_A" patch kvcachepoolbindings.worker.gpustack.ai bind-a --subresource=status --type=merge \
-  -p '{"status":{"usedBy":null}}' >/dev/null 2>&1
+if ! kubectl -n "$NS_A" patch kvcachepoolbindings.v1alpha1.worker.gpustack.ai bind-a --subresource=status --type=merge \
+  -p '{"status":{"usedBy":null}}' >/dev/null; then
+  record FAIL "the holder fixture is released" "the storage-version status patch failed"
+  print_rows
+  exit 1
+fi
 # Deliberately NOT wait_for with an empty expected value. That helper reads kubectl under
 # 2>/dev/null and compares the output, so a transport error, an RBAC denial or a webhook intercepting
 # the GET yields the same empty string a deleted object does — and with an empty `want` the FIRST

.agents/skills/gpustack-operator-e2e/cases/case-45.sh

@@ -61,11 +61,11 @@
 # Inputs:      All real, nothing mocked. Server-side dry-run (`--dry-run=server`) runs the schema
 #              and the webhook and persists nothing; the one row that needs a controller verdict
 #              creates a ModelDeployment naming a Binding that does not exist and deletes it again.
-#              Before it is deleted that stored object is edited three times, because a frozen field
-#              is a rule about an UPDATE and no dry-run create reaches it: `size` and `model` must be
-#              refused (the model refusal must name spec.model, state that the value describes a
-#              different deployment, and say nothing of conflict, concurrency, locks or races), and
-#              `replicas` must be accepted and read back from storage.
+#              Before it is deleted that stored object is edited three times, because an UPDATE
+#              rule is not exercised by a dry-run create: `size` and `replicas` must be accepted,
+#              and the accepted values must be read back from storage. The `model` refusal must
+#              name spec.model and state that the value describes a different deployment. It must
+#              say nothing of conflict, concurrency, locks or races.
 #
 # Deferred:    The serving half of this case — `replicas: 2` reaching `status.roles[0].ready == 2`
 #              and `status.endpoint` answering an inference request — is NOT here. It needs the
@@ -460,19 +460,17 @@ record SKIP "the unregistered domain costs the replicas their connector and noth
 # cannot fire on a create at all and no dry run reaches it. These two patch the deployment that is
 # still standing.
 #
-# THE PAIR IS THE ROW. `replicas` and `size` sit side by side in the same struct and are edited
-# through the same patch shape, so a webhook that refused both -- or a patch that failed for a
-# reason having nothing to do with either -- reads exactly like a correct one through the refusal
-# alone. The acceptance is what says the instrument can still say yes.
+# Ordinary size and replica-count edits are accepted. Model and engine identity remain frozen.
 if [ "$nobind_ready" = yes ]; then
   size_out="$(kubectl -n "$NS" patch modeldeployments.worker.gpustack.ai case45-nobind --type=json \
     -p '[{"op":"replace","path":"/spec/roles/0/size","value":2}]' 2>&1 | tr '\n' ' ')"
-  if [ -n "$size_out" ] && [ -z "${size_out##*roles\[0\].size*}" ]; then
-    record PASS "an edit to a role's size is refused, naming the field" \
-      "refused at spec.roles[0].size: every member of a running instance was built for the rank layout the old value described"
+  stored_size="$(kubectl -n "$NS" get modeldeployments.worker.gpustack.ai case45-nobind \
+    -o jsonpath='{.spec.roles[0].size}' 2>/dev/null)"
+  if [ -n "$size_out" ] && [ -z "${size_out##*patched*}" ] && [ "$stored_size" = 2 ]; then
+    record PASS "an ordinary role size edit is accepted and stored" "size=${stored_size}"
   else
-    record FAIL "an edit to a role's size is refused, naming the field" \
-      "wanted a refusal naming roles[0].size, got: $(echo "$size_out" | cut -c1-160)"
+    record FAIL "an ordinary role size edit is accepted and stored" \
+      "wanted size=2, stored=[${stored_size:-none}], patch: $(echo "$size_out" | cut -c1-160)"
   fi
 
   # THE MODEL SAYS WHICH DEPLOYMENT THIS IS, so it is frozen too, and its refusal is read for its
@@ -513,7 +511,7 @@ if [ "$nobind_ready" = yes ]; then
     -p '[{"op":"replace","path":"/spec/roles/0/replicas","value":2}]' 2>&1 | tr '\n' ' ')"
   if [ -n "$replicas_out" ] && [ -z "${replicas_out##*patched*}" ]; then
     record PASS "the control: an edit to a role's replicas is accepted" \
-      "patched, so the refusal above is this rule answering rather than the whole struct being frozen"
+      "patched alongside the accepted size edit"
   else
     record FAIL "the control: an edit to a role's replicas is accepted" \
       "wanted the patch to be accepted, got: $(echo "$replicas_out" | cut -c1-160)"
@@ -526,7 +524,7 @@ if [ "$nobind_ready" = yes ]; then
     record FAIL "the accepted replicas edit reached storage" "replicas=[${stored_replicas:-none}]"
   fi
 else
-  record SKIP "an edit to a role's size is refused, naming the field" \
+  record SKIP "an ordinary role size edit is accepted and stored" \
     "the subject deployment was never created, so there is no stored object to edit"
   record SKIP "an edit to the model is refused on a live object, naming the field" \
     "the subject deployment was never created, so there is no stored object to edit"

.agents/skills/gpustack-operator-e2e/cases/case-63.sh

@@ -276,6 +276,7 @@ spec:
     managed:
       leader:
         replicas: 3
+        multiTenancy: false
       members:
         - nodeSelector: {kubernetes.io/os: linux}
           medium: DRAM

.agents/skills/gpustack-operator-e2e/cases/case-65.sh

@@ -275,7 +275,7 @@ spec:
       # Omitting the key is refused at apply, which the host-directory gate above used to
       # hide -- that gate exits 0, so on any cluster without the directory this case reported
       # nothing rather than reporting that it could not build its own fixture.
-      leader: {electionBackend: None}
+      leader: {electionBackend: None, multiTenancy: false}
       members:
         - nodeSelector: {kubernetes.io/os: linux}
           medium: DRAM

.agents/skills/gpustack-operator-e2e/cases/case-68.sh

@@ -190,7 +190,7 @@ group_names() {
 # value on every round -- a wait that can only succeed immediately or run out the clock. It reads
 # like a wait and behaves like a single sample.
 group_count_is() { [ "$(group_names "$1" | wc -l | tr -d ' ')" = "$2" ]; }
-admitted_count_is_not() { [ "$(admitted_count)" != "$1" ]; }
+admitted_count_is_not() { [ "$(admitted_count "$MD")" != "$1" ]; }
 role_count_is() { # role_count_is <md> <role> <n>
   [ "$(k -n "$NS" get pods -l "app.kubernetes.io/instance=$1,app.kubernetes.io/component=$2" \
     -o name 2>/dev/null | grep -c .)" = "$3" ]
@@ -227,14 +227,12 @@ deployment_workloads() {
 }
 
 admitted_count() {
-  k -n "$NS" get workloads.kueue.x-k8s.io \
-    -o jsonpath='{range .items[*]}{.status.conditions[?(@.type=="Admitted")].status}{"\n"}{end}' 2>/dev/null \
-    | grep -c '^True$'
+  local counts
+  counts="$(deployment_admitted "$1")"
+  printf '%s\n' "${counts%%/*}"
 }
 
-# How many of ONE deployment's workloads report Admitted, as "<admitted>/<total>". The whole-cluster
-# count above cannot answer Phase E: another deployment being admitted would satisfy it, and the
-# claim there is about every group of one deployment.
+# How many of one deployment's workloads report Admitted, as "<admitted>/<total>".
 deployment_admitted() { # deployment_admitted <md>
   local wl total=0 ok=0
   for wl in $(deployment_workloads "$1"); do
@@ -513,10 +511,10 @@ else
 
   # NO ROLE ADMITTED -- and the control below is what makes this mean anything.
   sleep 30
-  if [ "$(admitted_count)" = 0 ]; then
+  if [ "$(admitted_count "$MD")" = 0 ]; then
     record PASS "no role admitted while one group cannot be placed" "0 admitted workloads"
   else
-    record FAIL "no role admitted while one group cannot be placed" "$(admitted_count) admitted"
+    record FAIL "no role admitted while one group cannot be placed" "$(admitted_count "$MD") admitted"
   fi
 
   # AND THE DEPLOYMENT MUST NOT CLAIM IT HOLDS QUOTA. This row exists because this cluster case