refactor(agents): consolidate the e2e case corpus without row loss

.agents/skills/gpustack-operator-e2e/cases/_kvcache-ha-lib.sh

@@ -0,0 +1,220 @@
+#!/usr/bin/env bash
+#
+# _kvcache-ha-lib.sh — the shared queries of the HA KV cache cases: the leader's Lease, the member
+#                     DaemonSet, the failover handoff and the backend-plus-Lease teardown.
+#
+# NOT A CASE. The leading underscore keeps it out of the `case-N.sh` namespace: it carries no case
+# header, no trap and no results table; those stay with the case that sources it.
+#
+#     # shellcheck source=/dev/null
+#     . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_kvcache-ha-lib.sh"
+#
+# Every helper takes what it needs as arguments and returns what the caller needs. Nothing here
+# reaches into a case's variables, and nothing here prints the results table.
+
+# kvha_wait_for <ns> <kind> <name> <path> <want> [secs] polls one jsonpath until it equals what is
+# wanted, and prints the LAST value seen, so a timeout says what the object was doing.
+kvha_wait_for() {
+  local ns="$1" kind="$2" name="$3" path="$4" want="$5" secs="${6:-180}"
+  local got="" i
+  for ((i = 0; i < secs; i += 3)); do
+    got="$(kubectl -n "$ns" get "$kind" "$name" -o "jsonpath=$path" 2>/dev/null)"
+    [ "$got" = "$want" ] && { echo "$got"; return 0; }
+    sleep 3
+  done
+  echo "$got"
+  return 1
+}
+
+# kvha_teardown <ns> <backend> <lease> deletes the backend and then the Lease it leaves behind: the
+# Lease carries no owner reference, so it goes after the backend.
+kvha_teardown() {
+  local ns="$1" backend="$2" lease="$3"
+  kubectl delete kvcachebackends.worker.gpustack.ai "$backend" --ignore-not-found --wait=false >/dev/null 2>&1 || true
+  kubectl wait --for=delete "kvcachebackends.worker.gpustack.ai/${backend}" --timeout=120s >/dev/null 2>&1 || true
+  kubectl -n "$ns" delete leases.coordination.k8s.io "$lease" --ignore-not-found --wait=false >/dev/null 2>&1 || true
+}
+
+# kvha_ready_leader_pod <ns> <leader-sel> prints the one leader Pod that is Ready, or nothing.
+kvha_ready_leader_pod() {
+  local ns="$1" leader_sel="$2"
+  kubectl -n "$ns" get pod -l "$leader_sel" \
+    -o jsonpath='{range .items[?(@.status.containerStatuses[0].ready==true)]}{.metadata.name}{"\n"}{end}' \
+    2>/dev/null | head -1
+}
+
+# kvha_lease_holder <ns> <lease> prints the Lease's holderIdentity, empty when it is absent.
+kvha_lease_holder() {
+  local ns="$1" lease="$2"
+  kubectl -n "$ns" get leases.coordination.k8s.io "$lease" \
+    -o jsonpath='{.spec.holderIdentity}' 2>/dev/null
+}
+
+# kvha_lease_renew_time <ns> <lease> prints the Lease's renewTime, empty when it is absent.
+kvha_lease_renew_time() {
+  local ns="$1" lease="$2"
+  kubectl -n "$ns" get leases.coordination.k8s.io "$lease" \
+    -o jsonpath='{.spec.renewTime}' 2>/dev/null
+}
+
+# kvha_lease_transitions <ns> <lease> prints the Lease's handover count, zero when it is absent.
+kvha_lease_transitions() {
+  local ns="$1" lease="$2" t
+  t="$(kubectl -n "$ns" get leases.coordination.k8s.io "$lease" -o jsonpath='{.spec.leaseTransitions}' 2>/dev/null)"
+  echo "${t:-0}"
+}
+
+# kvha_holder_pod_uid <ns> <leader-sel> <holder> prints the UID of the Pod the holderIdentity
+# names: the holder string is the serving process's rpc address, so it names a Pod by its IP.
+kvha_holder_pod_uid() {
+  local ns="$1" leader_sel="$2" holder="$3"
+  local holder_ip pod_uid pod_ip deleting phase
+  case "$holder" in
+    \[*\]:*)
+      holder_ip="${holder#\[}"
+      holder_ip="${holder_ip%%\]*}"
+      ;;
+    *:*) holder_ip="${holder%:*}" ;;
+    *) holder_ip="$holder" ;;
+  esac
+  while IFS='|' read -r pod_uid pod_ip deleting phase; do
+    if [ "$phase" != "Running" ] || [ -n "$deleting" ] || [ -z "$pod_ip" ] \
+      || [ "$pod_ip" != "$holder_ip" ]; then
+      continue
+    fi
+    echo "$pod_uid"
+    return 0
+  done < <(kubectl -n "$ns" get pod -l "$leader_sel" \
+    -o jsonpath='{range .items[*]}{.metadata.uid}{"|"}{.status.podIP}{"|"}{.metadata.deletionTimestamp}{"|"}{.status.phase}{"\n"}{end}' \
+    2>/dev/null)
+}
+
+# kvha_pod_uid <ns> <pod> prints one Pod's UID, empty when the Pod cannot be read.
+kvha_pod_uid() {
+  local ns="$1" pod="$2"
+  kubectl -n "$ns" get pod "$pod" -o jsonpath='{.metadata.uid}' 2>/dev/null
+}
+
+# kvha_member_snapshot <ns> <member-sel> prints the member Pods as name=UID:restarts, so a
+# replacement Pod is caught by its UID.
+kvha_member_snapshot() {
+  local ns="$1" member_sel="$2"
+  kubectl -n "$ns" get pod -l "$member_sel" \
+    -o jsonpath='{range .items[*]}{.metadata.name}={.metadata.uid}:{.status.containerStatuses[0].restartCount}{"\n"}{end}' \
+    2>/dev/null | sort
+}
+
+# kvha_members_mounted_sample <ns> <backend> prints one reading of the backend's own report, as
+# phase|MembersMounted status/reason.
+kvha_members_mounted_sample() {
+  local ns="$1" backend="$2"
+  kubectl -n "$ns" get kvcachebackends.worker.gpustack.ai "$backend" \
+    -o jsonpath='{.status.phase}{"|"}{.status.conditions[?(@.type=="MembersMounted")].status}{"/"}{.status.conditions[?(@.type=="MembersMounted")].reason}{"\n"}' \
+    2>/dev/null
+}
+
+# kvha_observe_handoff_snapshot <ns> <backend> <lease> <leader-sel> <traj-file> <delete-epoch>
+# <old-ready-uid> <old-holder> watches the Lease until the handoff is observable and prints what it
+# saw, as one pipe-separated record:
+#
+#   moved-at|moved-holder|moved-holder-pod-uid|move-evidence|handoff-observed|new-holder|
+#   new-holder-pod-uid|new-ready|new-ready-uid|wait-secs
+#
+# The watch and the detection are one function because the detection is only meaningful between two
+# reads of the same Lease: a changed holderIdentity or a changed renewTime timestamps the move
+# directly, while a Ready holder is an upper bound. The deadline is the delete epoch plus 240s.
+kvha_observe_handoff_snapshot() {
+  local ns="$1" backend="$2" lease="$3" leader_sel="$4" traj_file="$5"
+  local delete_epoch="$6" old_ready_uid="$7" old_holder="$8"
+  local new_holder="" new_holder_pod_uid="" new_ready="" new_ready_uid="" new_renew_time=""
+  local reused_holder_pod_uid="" reused_holder_renew_time=""
+  local moved_at="" moved_holder="" moved_holder_pod_uid="" move_evidence="" handoff_observed=0
+  local observed_at deadline wait_secs
+
+  deadline=$((delete_epoch + 240))
+  while [ "$(date +%s)" -lt "$deadline" ]; do
+    kvha_members_mounted_sample "$ns" "$backend" >>"$traj_file"
+    new_holder="$(kvha_lease_holder "$ns" "$lease")"
+    new_holder_pod_uid="$(kvha_holder_pod_uid "$ns" "$leader_sel" "$new_holder")"
+    new_renew_time=""
+    if [ -n "$new_holder_pod_uid" ] && [ "$new_holder_pod_uid" != "$old_ready_uid" ] \
+      && [ "$new_holder" = "$old_holder" ]; then
+      new_renew_time="$(kvha_lease_renew_time "$ns" "$lease")"
+    fi
+    new_ready="$(kvha_ready_leader_pod "$ns" "$leader_sel")"
+    new_ready_uid=""
+    if [ -n "$new_ready" ]; then
+      new_ready_uid="$(kvha_pod_uid "$ns" "$new_ready")"
+    fi
+
+    observed_at="$(date +%s)"
+    if [ -z "$moved_at" ] && [ -n "$new_holder_pod_uid" ] \
+      && [ "$new_holder_pod_uid" != "$old_ready_uid" ]; then
+      if [ "$new_holder" != "$old_holder" ]; then
+        moved_at="$observed_at"
+        moved_holder="$new_holder"
+        moved_holder_pod_uid="$new_holder_pod_uid"
+        move_evidence="holderIdentity changed"
+      elif [ -n "$new_renew_time" ]; then
+        if [ "$reused_holder_pod_uid" != "$new_holder_pod_uid" ] \
+          || [ -z "$reused_holder_renew_time" ]; then
+          reused_holder_pod_uid="$new_holder_pod_uid"
+          reused_holder_renew_time="$new_renew_time"
+        elif [ "$new_renew_time" != "$reused_holder_renew_time" ]; then
+          moved_at="$observed_at"
+          moved_holder="$new_holder"
+          moved_holder_pod_uid="$new_holder_pod_uid"
+          move_evidence="replacement renewed the reused holderIdentity"
+        fi
+      fi
+    fi
+    if [ -n "$new_holder_pod_uid" ] && [ "$new_holder_pod_uid" != "$old_ready_uid" ] \
+      && [ "$new_holder_pod_uid" = "$new_ready_uid" ]; then
+      handoff_observed=1
+      if [ -z "$moved_at" ]; then
+        moved_at="$observed_at"
+        moved_holder="$new_holder"
+        moved_holder_pod_uid="$new_holder_pod_uid"
+        move_evidence="Ready replacement confirmed the reused holderIdentity"
+      fi
+    fi
+    [ "$handoff_observed" = "1" ] && break
+    sleep 2
+  done
+  wait_secs=$(( $(date +%s) - delete_epoch ))
+  printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' \
+    "$moved_at" "$moved_holder" "$moved_holder_pod_uid" "$move_evidence" "$handoff_observed" \
+    "$new_holder" "$new_holder_pod_uid" "$new_ready" "$new_ready_uid" "$wait_secs"
+}
+
+# kvha_failover_bound_verdict <moved-at> <delete-epoch> <bound-secs> <evidence> judges one failover
+# measurement and prints the verdict as STATUS|object: PASS, FAIL or SKIP, and the text that goes
+# into the row. The row itself stays with the case, which calls `record`. The Ready replacement is
+# an upper bound rather than a direct reading of the move, so it SKIPs.
+kvha_failover_bound_verdict() {
+  local moved_at="$1" delete_epoch="$2" bound_secs="$3" evidence="$4"
+  local moved_secs=$((moved_at - delete_epoch))
+  case "$evidence" in
+    "holderIdentity changed"|"replacement renewed the reused holderIdentity")
+      if [ "$moved_secs" -le "$bound_secs" ]; then
+        printf 'PASS|%ss <= %ss; %s\n' "$moved_secs" "$bound_secs" "$evidence"
+      else
+        printf 'FAIL|%ss > %ss -- %s; the election still works but no longer meets the failover budget\n' \
+          "$moved_secs" "$bound_secs" "$evidence"
+      fi
+      ;;
+    "Ready replacement confirmed the reused holderIdentity")
+      printf 'SKIP|%ss after the failover trigger; the Ready replacement upper bound includes Pod scheduling, image pulling, and readiness probing\n' \
+        "$moved_secs"
+      ;;
+    *)
+      printf 'FAIL|unrecognized move evidence: %s\n' "${evidence:-<none>}"
+      ;;
+  esac
+}
+
+# kvha_resolve_rollout_images <default-start> <default-roll-to> prints the two images of a rollout
+# pair, separated by a space: the start image, then the rollout target.
+kvha_resolve_rollout_images() {
+  printf '%s %s\n' "${E2E_MOONCAKE_IMAGE:-$1}" "${E2E_MOONCAKE_ROLLOUT_IMAGE:-$2}"
+}

.agents/skills/gpustack-operator-e2e/cases/_kvcache-inject-lib.sh

@@ -27,8 +27,8 @@
 #
 # Environment the caller may set:
 #   E2E_MOONCAKE_IMAGE   the store image                 (default docker.io/kvcacheai/mooncake:0.3.13)
-#   E2E_VLLM_IMAGE       a vLLM image, for case 59       (unset => that half SKIPs, loudly)
-#   E2E_SGLANG_IMAGE     an SGLang image, for case 59    (unset => that half SKIPs, loudly)
+#   E2E_VLLM_IMAGE       a vLLM image, for case 53       (unset => that half SKIPs, loudly)
+#   E2E_SGLANG_IMAGE     an SGLang image, for case 53    (unset => that half SKIPs, loudly)
 #   E2E_CLIENT_IMAGE     an image carrying the mooncake python client, for the read/write probes
 #
 # EXERCISED 2026-09-04 on a three-node k3s cluster: all seven completed with no failures, and case
@@ -37,8 +37,8 @@
 #
 # Four of the seven failed on their first real run, and not one was failing for the reason it printed:
 # case 53 hit a missing prerequisite (no Binding registered the tenant its writes land on) and then
-# read usage off the wrong Binding; case 56 could not have passed at all, because it compared fields
-# the API server defaults onto every Pod; case 59 timed out on a 19.1GB image pull. Reading them had
+# read usage off the wrong Binding; case 53 could not have passed at all, because it compared fields
+# the API server defaults onto every Pod; case 53 timed out on a 19.1GB image pull. Reading them had
 # shown none of that, and three of the four were defects in the cases rather than in the webhook.
 
 E2E_SHIM_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../_e2e-lib/scripts/kubectl-shim" 2>/dev/null && pwd)"
@@ -98,7 +98,7 @@ DOMAIN="dom-i-${SFX}"
 ENDPOINT=""
 # The launch stub kvi_setup projects into every injected fixture, and where each Pod mounts it. The
 # three names are shared because four manifests in this family declare the same volume: this file's
-# kvi_pod_manifest, case 53's Deployment, case 54's LWS leader and case 56's submit().
+# kvi_pod_manifest, case 53's Deployment, LWS leader and submit().
 #
 # LAUNCH_DIR is NOT on any image's PATH, and must not be: the stub is reached by the absolute path in
 # `command`, while a probe running `python3` inside the container has to get the image's real
@@ -120,15 +120,19 @@ record() {
 . "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/_rows-lib.sh"
 
 
-# kvi_wait_for polls one jsonpath until it equals want.
+# kvi_wait_for is the family's own jsonpath poller, self-contained on purpose: a consumer may
+# define a wait_for of its own (case-70 replaces its argument list with seconds-then-predicate),
+# and forwarding through a replaceable global would run this family's kind/name/path/want calls
+# against that incompatible signature. The base corpus had this implementation inline; the
+# extraction keeps it whole rather than aliasing it.
 kvi_wait_for() {
   local kind="$1" name="$2" path="$3" want="$4" secs="${5:-120}" ns_args=()
   [ -n "${6:-}" ] && ns_args=(-n "$6")
   local got="" i
   # ${ns_args[@]+...} rather than a bare "${ns_args[@]}": under set -u, bash 3.2 - macOS's stock
   # shell, which this family is usually launched from - treats expanding an EMPTY array as an unbound
   # variable and aborts the case mid-run. Every cluster-scoped wait passes no namespace, so the empty
-  # case is the common one, not the edge one. Measured both ways on bash 3.2.57.
+  # case is the common one, not the edge one.
   for ((i = 0; i < secs; i += 3)); do
     got="$(kubectl ${ns_args[@]+"${ns_args[@]}"} get "$kind" "$name" -o "jsonpath=$path" 2>/dev/null)"
     [ "$got" = "$want" ] && { echo "$got"; return 0; }
@@ -398,6 +402,52 @@ kvi_env() {
     -o "jsonpath={.spec.containers[0].env[?(@.name=='${name}')].value}" 2>/dev/null
 }
 
+# kvi_store_roundtrip_py POD CONTAINER KEY PAYLOAD runs the mooncake client library inside an injected
+# Pod and prints what it learned: the tenant the projected file named, the store's setup status, the
+# put's status, and the length of the bytes that came back with whether they match what was written.
+#
+# The probe reads the file the webhook PROJECTED - not an address this suite knows - and calls setup()
+# with the seven positional arguments plus the file's tenant_id. The tenant rides the keyword: the next
+# positional slot is a TransferEngine pointer, and passing the tenant there raises instead of
+# registering it. A file without a tenant_id falls back to the store's own "default", which the usage
+# rows catch.
+#
+# The python source stays at column 0: it is a program carried in a quoted argument, so indenting it
+# here would change the program rather than the shell.
+#
+# The exit status is the exec's own. A Pod that cannot run the client fails here, and the caller reads
+# the printed lines to say which half of the round trip was reached.
+kvi_store_roundtrip_py() {
+  local pod="$1" container="$2" key="$3" payload="$4" out rc
+  out="$(kubectl -n "$TEST_NS" exec "$pod" -c "$container" -- python3 -c '
+import json, os, sys
+from mooncake.store import MooncakeDistributedStore
+
+cfg = json.load(open(os.environ["MOONCAKE_CONFIG_PATH"]))
+tenant = cfg.get("tenant_id") or "default"
+print("TENANT %s" % tenant)
+store = MooncakeDistributedStore()
+rc = store.setup(
+    os.environ.get("POD_IP", "127.0.0.1"),
+    cfg["metadata_server"],
+    cfg["global_segment_size"],
+    cfg["local_buffer_size"],
+    cfg["protocol"],
+    cfg["device_name"],
+    cfg["master_server_address"],
+    tenant_id=tenant,
+)
+print("SETUP rc=%d" % rc)
+if rc != 0:
+    sys.exit(0)
+print("PUT rc=%d" % store.put(sys.argv[1], sys.argv[2].encode()))
+got = store.get(sys.argv[1])
+print("GET len=%d match=%s" % (len(got or b""), (got == sys.argv[2].encode())))
+' "$key" "$payload" 2>&1)" && rc=0 || rc=$?
+  printf '%s\n' "$out"
+  return "$rc"
+}
+
 # kvi_refused applies a manifest that must be REJECTED, and reports whether the message names what it
 # should. A create that succeeds is the failure: this family exists to prove the webhook refuses
 # rather than injecting something that looks configured and is not.
@@ -539,28 +589,24 @@ kvi_teardown() {
 
   kubectl -n "$TEST_NS" delete kvcachepoolbindings.worker.gpustack.ai "$BINDING" "$BINDING_DEFAULT" \
     --ignore-not-found --wait=false >/dev/null 2>&1 || true
-  local left="" _
-  for _ in $(seq 1 20); do
-    left="$(kubectl -n "$TEST_NS" get kvcachepoolbindings.worker.gpustack.ai -o name 2>/dev/null)"
-    [ -z "$left" ] && break
-    sleep 3
-  done
-  if [ -n "${left:-}" ]; then
-    echo "[kvcache-inject] a binding is still held after 60s; what the operator says about it:"
-    kubectl -n "$TEST_NS" get kvcachepoolbindings.worker.gpustack.ai \
-      -o 'jsonpath={range .items[*]}{.metadata.name}{"\t"}{range .status.conditions[*]}{.type}={.status}({.reason}) {.message}{"\n"}{end}{end}' 2>/dev/null \
-      | sed 's/^/    /'
-    # Forcing the finalizer off bypasses the operator's own release path, which removes it only once
-    # the master's ledger entry for that tenant is gone (kv_cache_pool.go, releaseKVCachePoolBinding).
-    # So this can leave a tenant registered in a master with no Binding owning it. It is still the
-    # right trade here - a Binding left held forever poisons every later run of this family, and these
-    # names are cluster-scoped - but it is not clean, and a forced action that prints nothing reads as
-    # a clean one.
-    echo "[kvcache-inject] forcing the finalizer off; a tenant may remain registered in the master."
-    echo "[kvcache-inject] the pool and backend are deleted below, which normally takes it with them;"
-    echo "[kvcache-inject] if either is also stuck, check the master's tenant list by hand."
-    kubectl -n "$TEST_NS" get kvcachepoolbindings.worker.gpustack.ai -o name 2>/dev/null | while read -r b; do
-      kubectl -n "$TEST_NS" patch "$b" --type=merge -p '{"metadata":{"finalizers":null}}' >/dev/null 2>&1 || true
+  # One binding at a time: each is waited for, and one still held after 60s has its finalizer forced
+  # off, with a report of what the operator said about it. The pool and the backend are deleted
+  # below, which normally takes the tenant with them; if either is also stuck, the master's tenant
+  # list has to be read by hand.
+  kvi_release_binding_finalizer "$TEST_NS" "$BINDING"
+  kvi_release_binding_finalizer "$TEST_NS" "$BINDING_DEFAULT"
+  # A case's own probe can leave a binding this teardown never named - the refusal probe creates
+  # bind-term with a case-added finalizer, and an interrupted run leaves it held with no controller
+  # to release it. Sweep every binding still standing in the fixture namespace, so a leftover of
+  # any name releases the same way the named ones do and cannot wedge the namespace in
+  # Terminating against the next run.
+  local leftover=""
+  leftover="$(kubectl -n "$TEST_NS" get kvcachepoolbindings.worker.gpustack.ai -o name 2>/dev/null || true)"
+  if [ -n "$leftover" ]; then
+    echo "[kvcache-inject] releasing binding(s) the fixture did not name:"
+    printf '%s\n' "$leftover" | sed 's/^/    /'
+    printf '%s\n' "$leftover" | while read -r b; do
+      [ -n "$b" ] && kvi_release_binding_finalizer "$TEST_NS" "$(printf '%s' "$b" | awk -F/ '{print $NF}')"
     done
   fi