sglang 0.9.18, vllm 0.8.17: pd-ds drift liveness also detects in-place engine restart

charts/sglang/Chart.yaml

@@ -5,7 +5,7 @@ home: https://github.com/modelsphere/helm-charts
 sources:
   - https://github.com/modelsphere/helm-charts
 type: application
-version: 0.9.17
+version: 0.9.18
 appVersion: "1.0.0"
 
 # CART is vendored under charts/cart -- already present, so no `helm dependency

charts/sglang/templates/_pod.tpl

@@ -957,33 +957,99 @@ exit 0
        sglang.pdRouterProbes -- separate concerns. */ -}}
 {{- define "sglang.pdRouterLivenessProbe" -}}
 {{- $roles := .roles -}}
+{{- $root := .root -}}
 livenessProbe:
   exec:
-    command: ["bash", "-lc", {{ include "sglang.pdRouterLivenessProbeScript" (dict "roles" $roles) | quote }}]
+    command: ["bash", "-lc", {{ include "sglang.pdRouterLivenessProbeScript" (dict "root" $root "roles" $roles) | quote }}]
   periodSeconds: 10
   timeoutSeconds: 10
   failureThreshold: 2
 {{- end -}}
 
 {{- define "sglang.pdRouterLivenessProbeScript" -}}
 {{- $roles := .roles -}}
+{{- $root := .root -}}
 set -u
 rc=0
+HW_PORT={{ ternary (int $root.Values.hangWatcher.port) 0 $root.Values.hangWatcher.enabled }}
+# Read the engine-side identity that the router boot script recorded for $ip.
+# Empty tuple means "no identity info" -- old sidecar, HW_PORT=0 at boot, or
+# the boot-time 90min budget exhausted. The live probe below returns '|' for
+# the same condition. Fires iff the two sides disagree on identity: boot said
+# sentinel while live says real (identity appeared late), OR boot said real
+# while live says a different real (engine restarted in place).
+want_of() { awk -F'|' -v ip="$1" '$1==ip { print $2"|"$3; exit }' "$2"; }
+# Probe the live identity. 404 = old sidecar, same as boot sentinel; any
+# other non-200 (503 mid-id-pin, conn refused, timeout) = skip-tick. Both
+# shapes return exactly one parseable line on success and nothing else.
+live_id() {
+  local ip=$1
+  [ "$HW_PORT" -eq 0 ] && { echo '|'; return 0; }
+  python3 - "$ip" "$HW_PORT" <<'PY' 2>/dev/null
+import json, sys, urllib.request, urllib.error
+ip, port = sys.argv[1], sys.argv[2]
+try:
+    r = urllib.request.urlopen(f"http://{ip}:{port}/engine", timeout=2)
+    j = json.loads(r.read())
+    print(f"{j['restarts']}|{j['watcher_started_at']}")
+except urllib.error.HTTPError as e:
+    if e.code == 404:
+        print('|')
+    else:
+        sys.exit(2)
+except Exception:
+    sys.exit(2)
+PY
+}
 check() {
   local role=$1 svc=$2 file="/run/pd-router/$3" port=$4
   [ -f "$file" ] || return 0
   local ip
   while read -r ip; do
     [ -n "$ip" ] || continue
-    grep -qx "$ip" "$file" && continue
-    # New IP not in argv. Only count drift once it is actually SERVING:
-    # restarting into a still-loading engine is wasted (router can't route
-    # to it) and on engines whose router blocks its own listener on a
-    # per-upstream bootstrap handshake (vllm + mooncake), is actively
-    # harmful -- the whole slice sits dark until the retry budget drains.
-    # Short timeout so a slow engine never stalls the probe itself.
+    # Fast path: does this ip appear in the boot snapshot at all. Fixed-string
+    # match on the ip+pipe prefix -- '|' is a regex meta and would silently
+    # mis-grep (e.g. '10' matching '1.0.x.x'). -F treats it as text.
+    grep -qF "$ip|" "$file" || { unknown_ip=1; }
+    if [ "${unknown_ip:-0}" -eq 1 ]; then
+      # New IP not in argv. Only count drift once it is actually SERVING:
+      # restarting into a still-loading engine is wasted (router can't route
+      # to it) and on engines whose router blocks its own listener on a
+      # per-upstream bootstrap handshake (vllm + mooncake), is actively
+      # harmful -- the whole slice sits dark until the retry budget drains.
+      # Short timeout so a slow engine never stalls the probe itself.
+      if python3 -c "import urllib.request; urllib.request.urlopen('http://$ip:$port/health', timeout=2)" 2>/dev/null; then
+        echo "router stale: $role upstream $ip serving but not in argv" >&2
+        rc=1
+      fi
+      unknown_ip=0
+      continue
+    fi
+    # IP is in argv. Compare identities. want comes from the boot snapshot,
+    # live comes from the sidecar NOW; both may be the '|' sentinel for "no
+    # identity info". Any disagreement means the engine behind this IP is no
+    # longer the incarnation the router argv pined at boot:
+    #
+    #   want REAL, live REAL, different -> in-place restart  (the mooncake
+    #     boot-pinned engine_id can't see this on its own).
+    #   want '|', live REAL              -> identity arrived after boot (slow
+    #     sidecar, or boot budget exhausted mid-model-load). Restart so the
+    #     router re-snapshots with identity -- otherwise that IP stays
+    #     protected-by-silence for the life of this router process.
+    #   want REAL, live '|'              -> probe flake or sidecar wedged.
+    #     We can't tell; treat as no data and skip. An actually-restarted
+    #     engine answers with a REAL different id, so we do NOT miss the
+    #     real engine-swap case by skipping this one.
+    #
+    # Probe errors (skip-tick) are the same rule 1f0d236 uses for DNS blips:
+    # never fire on a missing answer, only when both sides positively
+    # answered and they disagree.
+    want=$(want_of "$ip" "$file")
+    live=$(live_id "$ip") || continue    # probe failed: skip-tick
+    [ "$live" = '|' ] && continue        # live has no data; skip
+    [ "$live" = "$want" ] && continue
     if python3 -c "import urllib.request; urllib.request.urlopen('http://$ip:$port/health', timeout=2)" 2>/dev/null; then
-      echo "router stale: $role upstream $ip serving but not in argv" >&2
+      echo "router stale: $role upstream $ip identity changed ($want -> $live) while serving" >&2
       rc=1
     fi
   done < <(timeout 2 getent hosts "$svc" 2>/dev/null | awk '{print $1}' | sort -u)

charts/sglang/templates/pd-ds.yaml

@@ -247,9 +247,74 @@ spec:
           # Persist the boot-time upstream set for the liveness drift probe.
           # Router argv is immutable for the life of the process, so the file
           # is the probe's only handle on "what upstreams were baked in".
+          #
+          # Each line is "<ip>|<restarts>|<watcher_started_at>" from the
+          # engine pod's hang-watcher /engine endpoint. The ip alone isn't a
+          # stable identity: sglang-router pins bootstrap info at boot on
+          # TOP of the IP, so an engine that crashes and is restarted in place
+          # by the kubelet keeps its IP but swaps engine_id -- the router's
+          # transfer sessions then point at a dead incarnation while DNS
+          # shows nothing new. Comparing (restarts, watcher_started_at) is
+          # version-agnostic: any hang-watcher that exposes /engine yields a
+          # fresh tuple for every engine process replacement.
+          #
+          # HW_PORT=0 is the "sidecar disabled at render time" sentinel and
+          # behaves identically to the older-sidecar 404: record "<ip>||" and
+          # move on, the runtime probe treats both as "no identity info".
+          # 503 = sidecar alive but engine id not yet pinned; retry up to
+          # 90min, then give up and record the same sentinel (fail-open:
+          # identity unknown is not a reason to keep the router from ever
+          # booting). engine not yet listening (conn refused) and CoreDNS
+          # slop (timeout) also retry on the same budget.
+          HW_PORT={{ ternary (int $root.Values.hangWatcher.port) 0 $root.Values.hangWatcher.enabled }}
+          probe_identity() {
+            local ip=$1 budget=${2:-5400} waited=0
+            [ "$HW_PORT" -eq 0 ] && { echo '|'; return 0; }
+            while :; do
+              local out
+              # python3 heredoc: body and terminator sit at the block scalar's
+              # own indent, so the rendered yaml strips them to column 0 for
+              # bash+python. Exit 0 prints '<restarts>|<ts>' on 200 or '|' on
+              # 404 sentinel; exit 2 on 503 / conn refused / timeout / bad
+              # json = transient, caller retries on its own budget.
+              out=$(python3 - "$ip" "$HW_PORT" <<'PY' 2>/dev/null
+          import json, sys, urllib.request, urllib.error
+          ip, port = sys.argv[1], sys.argv[2]
+          try:
+              r = urllib.request.urlopen(f"http://{ip}:{port}/engine", timeout=5)
+              j = json.loads(r.read())
+              print(f"{j['restarts']}|{j['watcher_started_at']}")
+          except urllib.error.HTTPError as e:
+              if e.code == 404:
+                  print('|')
+              else:
+                  sys.exit(2)
+          except Exception:
+              sys.exit(2)
+          PY
+          )
+              local rrc=$?
+              if [ $rrc -eq 0 ]; then echo "$out"; return 0; fi
+              if [ $waited -ge $budget ]; then
+                echo "router: $ip identity probe gave no usable answer after ${budget}s; recording ip||" >&2
+                echo '|'; return 0
+              fi
+              sleep 10; waited=$((waited+10))
+            done
+          }
           install -d -m 0755 /run/pd-router
-          T=$(mktemp -p /run/pd-router); printf '%s\n' "$PF_IPS" >"$T"; mv "$T" /run/pd-router/prefill-ips
-          T=$(mktemp -p /run/pd-router); printf '%s\n' "$DC_IPS" >"$T"; mv "$T" /run/pd-router/decode-ips
+          write_role_snapshot() {
+            local role=$1 ips=$2
+            local T=$(mktemp -p /run/pd-router)
+            local ip id
+            for ip in $ips; do
+              id=$(probe_identity "$ip")
+              printf '%s|%s\n' "$ip" "$id" >>"$T"
+            done
+            mv "$T" "/run/pd-router/${role}-ips"
+          }
+          write_role_snapshot prefill "$PF_IPS"
+          write_role_snapshot decode  "$DC_IPS"
           # shellcheck disable=SC2086
           exec python3 -m sglang_router.launch_router \
             --pd-disaggregation \
@@ -273,7 +338,7 @@ spec:
         {{- /* Drift liveness: restart the router if a new prefill/decode IP
                appears in DNS after boot -- argv is immutable and the running
                router can never learn about post-boot upstreams. */}}
-        {{- include "sglang.pdRouterLivenessProbe" (dict "roles" (dict
+        {{- include "sglang.pdRouterLivenessProbe" (dict "root" $root "roles" (dict
              "prefill" (dict "host" (printf "%s-prefill-hl-s%d.%s.svc.cluster.local." $fullname $s $root.Release.Namespace) "port" ($pd.prefill.httpPort | default 30001))
              "decode"  (dict "host" (printf "%s-decode-hl-s%d.%s.svc.cluster.local."  $fullname $s $root.Release.Namespace) "port" ($pd.decode.httpPort  | default 30002)))) | nindent 8 }}
         ports:

charts/sglang/values.yaml

@@ -488,7 +488,12 @@ hangWatcher:
     # silently (unknown JSON fields are dropped), so the grace period would look
     # configured and do nothing -- pin the tag rather than rely on the cluster
     # happening to run a new enough image.
-    tag: "0.1.26"
+    # >= 0.1.27 is required for GET /engine, the endpoint the pd router's
+    # drift-liveness probe uses to detect in-place engine restart. Older
+    # sidecars return 404 and the chart falls back to pre-identity behavior
+    # (IP-growth only) -- silent, no log. If you must run an older sidecar,
+    # expect that failure mode not this protection.
+    tag: "0.1.27"
   port: 9090                 # /healthz port (just needs to differ from service.port)
   # Thresholds -> ConfigMap the sidecar hot-reloads (helm upgrade retunes, no pod restart).
   config:

charts/vllm/Chart.yaml

@@ -5,7 +5,7 @@ home: https://github.com/modelsphere/helm-charts
 sources:
   - https://github.com/modelsphere/helm-charts
 type: application
-version: 0.8.16
+version: 0.8.17
 appVersion: "1.0.0"
 
 # CART is vendored under charts/cart -- already present, so no `helm dependency

charts/vllm/templates/_pod.tpl

@@ -996,33 +996,99 @@ exit 0
        vllm.pdRouterProbes -- separate concerns. */ -}}
 {{- define "vllm.pdRouterLivenessProbe" -}}
 {{- $roles := .roles -}}
+{{- $root := .root -}}
 livenessProbe:
   exec:
-    command: ["bash", "-lc", {{ include "vllm.pdRouterLivenessProbeScript" (dict "roles" $roles) | quote }}]
+    command: ["bash", "-lc", {{ include "vllm.pdRouterLivenessProbeScript" (dict "root" $root "roles" $roles) | quote }}]
   periodSeconds: 10
   timeoutSeconds: 10
   failureThreshold: 2
 {{- end -}}
 
 {{- define "vllm.pdRouterLivenessProbeScript" -}}
 {{- $roles := .roles -}}
+{{- $root := .root -}}
 set -u
 rc=0
+HW_PORT={{ ternary (int $root.Values.hangWatcher.port) 0 $root.Values.hangWatcher.enabled }}
+# Read the engine-side identity that the router boot script recorded for $ip.
+# Empty tuple means "no identity info" -- old sidecar, HW_PORT=0 at boot, or
+# the boot-time 90min budget exhausted. The live probe below returns '|' for
+# the same condition. Fires iff the two sides disagree on identity: boot said
+# sentinel while live says real (identity appeared late), OR boot said real
+# while live says a different real (engine restarted in place).
+want_of() { awk -F'|' -v ip="$1" '$1==ip { print $2"|"$3; exit }' "$2"; }
+# Probe the live identity. 404 = old sidecar, same as boot sentinel; any
+# other non-200 (503 mid-id-pin, conn refused, timeout) = skip-tick. Both
+# shapes return exactly one parseable line on success and nothing else.
+live_id() {
+  local ip=$1
+  [ "$HW_PORT" -eq 0 ] && { echo '|'; return 0; }
+  python3 - "$ip" "$HW_PORT" <<'PY' 2>/dev/null
+import json, sys, urllib.request, urllib.error
+ip, port = sys.argv[1], sys.argv[2]
+try:
+    r = urllib.request.urlopen(f"http://{ip}:{port}/engine", timeout=2)
+    j = json.loads(r.read())
+    print(f"{j['restarts']}|{j['watcher_started_at']}")
+except urllib.error.HTTPError as e:
+    if e.code == 404:
+        print('|')
+    else:
+        sys.exit(2)
+except Exception:
+    sys.exit(2)
+PY
+}
 check() {
   local role=$1 svc=$2 file="/run/pd-router/$3" port=$4
   [ -f "$file" ] || return 0
   local ip
   while read -r ip; do
     [ -n "$ip" ] || continue
-    grep -qx "$ip" "$file" && continue
-    # New IP not in argv. Only count drift once it is actually SERVING:
-    # restarting into a still-loading engine is wasted (router can't route
-    # to it) and on engines whose router blocks its own listener on a
-    # per-upstream bootstrap handshake (vllm + mooncake), is actively
-    # harmful -- the whole slice sits dark until the retry budget drains.
-    # Short timeout so a slow engine never stalls the probe itself.
+    # Fast path: does this ip appear in the boot snapshot at all. Fixed-string
+    # match on the ip+pipe prefix -- '|' is a regex meta and would silently
+    # mis-grep (e.g. '10' matching '1.0.x.x'). -F treats it as text.
+    grep -qF "$ip|" "$file" || { unknown_ip=1; }
+    if [ "${unknown_ip:-0}" -eq 1 ]; then
+      # New IP not in argv. Only count drift once it is actually SERVING:
+      # restarting into a still-loading engine is wasted (router can't route
+      # to it) and on engines whose router blocks its own listener on a
+      # per-upstream bootstrap handshake (vllm + mooncake), is actively
+      # harmful -- the whole slice sits dark until the retry budget drains.
+      # Short timeout so a slow engine never stalls the probe itself.
+      if python3 -c "import urllib.request; urllib.request.urlopen('http://$ip:$port/health', timeout=2)" 2>/dev/null; then
+        echo "router stale: $role upstream $ip serving but not in argv" >&2
+        rc=1
+      fi
+      unknown_ip=0
+      continue
+    fi
+    # IP is in argv. Compare identities. want comes from the boot snapshot,
+    # live comes from the sidecar NOW; both may be the '|' sentinel for "no
+    # identity info". Any disagreement means the engine behind this IP is no
+    # longer the incarnation the router argv pined at boot:
+    #
+    #   want REAL, live REAL, different -> in-place restart  (the mooncake
+    #     boot-pinned engine_id can't see this on its own).
+    #   want '|', live REAL              -> identity arrived after boot (slow
+    #     sidecar, or boot budget exhausted mid-model-load). Restart so the
+    #     router re-snapshots with identity -- otherwise that IP stays
+    #     protected-by-silence for the life of this router process.
+    #   want REAL, live '|'              -> probe flake or sidecar wedged.
+    #     We can't tell; treat as no data and skip. An actually-restarted
+    #     engine answers with a REAL different id, so we do NOT miss the
+    #     real engine-swap case by skipping this one.
+    #
+    # Probe errors (skip-tick) are the same rule 1f0d236 uses for DNS blips:
+    # never fire on a missing answer, only when both sides positively
+    # answered and they disagree.
+    want=$(want_of "$ip" "$file")
+    live=$(live_id "$ip") || continue    # probe failed: skip-tick
+    [ "$live" = '|' ] && continue        # live has no data; skip
+    [ "$live" = "$want" ] && continue
     if python3 -c "import urllib.request; urllib.request.urlopen('http://$ip:$port/health', timeout=2)" 2>/dev/null; then
-      echo "router stale: $role upstream $ip serving but not in argv" >&2
+      echo "router stale: $role upstream $ip identity changed ($want -> $live) while serving" >&2
       rc=1
     fi
   done < <(timeout 2 getent hosts "$svc" 2>/dev/null | awk '{print $1}' | sort -u)