diff --git a/charts/sglang/Chart.yaml b/charts/sglang/Chart.yaml index 19722d0..31035b4 100644 --- a/charts/sglang/Chart.yaml +++ b/charts/sglang/Chart.yaml @@ -5,7 +5,7 @@ home: https://github.com/modelsphere/helm-charts sources: - https://github.com/modelsphere/helm-charts type: application -version: 0.9.17 +version: 0.9.18 appVersion: "1.0.0" # CART is vendored under charts/cart -- already present, so no `helm dependency diff --git a/charts/sglang/templates/_pod.tpl b/charts/sglang/templates/_pod.tpl index 1698b9f..563f471 100644 --- a/charts/sglang/templates/_pod.tpl +++ b/charts/sglang/templates/_pod.tpl @@ -957,9 +957,10 @@ exit 0 sglang.pdRouterProbes -- separate concerns. */ -}} {{- define "sglang.pdRouterLivenessProbe" -}} {{- $roles := .roles -}} +{{- $root := .root -}} livenessProbe: exec: - command: ["bash", "-lc", {{ include "sglang.pdRouterLivenessProbeScript" (dict "roles" $roles) | quote }}] + command: ["bash", "-lc", {{ include "sglang.pdRouterLivenessProbeScript" (dict "root" $root "roles" $roles) | quote }}] periodSeconds: 10 timeoutSeconds: 10 failureThreshold: 2 @@ -967,23 +968,88 @@ livenessProbe: {{- define "sglang.pdRouterLivenessProbeScript" -}} {{- $roles := .roles -}} +{{- $root := .root -}} set -u rc=0 +HW_PORT={{ ternary (int $root.Values.hangWatcher.port) 0 $root.Values.hangWatcher.enabled }} +# Read the engine-side identity that the router boot script recorded for $ip. +# Empty tuple means "no identity info" -- old sidecar, HW_PORT=0 at boot, or +# the boot-time 90min budget exhausted. The live probe below returns '|' for +# the same condition. Fires iff the two sides disagree on identity: boot said +# sentinel while live says real (identity appeared late), OR boot said real +# while live says a different real (engine restarted in place). +want_of() { awk -F'|' -v ip="$1" '$1==ip { print $2"|"$3; exit }' "$2"; } +# Probe the live identity. 404 = old sidecar, same as boot sentinel; any +# other non-200 (503 mid-id-pin, conn refused, timeout) = skip-tick. Both +# shapes return exactly one parseable line on success and nothing else. +live_id() { + local ip=$1 + [ "$HW_PORT" -eq 0 ] && { echo '|'; return 0; } + python3 - "$ip" "$HW_PORT" <<'PY' 2>/dev/null +import json, sys, urllib.request, urllib.error +ip, port = sys.argv[1], sys.argv[2] +try: + r = urllib.request.urlopen(f"http://{ip}:{port}/engine", timeout=2) + j = json.loads(r.read()) + print(f"{j['restarts']}|{j['watcher_started_at']}") +except urllib.error.HTTPError as e: + if e.code == 404: + print('|') + else: + sys.exit(2) +except Exception: + sys.exit(2) +PY +} check() { local role=$1 svc=$2 file="/run/pd-router/$3" port=$4 [ -f "$file" ] || return 0 local ip while read -r ip; do [ -n "$ip" ] || continue - grep -qx "$ip" "$file" && continue - # New IP not in argv. Only count drift once it is actually SERVING: - # restarting into a still-loading engine is wasted (router can't route - # to it) and on engines whose router blocks its own listener on a - # per-upstream bootstrap handshake (vllm + mooncake), is actively - # harmful -- the whole slice sits dark until the retry budget drains. - # Short timeout so a slow engine never stalls the probe itself. + # Fast path: does this ip appear in the boot snapshot at all. Fixed-string + # match on the ip+pipe prefix -- '|' is a regex meta and would silently + # mis-grep (e.g. '10' matching '1.0.x.x'). -F treats it as text. + grep -qF "$ip|" "$file" || { unknown_ip=1; } + if [ "${unknown_ip:-0}" -eq 1 ]; then + # New IP not in argv. Only count drift once it is actually SERVING: + # restarting into a still-loading engine is wasted (router can't route + # to it) and on engines whose router blocks its own listener on a + # per-upstream bootstrap handshake (vllm + mooncake), is actively + # harmful -- the whole slice sits dark until the retry budget drains. + # Short timeout so a slow engine never stalls the probe itself. + if python3 -c "import urllib.request; urllib.request.urlopen('http://$ip:$port/health', timeout=2)" 2>/dev/null; then + echo "router stale: $role upstream $ip serving but not in argv" >&2 + rc=1 + fi + unknown_ip=0 + continue + fi + # IP is in argv. Compare identities. want comes from the boot snapshot, + # live comes from the sidecar NOW; both may be the '|' sentinel for "no + # identity info". Any disagreement means the engine behind this IP is no + # longer the incarnation the router argv pined at boot: + # + # want REAL, live REAL, different -> in-place restart (the mooncake + # boot-pinned engine_id can't see this on its own). + # want '|', live REAL -> identity arrived after boot (slow + # sidecar, or boot budget exhausted mid-model-load). Restart so the + # router re-snapshots with identity -- otherwise that IP stays + # protected-by-silence for the life of this router process. + # want REAL, live '|' -> probe flake or sidecar wedged. + # We can't tell; treat as no data and skip. An actually-restarted + # engine answers with a REAL different id, so we do NOT miss the + # real engine-swap case by skipping this one. + # + # Probe errors (skip-tick) are the same rule 1f0d236 uses for DNS blips: + # never fire on a missing answer, only when both sides positively + # answered and they disagree. + want=$(want_of "$ip" "$file") + live=$(live_id "$ip") || continue # probe failed: skip-tick + [ "$live" = '|' ] && continue # live has no data; skip + [ "$live" = "$want" ] && continue if python3 -c "import urllib.request; urllib.request.urlopen('http://$ip:$port/health', timeout=2)" 2>/dev/null; then - echo "router stale: $role upstream $ip serving but not in argv" >&2 + echo "router stale: $role upstream $ip identity changed ($want -> $live) while serving" >&2 rc=1 fi done < <(timeout 2 getent hosts "$svc" 2>/dev/null | awk '{print $1}' | sort -u) diff --git a/charts/sglang/templates/pd-ds.yaml b/charts/sglang/templates/pd-ds.yaml index 012b877..715e7fa 100644 --- a/charts/sglang/templates/pd-ds.yaml +++ b/charts/sglang/templates/pd-ds.yaml @@ -247,9 +247,74 @@ spec: # Persist the boot-time upstream set for the liveness drift probe. # Router argv is immutable for the life of the process, so the file # is the probe's only handle on "what upstreams were baked in". + # + # Each line is "||" from the + # engine pod's hang-watcher /engine endpoint. The ip alone isn't a + # stable identity: sglang-router pins bootstrap info at boot on + # TOP of the IP, so an engine that crashes and is restarted in place + # by the kubelet keeps its IP but swaps engine_id -- the router's + # transfer sessions then point at a dead incarnation while DNS + # shows nothing new. Comparing (restarts, watcher_started_at) is + # version-agnostic: any hang-watcher that exposes /engine yields a + # fresh tuple for every engine process replacement. + # + # HW_PORT=0 is the "sidecar disabled at render time" sentinel and + # behaves identically to the older-sidecar 404: record "||" and + # move on, the runtime probe treats both as "no identity info". + # 503 = sidecar alive but engine id not yet pinned; retry up to + # 90min, then give up and record the same sentinel (fail-open: + # identity unknown is not a reason to keep the router from ever + # booting). engine not yet listening (conn refused) and CoreDNS + # slop (timeout) also retry on the same budget. + HW_PORT={{ ternary (int $root.Values.hangWatcher.port) 0 $root.Values.hangWatcher.enabled }} + probe_identity() { + local ip=$1 budget=${2:-5400} waited=0 + [ "$HW_PORT" -eq 0 ] && { echo '|'; return 0; } + while :; do + local out + # python3 heredoc: body and terminator sit at the block scalar's + # own indent, so the rendered yaml strips them to column 0 for + # bash+python. Exit 0 prints '|' on 200 or '|' on + # 404 sentinel; exit 2 on 503 / conn refused / timeout / bad + # json = transient, caller retries on its own budget. + out=$(python3 - "$ip" "$HW_PORT" <<'PY' 2>/dev/null + import json, sys, urllib.request, urllib.error + ip, port = sys.argv[1], sys.argv[2] + try: + r = urllib.request.urlopen(f"http://{ip}:{port}/engine", timeout=5) + j = json.loads(r.read()) + print(f"{j['restarts']}|{j['watcher_started_at']}") + except urllib.error.HTTPError as e: + if e.code == 404: + print('|') + else: + sys.exit(2) + except Exception: + sys.exit(2) + PY + ) + local rrc=$? + if [ $rrc -eq 0 ]; then echo "$out"; return 0; fi + if [ $waited -ge $budget ]; then + echo "router: $ip identity probe gave no usable answer after ${budget}s; recording ip||" >&2 + echo '|'; return 0 + fi + sleep 10; waited=$((waited+10)) + done + } install -d -m 0755 /run/pd-router - T=$(mktemp -p /run/pd-router); printf '%s\n' "$PF_IPS" >"$T"; mv "$T" /run/pd-router/prefill-ips - T=$(mktemp -p /run/pd-router); printf '%s\n' "$DC_IPS" >"$T"; mv "$T" /run/pd-router/decode-ips + write_role_snapshot() { + local role=$1 ips=$2 + local T=$(mktemp -p /run/pd-router) + local ip id + for ip in $ips; do + id=$(probe_identity "$ip") + printf '%s|%s\n' "$ip" "$id" >>"$T" + done + mv "$T" "/run/pd-router/${role}-ips" + } + write_role_snapshot prefill "$PF_IPS" + write_role_snapshot decode "$DC_IPS" # shellcheck disable=SC2086 exec python3 -m sglang_router.launch_router \ --pd-disaggregation \ @@ -273,7 +338,7 @@ spec: {{- /* Drift liveness: restart the router if a new prefill/decode IP appears in DNS after boot -- argv is immutable and the running router can never learn about post-boot upstreams. */}} - {{- include "sglang.pdRouterLivenessProbe" (dict "roles" (dict + {{- include "sglang.pdRouterLivenessProbe" (dict "root" $root "roles" (dict "prefill" (dict "host" (printf "%s-prefill-hl-s%d.%s.svc.cluster.local." $fullname $s $root.Release.Namespace) "port" ($pd.prefill.httpPort | default 30001)) "decode" (dict "host" (printf "%s-decode-hl-s%d.%s.svc.cluster.local." $fullname $s $root.Release.Namespace) "port" ($pd.decode.httpPort | default 30002)))) | nindent 8 }} ports: diff --git a/charts/sglang/values.yaml b/charts/sglang/values.yaml index cc67b47..5d6a6b7 100644 --- a/charts/sglang/values.yaml +++ b/charts/sglang/values.yaml @@ -488,7 +488,12 @@ hangWatcher: # silently (unknown JSON fields are dropped), so the grace period would look # configured and do nothing -- pin the tag rather than rely on the cluster # happening to run a new enough image. - tag: "0.1.26" + # >= 0.1.27 is required for GET /engine, the endpoint the pd router's + # drift-liveness probe uses to detect in-place engine restart. Older + # sidecars return 404 and the chart falls back to pre-identity behavior + # (IP-growth only) -- silent, no log. If you must run an older sidecar, + # expect that failure mode not this protection. + tag: "0.1.27" port: 9090 # /healthz port (just needs to differ from service.port) # Thresholds -> ConfigMap the sidecar hot-reloads (helm upgrade retunes, no pod restart). config: diff --git a/charts/vllm/Chart.yaml b/charts/vllm/Chart.yaml index e81bc55..f40eb8d 100644 --- a/charts/vllm/Chart.yaml +++ b/charts/vllm/Chart.yaml @@ -5,7 +5,7 @@ home: https://github.com/modelsphere/helm-charts sources: - https://github.com/modelsphere/helm-charts type: application -version: 0.8.16 +version: 0.8.17 appVersion: "1.0.0" # CART is vendored under charts/cart -- already present, so no `helm dependency diff --git a/charts/vllm/templates/_pod.tpl b/charts/vllm/templates/_pod.tpl index 0ae40c3..1f0afa4 100644 --- a/charts/vllm/templates/_pod.tpl +++ b/charts/vllm/templates/_pod.tpl @@ -996,9 +996,10 @@ exit 0 vllm.pdRouterProbes -- separate concerns. */ -}} {{- define "vllm.pdRouterLivenessProbe" -}} {{- $roles := .roles -}} +{{- $root := .root -}} livenessProbe: exec: - command: ["bash", "-lc", {{ include "vllm.pdRouterLivenessProbeScript" (dict "roles" $roles) | quote }}] + command: ["bash", "-lc", {{ include "vllm.pdRouterLivenessProbeScript" (dict "root" $root "roles" $roles) | quote }}] periodSeconds: 10 timeoutSeconds: 10 failureThreshold: 2 @@ -1006,23 +1007,88 @@ livenessProbe: {{- define "vllm.pdRouterLivenessProbeScript" -}} {{- $roles := .roles -}} +{{- $root := .root -}} set -u rc=0 +HW_PORT={{ ternary (int $root.Values.hangWatcher.port) 0 $root.Values.hangWatcher.enabled }} +# Read the engine-side identity that the router boot script recorded for $ip. +# Empty tuple means "no identity info" -- old sidecar, HW_PORT=0 at boot, or +# the boot-time 90min budget exhausted. The live probe below returns '|' for +# the same condition. Fires iff the two sides disagree on identity: boot said +# sentinel while live says real (identity appeared late), OR boot said real +# while live says a different real (engine restarted in place). +want_of() { awk -F'|' -v ip="$1" '$1==ip { print $2"|"$3; exit }' "$2"; } +# Probe the live identity. 404 = old sidecar, same as boot sentinel; any +# other non-200 (503 mid-id-pin, conn refused, timeout) = skip-tick. Both +# shapes return exactly one parseable line on success and nothing else. +live_id() { + local ip=$1 + [ "$HW_PORT" -eq 0 ] && { echo '|'; return 0; } + python3 - "$ip" "$HW_PORT" <<'PY' 2>/dev/null +import json, sys, urllib.request, urllib.error +ip, port = sys.argv[1], sys.argv[2] +try: + r = urllib.request.urlopen(f"http://{ip}:{port}/engine", timeout=2) + j = json.loads(r.read()) + print(f"{j['restarts']}|{j['watcher_started_at']}") +except urllib.error.HTTPError as e: + if e.code == 404: + print('|') + else: + sys.exit(2) +except Exception: + sys.exit(2) +PY +} check() { local role=$1 svc=$2 file="/run/pd-router/$3" port=$4 [ -f "$file" ] || return 0 local ip while read -r ip; do [ -n "$ip" ] || continue - grep -qx "$ip" "$file" && continue - # New IP not in argv. Only count drift once it is actually SERVING: - # restarting into a still-loading engine is wasted (router can't route - # to it) and on engines whose router blocks its own listener on a - # per-upstream bootstrap handshake (vllm + mooncake), is actively - # harmful -- the whole slice sits dark until the retry budget drains. - # Short timeout so a slow engine never stalls the probe itself. + # Fast path: does this ip appear in the boot snapshot at all. Fixed-string + # match on the ip+pipe prefix -- '|' is a regex meta and would silently + # mis-grep (e.g. '10' matching '1.0.x.x'). -F treats it as text. + grep -qF "$ip|" "$file" || { unknown_ip=1; } + if [ "${unknown_ip:-0}" -eq 1 ]; then + # New IP not in argv. Only count drift once it is actually SERVING: + # restarting into a still-loading engine is wasted (router can't route + # to it) and on engines whose router blocks its own listener on a + # per-upstream bootstrap handshake (vllm + mooncake), is actively + # harmful -- the whole slice sits dark until the retry budget drains. + # Short timeout so a slow engine never stalls the probe itself. + if python3 -c "import urllib.request; urllib.request.urlopen('http://$ip:$port/health', timeout=2)" 2>/dev/null; then + echo "router stale: $role upstream $ip serving but not in argv" >&2 + rc=1 + fi + unknown_ip=0 + continue + fi + # IP is in argv. Compare identities. want comes from the boot snapshot, + # live comes from the sidecar NOW; both may be the '|' sentinel for "no + # identity info". Any disagreement means the engine behind this IP is no + # longer the incarnation the router argv pined at boot: + # + # want REAL, live REAL, different -> in-place restart (the mooncake + # boot-pinned engine_id can't see this on its own). + # want '|', live REAL -> identity arrived after boot (slow + # sidecar, or boot budget exhausted mid-model-load). Restart so the + # router re-snapshots with identity -- otherwise that IP stays + # protected-by-silence for the life of this router process. + # want REAL, live '|' -> probe flake or sidecar wedged. + # We can't tell; treat as no data and skip. An actually-restarted + # engine answers with a REAL different id, so we do NOT miss the + # real engine-swap case by skipping this one. + # + # Probe errors (skip-tick) are the same rule 1f0d236 uses for DNS blips: + # never fire on a missing answer, only when both sides positively + # answered and they disagree. + want=$(want_of "$ip" "$file") + live=$(live_id "$ip") || continue # probe failed: skip-tick + [ "$live" = '|' ] && continue # live has no data; skip + [ "$live" = "$want" ] && continue if python3 -c "import urllib.request; urllib.request.urlopen('http://$ip:$port/health', timeout=2)" 2>/dev/null; then - echo "router stale: $role upstream $ip serving but not in argv" >&2 + echo "router stale: $role upstream $ip identity changed ($want -> $live) while serving" >&2 rc=1 fi done < <(timeout 2 getent hosts "$svc" 2>/dev/null | awk '{print $1}' | sort -u) diff --git a/charts/vllm/templates/pd-ds.yaml b/charts/vllm/templates/pd-ds.yaml index 3cf62a6..cd0c8dd 100644 --- a/charts/vllm/templates/pd-ds.yaml +++ b/charts/vllm/templates/pd-ds.yaml @@ -296,9 +296,74 @@ spec: # Persist the boot-time upstream set for the liveness drift probe. # Router argv is immutable for the life of the process, so the file # is the probe's only handle on "what upstreams were baked in". + # + # Each line is "||" from the + # engine pod's hang-watcher /engine endpoint. The ip alone isn't a + # stable identity: vllm-router (mooncake) pins engine_id at boot on + # TOP of the IP, so an engine that crashes and is restarted in place + # by the kubelet keeps its IP but swaps engine_id -- the router's + # transfer sessions then point at a dead incarnation while DNS + # shows nothing new. Comparing (restarts, watcher_started_at) is + # version-agnostic: any hang-watcher that exposes /engine yields a + # fresh tuple for every engine process replacement. + # + # HW_PORT=0 is the "sidecar disabled at render time" sentinel and + # behaves identically to the older-sidecar 404: record "||" and + # move on, the runtime probe treats both as "no identity info". + # 503 = sidecar alive but engine id not yet pinned; retry up to + # 90min, then give up and record the same sentinel (fail-open: + # identity unknown is not a reason to keep the router from ever + # booting). engine not yet listening (conn refused) and CoreDNS + # slop (timeout) also retry on the same budget. + HW_PORT={{ ternary (int $root.Values.hangWatcher.port) 0 $root.Values.hangWatcher.enabled }} + probe_identity() { + local ip=$1 budget=${2:-5400} waited=0 + [ "$HW_PORT" -eq 0 ] && { echo '|'; return 0; } + while :; do + local out + # python3 heredoc: body and terminator sit at the block scalar's + # own indent, so the rendered yaml strips them to column 0 for + # bash+python. Exit 0 prints '|' on 200 or '|' on + # 404 sentinel; exit 2 on 503 / conn refused / timeout / bad + # json = transient, caller retries on its own budget. + out=$(python3 - "$ip" "$HW_PORT" <<'PY' 2>/dev/null + import json, sys, urllib.request, urllib.error + ip, port = sys.argv[1], sys.argv[2] + try: + r = urllib.request.urlopen(f"http://{ip}:{port}/engine", timeout=5) + j = json.loads(r.read()) + print(f"{j['restarts']}|{j['watcher_started_at']}") + except urllib.error.HTTPError as e: + if e.code == 404: + print('|') + else: + sys.exit(2) + except Exception: + sys.exit(2) + PY + ) + local rrc=$? + if [ $rrc -eq 0 ]; then echo "$out"; return 0; fi + if [ $waited -ge $budget ]; then + echo "router: $ip identity probe gave no usable answer after ${budget}s; recording ip||" >&2 + echo '|'; return 0 + fi + sleep 10; waited=$((waited+10)) + done + } install -d -m 0755 /run/pd-router - T=$(mktemp -p /run/pd-router); printf '%s\n' "$PF_IPS" >"$T"; mv "$T" /run/pd-router/prefill-ips - T=$(mktemp -p /run/pd-router); printf '%s\n' "$DC_IPS" >"$T"; mv "$T" /run/pd-router/decode-ips + write_role_snapshot() { + local role=$1 ips=$2 + local T=$(mktemp -p /run/pd-router) + local ip id + for ip in $ips; do + id=$(probe_identity "$ip") + printf '%s|%s\n' "$ip" "$id" >>"$T" + done + mv "$T" "/run/pd-router/${role}-ips" + } + write_role_snapshot prefill "$PF_IPS" + write_role_snapshot decode "$DC_IPS" # shellcheck disable=SC2086 exec vllm-router \ --vllm-pd-disaggregation \ @@ -322,7 +387,7 @@ spec: {{- /* Drift liveness: restart the router if a new prefill/decode IP appears in DNS after boot -- argv is immutable and the running router can never learn about post-boot upstreams. */}} - {{- include "vllm.pdRouterLivenessProbe" (dict "roles" $probeRoles) | nindent 8 }} + {{- include "vllm.pdRouterLivenessProbe" (dict "root" $root "roles" $probeRoles) | nindent 8 }} {{- with $root.Values.env }} env: {{- include "vllm.mergeEnv" (dict "chart" list "user" . "padding" 8) }} diff --git a/charts/vllm/values.yaml b/charts/vllm/values.yaml index 42038b0..f4afee8 100644 --- a/charts/vllm/values.yaml +++ b/charts/vllm/values.yaml @@ -609,7 +609,12 @@ hangWatcher: # silently (unknown JSON fields are dropped), so the grace period would look # configured and do nothing -- pin the tag rather than rely on the cluster # happening to run a new enough image. - tag: "0.1.26" + # >= 0.1.27 is required for GET /engine, the endpoint the pd router's + # drift-liveness probe uses to detect in-place engine restart. Older + # sidecars return 404 and the chart falls back to pre-identity behavior + # (IP-growth only) -- silent, no log. If you must run an older sidecar, + # expect that failure mode not this protection. + tag: "0.1.27" port: 9090 # /healthz port (just needs to differ from service.port) # Thresholds -> ConfigMap the sidecar hot-reloads (helm upgrade retunes, no pod restart). config: