diff --git a/web/src/modules/inferences/lib.test.ts b/web/src/modules/inferences/lib.test.ts index e8dcb89..bab4d3f 100644 --- a/web/src/modules/inferences/lib.test.ts +++ b/web/src/modules/inferences/lib.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import type { ObjectResult, Plan, ReleaseStatus, Run } from "@swiss/lib/api"; +import type { LLMScalerStatus, ObjectResult, Plan, ReleaseStatus, Run } from "@swiss/lib/api"; import { age, defaultContainer, @@ -191,6 +191,31 @@ describe("installSteps", () => { it("calls a deleted ModelRoute bad", () => { expect(installSteps(status(), route({ missing: true }) as never, {})[2]!.state).toBe("bad"); }); + + const scaler = (s: LLMScalerStatus) => ({ + result: { ref: { apiVersion: "v1", kind: "LLMScaler", name: "s" } } as ObjectResult, + status: s, + }); + + it("treats ScalingDeferred NotDeferred as steady", () => { + expect(installSteps(status(), {}, scaler({ + currentReplicas: 2, + desiredReplicas: 2, + conditions: [{ type: "ScalingDeferred", status: "False", reason: "NotDeferred" }], + }))[3]).toEqual({ key: "scaler", state: "ok", detail: "steady", params: { n: 2 } }); + }); + it("reports Ready False as failing", () => { + expect(installSteps(status(), {}, scaler({ + conditions: [{ type: "Ready", status: "False", reason: "MetricsUnavailable" }], + }))[3]).toEqual({ key: "scaler", state: "bad", detail: "failing", params: { reason: "MetricsUnavailable" } }); + }); + it("waits while ScalingDeferred is true", () => { + expect(installSteps(status(), {}, scaler({ + currentReplicas: 2, + desiredReplicas: 3, + conditions: [{ type: "ScalingDeferred", status: "True", reason: "RolloutInProgress" }], + }))[3]).toEqual({ key: "scaler", state: "wait", detail: "deferred", params: { reason: "RolloutInProgress" } }); + }); }); describe("age", () => { diff --git a/web/src/modules/inferences/lib.ts b/web/src/modules/inferences/lib.ts index a123e95..75beb8d 100644 --- a/web/src/modules/inferences/lib.ts +++ b/web/src/modules/inferences/lib.ts @@ -12,6 +12,7 @@ import type { Run, } from "@swiss/lib/api"; import { isUninstalling } from "@swiss/lib/phase"; +import { scalerHealth } from "@swiss/lib/scaler"; export type Tone = "success" | "info" | "warning" | "error" | "muted"; @@ -164,8 +165,9 @@ function scalerStep({ result, status }: Picked): if (!result) return { key, state: "off", detail: "none" }; if (result.error) return { key, state: "wait", detail: "unreadable" }; if (result.missing) return { key, state: "bad", detail: "deleted" }; - const failing = status?.conditions?.find((c) => c.status === "False"); - if (failing) return { key, state: "bad", detail: "failing", params: { reason: failing.reason || failing.type } }; + const health = scalerHealth(status); + if (health.kind === "failing") return { key, state: "bad", detail: "failing", params: { reason: health.reason } }; + if (health.kind === "deferred") return { key, state: "wait", detail: "deferred", params: { reason: health.reason } }; const cur = status?.currentReplicas; const want = status?.desiredReplicas; if (cur === undefined) return { key, state: "wait", detail: "noDecision" }; diff --git a/web/src/modules/inferences/locales/en-US.json b/web/src/modules/inferences/locales/en-US.json index 6cac5b3..1fa75a6 100644 --- a/web/src/modules/inferences/locales/en-US.json +++ b/web/src/modules/inferences/locales/en-US.json @@ -197,6 +197,7 @@ "unreadable": "Unreadable", "deleted": "LLMScaler deleted", "failing": "{reason}", + "deferred": "Deferred: {reason}", "noDecision": "No decision yet", "scaling": "{cur} → {want} replicas", "steady": "{n} replicas, steady" diff --git a/web/src/modules/inferences/locales/zh-CN.json b/web/src/modules/inferences/locales/zh-CN.json index 70d5854..67cee16 100644 --- a/web/src/modules/inferences/locales/zh-CN.json +++ b/web/src/modules/inferences/locales/zh-CN.json @@ -186,6 +186,7 @@ "unreadable": "读取失败", "deleted": "LLMScaler 已被删除", "failing": "{reason}", + "deferred": "暂缓伸缩:{reason}", "noDecision": "还没有伸缩决策", "scaling": "{cur} → {want} 副本", "steady": "{n} 副本,稳定" diff --git a/web/src/modules/swiss/components/ReleaseObjects.tsx b/web/src/modules/swiss/components/ReleaseObjects.tsx index 6bfa9e2..6dd1e09 100644 --- a/web/src/modules/swiss/components/ReleaseObjects.tsx +++ b/web/src/modules/swiss/components/ReleaseObjects.tsx @@ -17,6 +17,7 @@ import { HoverHint } from "@swiss/components/ui/hint"; import { Button } from "@swiss/components/ui/button"; import { Card } from "@swiss/components/ui/card"; import { ErrorState } from "@swiss/components/States"; +import { scalerHealth } from "@swiss/lib/scaler"; import { cn, timeAgo } from "@swiss/lib/utils"; // Unpolled, it loads once per visit and then moves only when the page's own @@ -246,12 +247,14 @@ function ScalerPanel({ ); const current = status?.currentReplicas; const desired = status?.desiredReplicas; - const failing = status?.conditions?.find((c) => c.status === "False"); + const health = scalerHealth(status); const custom = spec?.metricProvider === "Custom"; let state: React.ReactNode = null; - if (failing) - state = {failing.reason || failing.type}; + if (health.kind === "failing") + state = {health.reason}; + else if (health.kind === "deferred") + state = deferred: {health.reason}; else if (current !== undefined && desired !== undefined) { if (desired > current) state = scaling up; else if (desired < current) @@ -324,6 +327,9 @@ function ScalerPanel({ {spec.syncPeriodSeconds !== undefined && ( {spec.syncPeriodSeconds}s )} + {status?.lastScaleTime && ( + {timeAgo(status.lastScaleTime)} + )} {!custom && !!spec.metrics?.length && ( @@ -349,6 +355,10 @@ function ScalerPanel({ )} + )} diff --git a/web/src/modules/swiss/lib/api.ts b/web/src/modules/swiss/lib/api.ts index b5bcf73..9b920b3 100644 --- a/web/src/modules/swiss/lib/api.ts +++ b/web/src/modules/swiss/lib/api.ts @@ -759,6 +759,10 @@ export interface LLMScalerSpec { export interface LLMScalerStatus { currentReplicas?: number; desiredReplicas?: number; + appliedReplicas?: number; + observedGeneration?: number; + lastScaleTime?: string; + deferredSince?: string; conditions?: Condition[]; } diff --git a/web/src/modules/swiss/lib/scaler.test.ts b/web/src/modules/swiss/lib/scaler.test.ts new file mode 100644 index 0000000..027630e --- /dev/null +++ b/web/src/modules/swiss/lib/scaler.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from "vitest"; +import type { Condition } from "@swiss/lib/api"; +import { scalerHealth, type ScalerHealth } from "@swiss/lib/scaler"; + +const c = (type: string, status: Condition["status"], reason?: string): Condition => ({ type, status, reason }); + +describe("scalerHealth", () => { + it.each<{ name: string; conditions?: Condition[]; want: ScalerHealth }>([ + { + name: "Ready False is failing with its reason", + conditions: [c("Ready", "False", "TargetWriteFailed")], + want: { kind: "failing", reason: "TargetWriteFailed" }, + }, + { + name: "Ready True and ScalingDeferred RolloutInProgress is deferred", + conditions: [c("Ready", "True", "Synced"), c("ScalingDeferred", "True", "RolloutInProgress")], + want: { kind: "deferred", reason: "RolloutInProgress" }, + }, + { + name: "Ready True and ScalingDeferred NotDeferred is ok", + conditions: [c("Ready", "True", "Synced"), c("ScalingDeferred", "False", "NotDeferred")], + want: { kind: "ok" }, + }, + { + name: "no Ready and ScalingDeferred NotDeferred is ok", + conditions: [c("ScalingDeferred", "False", "NotDeferred")], + want: { kind: "ok" }, + }, + { + name: "no Ready and DecisionAvailable False is failing", + conditions: [c("DecisionAvailable", "False", "DecisionUnavailable")], + want: { kind: "failing", reason: "DecisionUnavailable" }, + }, + { + name: "no Ready and ScalingDeferred ScaleDownSettling is deferred", + conditions: [c("ScalingDeferred", "True", "ScaleDownSettling")], + want: { kind: "deferred", reason: "ScaleDownSettling" }, + }, + { + name: "Ready True wins over DecisionAvailable False", + conditions: [c("Ready", "True", "Synced"), c("DecisionAvailable", "False", "DecisionUnavailable")], + want: { kind: "ok" }, + }, + { + name: "AbleToScale False is failing with its reason", + conditions: [c("AbleToScale", "False", "TargetNotFound")], + want: { kind: "failing", reason: "TargetNotFound" }, + }, + { + name: "Ready False outranks AbleToScale", + conditions: [c("Ready", "False", "DecisionUnavailable"), c("AbleToScale", "False", "TargetWriteFailed")], + want: { kind: "failing", reason: "DecisionUnavailable" }, + }, + { + name: "AbleToScale True with RolloutInProgress is deferred", + conditions: [c("Ready", "True", "Synced"), c("AbleToScale", "True", "RolloutInProgress")], + want: { kind: "deferred", reason: "RolloutInProgress" }, + }, + { + name: "AbleToScale True with ScaleDownSettling is deferred", + conditions: [c("Ready", "True", "Synced"), c("AbleToScale", "True", "ScaleDownSettling")], + want: { kind: "deferred", reason: "ScaleDownSettling" }, + }, + { + name: "AbleToScale True with Synced is ok", + conditions: [c("Ready", "True", "Synced"), c("AbleToScale", "True", "Synced")], + want: { kind: "ok" }, + }, + { + name: "AbleToScale True with DeferralTimedOut is ok", + conditions: [c("Ready", "True", "Synced"), c("AbleToScale", "True", "DeferralTimedOut")], + want: { kind: "ok" }, + }, + { + name: "AbleToScale wins over a leftover ScalingDeferred", + conditions: [c("AbleToScale", "True", "Synced"), c("ScalingDeferred", "True", "RolloutInProgress")], + want: { kind: "ok" }, + }, + { + name: "no Ready and ScalingDeferred DeferralTimedOut is ok", + conditions: [c("ScalingDeferred", "False", "DeferralTimedOut")], + want: { kind: "ok" }, + }, + { + name: "no conditions is ok", + want: { kind: "ok" }, + }, + ])("$name", ({ conditions, want }) => { + expect(scalerHealth(conditions ? { conditions } : {})).toEqual(want); + }); +}); diff --git a/web/src/modules/swiss/lib/scaler.ts b/web/src/modules/swiss/lib/scaler.ts new file mode 100644 index 0000000..38fa205 --- /dev/null +++ b/web/src/modules/swiss/lib/scaler.ts @@ -0,0 +1,37 @@ +import type { Condition, LLMScalerStatus } from "@swiss/lib/api"; + +export type ScalerHealth = + | { kind: "failing"; reason: string } + | { kind: "deferred"; reason: string } + | { kind: "ok" }; + +// AbleToScale=True with one of these means a guard is holding the write. +const HOLD_REASONS = new Set(["RolloutInProgress", "ScaleDownSettling"]); + +function condition(status: LLMScalerStatus | undefined, type: string): Condition | undefined { + return status?.conditions?.find((c) => c.type === type); +} + +function reason(c: Condition): string { + return c.reason || c.type; +} + +// Older operators publish neither Ready nor AbleToScale: DecisionAvailable=False +// is the only failure there, and ScalingDeferred=True (not False) is a hold. +export function scalerHealth(status?: LLMScalerStatus): ScalerHealth { + const ready = condition(status, "Ready"); + if (ready?.status === "False") return { kind: "failing", reason: reason(ready) }; + const able = condition(status, "AbleToScale"); + if (able?.status === "False") return { kind: "failing", reason: reason(able) }; + if (!ready) { + const decision = condition(status, "DecisionAvailable"); + if (decision?.status === "False") return { kind: "failing", reason: reason(decision) }; + } + if (able) { + if (able.status === "True" && able.reason && HOLD_REASONS.has(able.reason)) return { kind: "deferred", reason: able.reason }; + return { kind: "ok" }; + } + const deferred = condition(status, "ScalingDeferred"); + if (deferred?.status === "True") return { kind: "deferred", reason: reason(deferred) }; + return { kind: "ok" }; +} diff --git a/web/src/modules/swiss/routes/DeploymentDetail.tsx b/web/src/modules/swiss/routes/DeploymentDetail.tsx index 23c6f9e..17e7bfd 100644 --- a/web/src/modules/swiss/routes/DeploymentDetail.tsx +++ b/web/src/modules/swiss/routes/DeploymentDetail.tsx @@ -39,6 +39,7 @@ import { SLODialog } from "@swiss/components/SLOCard"; import { ErrorState, Loading } from "@swiss/components/States"; import { METHODS } from "@swiss/components/InstallMethod"; import { isUninstalling } from "@swiss/lib/phase"; +import { scalerHealth } from "@swiss/lib/scaler"; import { cn, timeAgo } from "@swiss/lib/utils"; export function DeploymentDetail() { @@ -342,8 +343,9 @@ function scalerStep({ result, status }: ReturnType c.status === "False"); - if (failing) return { label, state: "bad", detail: failing.reason || failing.type }; + const health = scalerHealth(status); + if (health.kind === "failing") return { label, state: "bad", detail: health.reason }; + if (health.kind === "deferred") return { label, state: "wait", detail: `deferred: ${health.reason}` }; const cur = status?.currentReplicas; const want = status?.desiredReplicas; if (cur === undefined) return { label, state: "wait", detail: "no decision yet" };