diff --git a/web/src/modules/docs/content/en-US/deploy.md b/web/src/modules/docs/content/en-US/deploy.md index eac76af..d639202 100644 --- a/web/src/modules/docs/content/en-US/deploy.md +++ b/web/src/modules/docs/content/en-US/deploy.md @@ -77,6 +77,7 @@ From a row's actions in the list, or the top of the detail page: - **Roll back**: pick a revision under Versions to restore its plan as it was. To change settings, upgrade instead. - **Edit SLO**: the latency and throughput targets and the replica bounds the scaler works with. Services deployed without SLO need it switched on in an upgrade first. - **Uninstall**: stops the service and deletes its pods; the activity log is kept. Reinstalling loads the weights again. +- While pods are still terminating the service stays listed as **Uninstalling**; upgrade, roll back, SLO edits and uninstall stay disabled until it disappears. "Behind catalog" in the list means the catalog has a newer version to upgrade to; "Untracked" means the release was not deployed by Swiss and cannot be upgraded here. diff --git a/web/src/modules/docs/content/zh-CN/deploy.md b/web/src/modules/docs/content/zh-CN/deploy.md index 0e276ba..1d28646 100644 --- a/web/src/modules/docs/content/zh-CN/deploy.md +++ b/web/src/modules/docs/content/zh-CN/deploy.md @@ -76,6 +76,7 @@ - **回滚**：在「版本管理」里选一个历史版本，原样恢复当时的计划。要改设置请用升级。 - **编辑 SLO**：调整扩缩容依据的延迟、吞吐目标和副本数上下限。部署时没开 SLO 的服务需要先在升级时打开。 - **卸载**：停止服务并删除 Pod，操作记录保留。重新安装需要重新加载权重。 +- Pod 还在退出时，服务会以 **卸载中** 留在列表里；在它消失之前，升级、回滚、编辑 SLO 和卸载都不可用。 列表上的「落后目录」表示目录里有更新的版本，可以升级；「未纳管」表示这个 release 不是通过 swiss 部署的，不能在这里升级。 diff --git a/web/src/modules/inferences/InferenceDetail.tsx b/web/src/modules/inferences/InferenceDetail.tsx index ab94f57..e656897 100644 --- a/web/src/modules/inferences/InferenceDetail.tsx +++ b/web/src/modules/inferences/InferenceDetail.tsx @@ -5,6 +5,7 @@ import { Alert, AlertDescription, DetailHeader, PanelTabs, SectionCard, Skeleton import { ArrowLeft, TriangleAlert } from "lucide-react"; import { useModulePath } from "@/shell"; import { api } from "@swiss/lib/api"; +import { isUninstalling } from "@swiss/lib/phase"; import { useReleaseObjects } from "@swiss/components/ReleaseObjects"; import { ReleaseObjects } from "@swiss/components/ReleaseObjects"; import { Endpoint } from "@swiss/components/Endpoint"; @@ -100,11 +101,13 @@ export function InferenceDetail() { } const readOnly = cluster.data ? !cluster.data.allowDeploy : false; + const removing = isUninstalling(s); + const blockedWrite = readOnly ? t("disabled.readOnly") : removing ? t("disabled.uninstalling") : false; const source = plan.data?.source; const model = modelLine(source ?? {}) || s.model; const pods = s.pods.map((p) => p.name); const blocked = sloBlocked(s, plan.data); - const sloOpen = editingSLO || (sloSetup && blocked !== "off" && s.planStatus?.phase !== "failed"); + const sloOpen = !removing && (editingSLO || (sloSetup && blocked !== "off" && s.planStatus?.phase !== "failed")); const closeSLO = () => { setEditingSLO(false); if (!sloSetup) return; @@ -127,20 +130,20 @@ export function InferenceDetail() { { key: "slo", label: t("actions.slo"), - disabled: blocked === "off" ? t("disabled.sloOff") : blocked === "pending" ? t("disabled.sloPending") : false, + disabled: blockedWrite || (blocked === "off" ? t("disabled.sloOff") : blocked === "pending" ? t("disabled.sloPending") : false), onClick: () => setEditingSLO(true), }, { key: "upgrade", label: t("actions.upgrade"), - disabled: readOnly ? t("disabled.readOnly") : !plan.data ? t("disabled.untracked") : false, + disabled: blockedWrite || (!plan.data ? t("disabled.untracked") : false), onClick: () => setUpgrading(true), }, { key: "uninstall", label: t("actions.uninstall"), danger: true, - disabled: readOnly ? t("disabled.readOnly") : !s.exists, + disabled: blockedWrite || !s.exists, onClick: () => setUninstalling(true), }, ]} @@ -173,7 +176,7 @@ export function InferenceDetail() { - + @@ -187,7 +190,7 @@ export function InferenceDetail() { )} - + setUpgrading(false)} /> (d.route ? /{d.route}/ : -), }, { key: "chart", title: t("list.columns.chart"), width: 180, render: (d) => {d.chart || "-"} }, - { key: "revision", title: t("list.columns.revision"), width: 70, align: "right", render: (d) => {d.revision} }, + { key: "revision", title: t("list.columns.revision"), width: 70, align: "right", render: (d) => {d.revision ?? "-"} }, { key: "updated", title: t("list.columns.updated"), @@ -97,19 +98,19 @@ export function InferenceList() { }, ]; - const writeBlocked = (d: Deployment): boolean | string => - readOnly ? t("disabled.readOnly") : !d.model ? t("disabled.untracked") : false; + const writeBlocked = (d: Deployment, untracked = false): boolean | string => + readOnly ? t("disabled.readOnly") : isUninstalling(d) ? t("disabled.uninstalling") : untracked && !d.model ? t("disabled.untracked") : false; const rowActions: ResourceRowAction[] = [ { key: "detail", label: t("actions.detail"), onClick: open }, - { key: "upgrade", label: t("actions.upgrade"), onClick: (d) => setUpgrading({ kind: "upgrade", namespace: d.namespace, release: d.release }), disabled: writeBlocked }, + { key: "upgrade", label: t("actions.upgrade"), onClick: (d) => setUpgrading({ kind: "upgrade", namespace: d.namespace, release: d.release }), disabled: (d) => writeBlocked(d, true) }, { key: "versions", label: t("actions.versions"), onClick: (d) => navigate(p(detailPath(d.release, d.namespace, "versions"))) }, { key: "uninstall", label: t("actions.uninstall"), danger: true, onClick: (d) => setUninstalling({ namespace: d.namespace, release: d.release }), - disabled: () => (readOnly ? t("disabled.readOnly") : false), + disabled: (d) => writeBlocked(d), }, ]; diff --git a/web/src/modules/inferences/components/LoadHint.tsx b/web/src/modules/inferences/components/LoadHint.tsx index 23d7d36..5004fc3 100644 --- a/web/src/modules/inferences/components/LoadHint.tsx +++ b/web/src/modules/inferences/components/LoadHint.tsx @@ -1,12 +1,13 @@ import { Button } from "@modelsphere/ui"; import { TriangleAlert } from "lucide-react"; import type { ReleaseStatus } from "@swiss/lib/api"; +import { isUninstalling } from "@swiss/lib/phase"; import { useT } from "@/modules/inferences/i18n"; import { stalled } from "@/modules/inferences/lib"; export function LoadHint({ status: s, onViewInstances }: { status: ReleaseStatus; onViewInstances?: () => void }) { const t = useT(); - if (s.total === 0 || s.ready >= s.total) return null; + if (isUninstalling(s) || s.total === 0 || s.ready >= s.total) return null; if (!stalled(s)) return

{t("overview.coldLoad")}

; return (
diff --git a/web/src/modules/inferences/components/StatusDot.tsx b/web/src/modules/inferences/components/StatusDot.tsx index 6da357b..471c195 100644 --- a/web/src/modules/inferences/components/StatusDot.tsx +++ b/web/src/modules/inferences/components/StatusDot.tsx @@ -14,11 +14,13 @@ const VARIANT: Record = { const label = (state: State, t: TFn, suffix?: string) => [t(`state.${state.key}`, { raw: state.raw ?? "" }), suffix].filter(Boolean).join(" "); +const inFlight = (key: string) => key === "applying" || key === "uninstalling"; + export function statusOf(state: State, t: TFn, suffix?: string) { - return { tone: VARIANT[state.tone], label: label(state, t, suffix), animated: state.key === "applying" }; + return { tone: VARIANT[state.tone], label: label(state, t, suffix), animated: inFlight(state.key) }; } export function StatusDot({ state, suffix }: { state: State; suffix?: string }) { const t = useT(); - return ; + return ; } diff --git a/web/src/modules/inferences/dashboard-lib.test.ts b/web/src/modules/inferences/dashboard-lib.test.ts index aabf526..0d866e3 100644 --- a/web/src/modules/inferences/dashboard-lib.test.ts +++ b/web/src/modules/inferences/dashboard-lib.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import type { CatalogResponse, Run } from "@swiss/lib/api"; -import { HISTORY_LIMIT, recordedTimeline, supportedTotals, timelineTotals } from "./dashboard-lib"; +import type { CatalogResponse, Deployment, ReleaseStatus, Run } from "@swiss/lib/api"; +import { currentState, healthBucket, HISTORY_LIMIT, recordedTimeline, supportedTotals, timelineTotals } from "./dashboard-lib"; const from = "2026-10-10T12:15:00.000Z"; const to = "2026-10-11T12:15:00.000Z"; @@ -53,6 +53,26 @@ describe("supported catalog totals", () => { }); }); +describe("service health while uninstalling", () => { + const row = (over: Partial = {}): Deployment => ({ release: "q", namespace: "models", revision: 1, ...over }); + const live = (over: Partial = {}): ReleaseStatus => ({ + release: "q", + namespace: "models", + exists: true, + revision: 1, + pods: [], + ready: 0, + total: 0, + ...over, + }); + + it("counts an uninstalling release as degraded, from the list row or the live status", () => { + expect(healthBucket(currentState(row({ phase: "uninstalling", status: "failed" })))).toBe("degraded"); + expect(healthBucket(currentState(row({ status: "deployed", phase: "applied" }), { data: live({ exists: false, helmStatus: "failed", planStatus: { phase: "uninstalling" } }) }))).toBe("degraded"); + expect(healthBucket(currentState(row({ status: "deployed" }), { data: live({ ready: 1, total: 1, llmservice: { phase: "uninstalling" } }) }))).toBe("degraded"); + }); +}); + describe("recorded deployment timeline", () => { it("zero-fills a complete 24-hour window including partial edge hours", () => { const data = recordedTimeline([], from, to, true); diff --git a/web/src/modules/inferences/lib.test.ts b/web/src/modules/inferences/lib.test.ts index 287f73a..e8dcb89 100644 --- a/web/src/modules/inferences/lib.test.ts +++ b/web/src/modules/inferences/lib.test.ts @@ -82,6 +82,36 @@ describe("releaseState", () => { }); }); +describe("uninstalling wins over the states a disappearing release would otherwise show", () => { + const rows: { name: string; d: Parameters[0]; key: string }[] = [ + { name: "a row phase of uninstalling", d: { phase: "uninstalling" }, key: "uninstalling" }, + { name: "a row phase of uninstalling ahead of helm failed", d: { phase: "uninstalling", status: "failed" }, key: "uninstalling" }, + { name: "a row phase of uninstalling ahead of helm pending", d: { phase: "uninstalling", status: "pending-upgrade" }, key: "uninstalling" }, + { name: "helm's own uninstalling status", d: { status: "uninstalling" }, key: "uninstalling" }, + { name: "helm's uninstalling status ahead of a failed plan", d: { status: "uninstalling", phase: "failed" }, key: "uninstalling" }, + { name: "helm's uninstalling status ahead of an applying plan", d: { status: "uninstalling", phase: "applying" }, key: "uninstalling" }, + ]; + for (const tc of rows) { + it(`reads ${tc.name} as uninstalling`, () => { + expect(rowState(tc.d)).toEqual({ key: tc.key, tone: "warning" }); + }); + } + + const releases: { name: string; s: Partial; key: string }[] = [ + { name: "planStatus.phase uninstalling with the helm release already gone", s: { exists: false, ready: 0, total: 0, planStatus: { phase: "uninstalling" } }, key: "uninstalling" }, + { name: "planStatus.phase uninstalling ahead of a failed helm status", s: { exists: false, ready: 0, total: 0, helmStatus: "failed", planStatus: { phase: "uninstalling" } }, key: "uninstalling" }, + { name: "planStatus.phase uninstalling ahead of an applying helm status", s: { helmStatus: "pending-upgrade", planStatus: { phase: "uninstalling" } }, key: "uninstalling" }, + { name: "llmservice.phase uninstalling while pods are still ready", s: { ready: 1, total: 1, llmservice: { phase: "uninstalling" } }, key: "uninstalling" }, + { name: "llmservice.phase uninstalling ahead of a failed plan and no pods", s: { exists: false, ready: 0, total: 0, planStatus: { phase: "failed" }, llmservice: { phase: "uninstalling" } }, key: "uninstalling" }, + { name: "helm status uninstalling ahead of no pods", s: { exists: true, ready: 0, total: 0, helmStatus: "uninstalling" }, key: "uninstalling" }, + ]; + for (const tc of releases) { + it(`reads ${tc.name} as uninstalling`, () => { + expect(releaseState(status(tc.s))).toEqual({ key: tc.key, tone: "warning" }); + }); + } +}); + describe("tabs", () => { it("drops the plan when the release has none", () => { expect(visibleTabs({ plan: false })).toEqual(["overview", "instances", "logs", "events", "resources", "check", "versions", "runs"]); @@ -143,6 +173,10 @@ describe("installSteps", () => { const steps = installSteps(status({ exists: false, total: 0, ready: 0 }), {}, {}); expect(steps.map((s) => s.state)).toEqual(["off", "off", "off", "off"]); }); + it("reports uninstalling on the applied step while pods are still leaving", () => { + const steps = installSteps(status({ exists: false, total: 0, ready: 0, helmStatus: "failed", planStatus: { phase: "uninstalling" } }), {}, {}); + expect(steps[0]).toEqual({ key: "applied", state: "wait", detail: "uninstalling" }); + }); it("reports a routed release with its backends and route", () => { const steps = installSteps(status(), route({ ready: true }) as never, {}); expect(steps[2]).toEqual({ key: "route", state: "ok", detail: "ready", params: { n: 2, route: "/qwen/" } }); diff --git a/web/src/modules/inferences/lib.ts b/web/src/modules/inferences/lib.ts index fbb8074..a123e95 100644 --- a/web/src/modules/inferences/lib.ts +++ b/web/src/modules/inferences/lib.ts @@ -11,6 +11,7 @@ import type { Revision, Run, } from "@swiss/lib/api"; +import { isUninstalling } from "@swiss/lib/phase"; export type Tone = "success" | "info" | "warning" | "error" | "muted"; @@ -22,15 +23,18 @@ export interface State { // A list row has no pod counts: those cost a status read per row. export function rowState(d: Pick): State { + // Ahead of failed and applying: a helm row can already have no release status + // while its pods are still terminating. + if (isUninstalling(d)) return { key: "uninstalling", tone: "warning" }; if (d.phase === "failed" || d.status === "failed") return { key: "failed", tone: "error" }; if (d.phase === "applying" || d.status?.startsWith("pending")) return { key: "applying", tone: "info" }; - if (d.status === "uninstalling") return { key: "uninstalling", tone: "warning" }; if (d.status === "deployed") return { key: "applied", tone: "success" }; if (!d.status) return { key: "unknown", tone: "muted" }; return { key: "other", tone: "muted", raw: d.status }; } export function releaseState(s: ReleaseStatus): State { + if (isUninstalling(s)) return { key: "uninstalling", tone: "warning" }; const p = s.planStatus; if (!s.exists && !p) return { key: "notInstalled", tone: "muted" }; if (p?.phase === "failed" || s.helmStatus === "failed") return { key: "failed", tone: "error" }; @@ -133,6 +137,7 @@ export function installSteps( function appliedStep(s: ReleaseStatus, p?: PlanStatus): Step { const key = "applied"; + if (isUninstalling(s)) return { key, state: "wait", detail: "uninstalling" }; if (p?.phase === "failed" || s.helmStatus === "failed") return { key, state: "bad", detail: "failed" }; if (p?.phase === "applying" || s.helmStatus?.startsWith("pending")) return { key, state: "wait", detail: "applying" }; if (s.exists) return { key, state: "ok", detail: "revision", params: { n: s.revision } }; diff --git a/web/src/modules/inferences/locales/en-US.json b/web/src/modules/inferences/locales/en-US.json index 2bdd77f..6cac5b3 100644 --- a/web/src/modules/inferences/locales/en-US.json +++ b/web/src/modules/inferences/locales/en-US.json @@ -104,7 +104,8 @@ "untracked": "A release with no plan beside it cannot be upgraded here", "current": "This is already the current revision", "sloOff": "SLO was not turned on at deploy; turn it on in an upgrade", - "sloPending": "SLO shows once the deploy has applied" + "sloPending": "SLO shows once the deploy has applied", + "uninstalling": "This release is uninstalling; wait until its pods are gone" }, "state": { "applied": "Applied", @@ -163,6 +164,7 @@ "noRuns": "No activity yet", "noStore": "This swissd has no database, so the activity log is empty", "applyingHint": "An apply was started and never recorded a result. It may still be running, or swissd may have been restarted mid-apply.", + "uninstallingHint": "Uninstall has started. This service stays listed until its pods are gone.", "coldLoad": "A cold load takes 20–40 minutes. A pod that is scheduled and not ready has most likely been reading weights, not failing.", "stalled": "A pod has not been ready for over an hour, well past a normal load. Check the instances.", "viewInstances": "View instances" @@ -172,6 +174,7 @@ "label": "Applied", "failed": "The last apply failed", "applying": "Applying", + "uninstalling": "Uninstalling", "revision": "Revision {n}", "notInstalled": "Not installed" }, diff --git a/web/src/modules/inferences/locales/zh-CN.json b/web/src/modules/inferences/locales/zh-CN.json index e210dc7..70d5854 100644 --- a/web/src/modules/inferences/locales/zh-CN.json +++ b/web/src/modules/inferences/locales/zh-CN.json @@ -93,7 +93,8 @@ "untracked": "没有纳管计划的 release 不能在这里升级", "current": "这已经是当前版本", "sloOff": "部署时没有开启 SLO，可在升级时打开", - "sloPending": "部署完成后才能查看 SLO" + "sloPending": "部署完成后才能查看 SLO", + "uninstalling": "这个服务正在卸载，等 Pod 消失后再操作" }, "state": { "applied": "已应用", @@ -152,6 +153,7 @@ "noRuns": "还没有操作记录", "noStore": "这个 swissd 没有数据库，操作记录为空", "applyingHint": "部署已开始但还没有记录结果：可能仍在执行，也可能 swissd 在部署中途重启过。", + "uninstallingHint": "卸载已开始。Pod 消失之前，这个服务仍会留在列表里。", "coldLoad": "冷启动加载需要 20–40 分钟。已调度但未就绪的 Pod 多半是在读取权重，而不是出错。", "stalled": "实例超过 1 小时仍未就绪，已超出正常加载时间，请查看实例。", "viewInstances": "查看实例" @@ -161,6 +163,7 @@ "label": "已应用", "failed": "上次部署失败", "applying": "部署中", + "uninstalling": "卸载中", "revision": "版本 {n}", "notInstalled": "未安装" }, diff --git a/web/src/modules/inferences/tabs/Overview.tsx b/web/src/modules/inferences/tabs/Overview.tsx index c13cbd2..fb1c1cb 100644 --- a/web/src/modules/inferences/tabs/Overview.tsx +++ b/web/src/modules/inferences/tabs/Overview.tsx @@ -14,6 +14,7 @@ import { type Run, } from "@swiss/lib/api"; import { pick } from "@swiss/components/ReleaseObjects"; +import { isUninstalling } from "@swiss/lib/phase"; import { AccessPoint } from "@swiss/components/Endpoint"; import { useT } from "@/modules/inferences/i18n"; import { installSteps, modelLine, releaseState, type Step, type StepState } from "@/modules/inferences/lib"; @@ -54,7 +55,7 @@ export function Overview({ queryFn: () => api.runs({ namespace, release, limit: 200 }), }); - if (!s.exists && !p) { + if (!s.exists && !p && !isUninstalling(s)) { return

{t("detail.notFound")}

; } @@ -80,7 +81,7 @@ export function Overview({ value: ( - {s.helmStatus && helm {s.helmStatus}} + {s.helmStatus && s.helmStatus !== "uninstalling" && helm {s.helmStatus}} ), }, @@ -105,6 +106,7 @@ export function Overview({ ))} {p?.error && {p.error}} + {isUninstalling(s) && {t("overview.uninstallingHint")}} {p?.phase === "applying" && {t("overview.applyingHint")}} {s.warning && {s.warning}} diff --git a/web/src/modules/inferences/tabs/Versions.tsx b/web/src/modules/inferences/tabs/Versions.tsx index 92160b0..a3be057 100644 --- a/web/src/modules/inferences/tabs/Versions.tsx +++ b/web/src/modules/inferences/tabs/Versions.tsx @@ -7,7 +7,7 @@ import { useT } from "@/modules/inferences/i18n"; import { modelLine, revisionRows, type RevisionRow } from "@/modules/inferences/lib"; import { RollbackSheet } from "@/modules/inferences/components/RollbackSheet"; -export function Versions({ namespace, release, canRollBack }: { namespace: string; release: string; canRollBack: boolean }) { +export function Versions({ namespace, release, rollbackBlocked }: { namespace: string; release: string; rollbackBlocked: string | false }) { const t = useT(); const [rollingBack, setRollingBack] = useState(null); const revisions = useQuery({ @@ -50,7 +50,7 @@ export function Versions({ namespace, release, canRollBack }: { namespace: strin key: "rollback", label: t("actions.rollback"), onClick: (r) => setRollingBack(r.revision), - disabled: (r) => (r.current ? t("disabled.current") : !canRollBack ? t("disabled.readOnly") : false), + disabled: (r) => (r.current ? t("disabled.current") : rollbackBlocked), }, ]; diff --git a/web/src/modules/inferences/uninstalling.test.tsx b/web/src/modules/inferences/uninstalling.test.tsx new file mode 100644 index 0000000..f591c30 --- /dev/null +++ b/web/src/modules/inferences/uninstalling.test.tsx @@ -0,0 +1,161 @@ +import { describe, expect, it } from "vitest"; +import { renderToStaticMarkup } from "react-dom/server"; +import { MemoryRouter, Route, Routes } from "react-router"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import type { Deployment, Plan, ReleaseStatus } from "@swiss/lib/api"; +import { ModuleProvider } from "@/shell/module"; +import { InferenceDetail } from "@/modules/inferences/InferenceDetail"; +import { InferenceList } from "@/modules/inferences/InferenceList"; + +const module = { id: "inferences", title: "模型服务", basePath: "/inferences", pages: [] }; +const cluster = { name: "c", profile: "p", catalog: "u", version: "0", allowDeploy: true }; +const plan = { + apiVersion: "v1", + release: { name: "qwen", namespace: "models" }, + source: { model: "qwen", version: "1", variant: "a" }, + chart: { name: "vllm", version: "1" }, + engine: "vllm", + profile: "p", + layers: { form: { sloRequirement: { enabled: true } } }, + hash: "h", +} as Plan; + +const deployment = (over: Partial): Deployment => ({ release: "qwen", namespace: "models", revision: 3, model: "qwen", ...over }); +const release = (over: Partial = {}): ReleaseStatus => ({ + release: "qwen", + namespace: "models", + exists: true, + revision: 3, + helmStatus: "deployed", + pods: [], + ready: 1, + total: 1, + planStatus: { phase: "applied" }, + ...over, +}); + +function client() { + return new QueryClient({ defaultOptions: { queries: { retry: false, retryOnMount: false } } }); +} + +function button(html: string, label: string) { + for (const m of html.matchAll(/]*)>([\s\S]*?)<\/button>/g)) { + const text = m[2].replace(/<[^>]+>/g, "").replace(/\s+/g, " ").trim(); + if (text !== label) continue; + const at = m.index ?? 0; + return { + disabled: /(?:^|\s)disabled(?:\s*=|\s|$)/.test(m[1]), + reason: /cursor-not-allowed/.test(html.slice(Math.max(0, at - 400), at)), + }; + } +} + +function renderList(rows: Deployment[]) { + const c = client(); + c.setQueryData(["cluster"], cluster); + c.setQueryData(["deployments", "inferences", 1, 25], { + cluster: "c", + catalogRef: "r", + deployments: rows, + page: 1, + perPage: 25, + summary: { total: rows.length, catalogBehind: 0 }, + }); + return renderToStaticMarkup( + + + + + + + , + ); +} + +function renderDetail(status: ReleaseStatus) { + const c = client(); + c.setQueryData(["cluster"], cluster); + c.setQueryData(["status", "models", "qwen"], status); + c.setQueryData(["releasePlan", "models", "qwen"], plan); + c.setQueryData(["objects", "models", "qwen"], { objects: [] }); + c.setQueryData(["revisions", "models", "qwen"], { revisions: [{ revision: 2 }, { revision: 3, current: true }] }); + c.setQueryData(["runs", "models", "qwen"], { hasStore: true, runs: [] }); + c.setQueryData(["k8s-events", "models", "qwen"], { items: [] }); + return renderToStaticMarkup( + + + + + } /> + + + + , + ); +} + +describe("inference list while a deployment is uninstalling", () => { + const html = renderList([ + deployment({ release: "going", phase: "uninstalling", status: undefined, revision: undefined, chart: undefined }), + deployment({ release: "steady", phase: "applied", status: "deployed" }), + ]); + + it("shows uninstalling for a row whose helm release is already gone, and applied for a live one", () => { + const going = html.split(/ row.includes("going")); + const steady = html.split(/ row.includes("steady")); + expect(going).toContain("卸载中"); + expect(steady).toContain("已应用"); + expect(steady).not.toContain("卸载中"); + }); +}); + +describe("inference detail write actions", () => { + const cases: { name: string; status: ReleaseStatus; label: string; enabled: boolean }[] = [ + { + name: "an uninstalling helm release", + status: release({ exists: false, ready: 0, total: 1, helmStatus: undefined, pods: [{ name: "qwen-0", phase: "Running", ready: false, restarts: 0, ageSeconds: 30 }], planStatus: { phase: "uninstalling" } }), + label: "卸载中", + enabled: false, + }, + { + name: "an LLMService with a deletion timestamp", + status: release({ ready: 1, total: 1, backend: "llmsvc", llmservice: { phase: "uninstalling" } }), + label: "卸载中", + enabled: false, + }, + { + name: "a serving release", + status: release(), + label: "运行中", + enabled: true, + }, + ]; + + for (const tc of cases) { + it(`${tc.name} shows ${tc.label} and ${tc.enabled ? "keeps" : "disables"} upgrade, uninstall, SLO edit and rollback`, () => { + const html = renderDetail(tc.status); + expect(html).toContain(tc.label); + // The header keeps the health check. Write actions carry their reason on a + // tooltip trigger; that popup is not in the static markup. + const check = button(html, "服务检查"); + expect(check?.disabled).toBe(false); + expect(check?.reason).toBe(false); + for (const label of ["升级", "卸载", "编辑 SLO"]) { + const found = button(html, label); + expect(found, label).toBeDefined(); + expect(found?.disabled, label).toBe(!tc.enabled); + expect(found?.reason, label).toBe(!tc.enabled); + } + // The current revision cannot roll back to itself. The older one can, until uninstall starts. + const backs = html + .split(/ row.includes("回滚到此版")) + .map((row) => ({ current: row.includes("当前"), ...button(row, "回滚到此版") })); + expect(backs.map((b) => b.current).sort()).toEqual([false, true]); + expect(backs.find((b) => b.current)?.disabled).toBe(true); + const older = backs.find((b) => !b.current); + expect(older?.disabled).toBe(!tc.enabled); + expect(older?.reason).toBe(!tc.enabled); + }); + } +}); diff --git a/web/src/modules/swiss/lib/api.ts b/web/src/modules/swiss/lib/api.ts index c6413b0..b5bcf73 100644 --- a/web/src/modules/swiss/lib/api.ts +++ b/web/src/modules/swiss/lib/api.ts @@ -70,7 +70,8 @@ export interface Deployment { namespace: string; chart?: string; status?: string; - revision: number; + // Missing while a helm release is uninstalling: the release record is already gone. + revision?: number; updated?: string; model?: string; variant?: string; @@ -79,6 +80,8 @@ export interface Deployment { // longer lists it. catalog?: string; version?: string; + // "uninstalling" after helm has dropped the release but pods remain, or while + // an LLMService has a deletionTimestamp. The row leaves the list once they are gone. phase?: string; drift?: string; // The path the entrypoint publishes this release on. Absent when the plan @@ -650,6 +653,9 @@ export interface ReleaseStatus { planStatus?: PlanStatus; // Set when swiss manages the release; it never changes after install. backend?: ApplyWith; + // Present on the llmsvc backend. phase is "uninstalling" while the LLMService + // has a deletionTimestamp. Absent on helm and on an older swissd. + llmservice?: { phase?: string }; } export interface ObjectRef { diff --git a/web/src/modules/swiss/lib/phase.ts b/web/src/modules/swiss/lib/phase.ts new file mode 100644 index 0000000..bf36679 --- /dev/null +++ b/web/src/modules/swiss/lib/phase.ts @@ -0,0 +1,18 @@ +// Helm lists a release as uninstalling after the release record is gone, until +// its pods are. An LLMService uses the same phase while it has a deletionTimestamp. +// Helm's own status string is the same word. +export function isUninstalling(s: { + phase?: string; + status?: string; + helmStatus?: string; + planStatus?: { phase?: string }; + llmservice?: { phase?: string }; +}): boolean { + return ( + s.phase === "uninstalling" || + s.status === "uninstalling" || + s.helmStatus === "uninstalling" || + s.planStatus?.phase === "uninstalling" || + s.llmservice?.phase === "uninstalling" + ); +} diff --git a/web/src/modules/swiss/routes/DeploymentDetail.tsx b/web/src/modules/swiss/routes/DeploymentDetail.tsx index 2af79f6..23c6f9e 100644 --- a/web/src/modules/swiss/routes/DeploymentDetail.tsx +++ b/web/src/modules/swiss/routes/DeploymentDetail.tsx @@ -38,6 +38,7 @@ import { ReleaseStatus } from "@swiss/components/ReleaseStatus"; import { SLODialog } from "@swiss/components/SLOCard"; import { ErrorState, Loading } from "@swiss/components/States"; import { METHODS } from "@swiss/components/InstallMethod"; +import { isUninstalling } from "@swiss/lib/phase"; import { cn, timeAgo } from "@swiss/lib/utils"; export function DeploymentDetail() { @@ -80,7 +81,8 @@ export function DeploymentDetail() { if (status.error) return ; const s = status.data; - const showSLO = s.planStatus?.phase === "applied" && sloEnabled(plan.data); + const removing = isUninstalling(s); + const showSLO = !removing && s.planStatus?.phase === "applied" && sloEnabled(plan.data); const openSLO = () => setSloOpen(true); return ( @@ -124,16 +126,21 @@ export function DeploymentDetail() { History & rollback - {plan.data && ( - - {/* The one action that changes the release, so the one filled button. */} - - - )} + {plan.data && + (removing ? ( + + + + ) : ( + + {/* The one action that changes the release, so the one filled button. */} + + + ))}
@@ -186,7 +193,7 @@ export function DeploymentDetail() { )} {cluster.data?.allowDeploy && ( - + )} ); @@ -200,7 +207,7 @@ export function DeploymentDetail() { function InstallStatus({ status, objects }: { status: Status; objects?: ObjectResult[] }) { const p: PlanStatus | undefined = status.planStatus; - if (!status.exists && !p) { + if (!status.exists && !p && !isUninstalling(status)) { return ( @@ -219,7 +226,7 @@ function InstallStatus({ status, objects }: { status: Status; objects?: ObjectRe const updated = p?.updatedAt ?? p?.startedAt; const summary = [ p?.action, - status.exists ? `revision ${status.revision}` : "not installed", + isUninstalling(status) ? "uninstalling" : status.exists ? `revision ${status.revision}` : "not installed", updated && `updated ${timeAgo(updated)}`, ].filter(Boolean); @@ -231,8 +238,12 @@ function InstallStatus({ status, objects }: { status: Status; objects?: ObjectRe

{summary.join(" · ")}

- {p && } - {status.helmStatus && helm {status.helmStatus}} + {isUninstalling(status) ? ( + uninstalling + ) : ( + p && + )} + {status.helmStatus && status.helmStatus !== "uninstalling" && helm {status.helmStatus}} {status.backend && ( {METHODS[status.backend].label} @@ -248,6 +259,9 @@ function InstallStatus({ status, objects }: { status: Status; objects?: ObjectRe {p?.error && {p.error}} + {isUninstalling(status) && ( + Uninstall has started. This release stays listed until its pods are gone. + )} {p?.phase === "applying" && ( An apply was started and never recorded a result. It may still be running, or @@ -298,6 +312,7 @@ interface Step { function appliedStep(s: Status, p?: PlanStatus): Step { const label = "Applied"; + if (isUninstalling(s)) return { label, state: "wait", detail: "uninstalling" }; if (p?.phase === "failed" || s.helmStatus === "failed") return { label, state: "bad", detail: "the last apply failed" }; if (p?.phase === "applying" || s.helmStatus?.startsWith("pending")) return { label, state: "wait", detail: s.helmStatus ?? "applying" }; @@ -401,10 +416,12 @@ function Uninstall({ namespace, release, exists, + uninstalling, }: { namespace: string; release: string; exists: boolean; + uninstalling: boolean; }) { const [open, setOpen] = useState(false); const [typed, setTyped] = useState(""); @@ -432,15 +449,19 @@ function Uninstall({ Uninstall

- {exists - ? "Removes this release from the cluster. Reloading the weights afterwards takes 20–40 minutes." - : "There is no live release to remove."} + {uninstalling + ? "This release is uninstalling; wait until its pods are gone." + : exists + ? "Removes this release from the cluster. Reloading the weights afterwards takes 20–40 minutes." + : "There is no live release to remove."}

- + + + uninstalling; if (phase === "applied") return applied; if (phase === "failed") return failed; if (phase === "applying") return applying; diff --git a/web/src/modules/swiss/routes/Deployments.tsx b/web/src/modules/swiss/routes/Deployments.tsx index 3cdfe0e..dd36a35 100644 --- a/web/src/modules/swiss/routes/Deployments.tsx +++ b/web/src/modules/swiss/routes/Deployments.tsx @@ -3,6 +3,7 @@ import { Link } from "@swiss/lib/host"; import { ChevronLeft, ChevronRight, Eye, Plus, Settings2 } from "lucide-react"; import { useQuery } from "@tanstack/react-query"; import { api, type Deployment } from "@swiss/lib/api"; +import { isUninstalling } from "@swiss/lib/phase"; import { Badge } from "@swiss/components/ui/badge"; import { Button } from "@swiss/components/ui/button"; import { Card, CardContent } from "@swiss/components/ui/card"; @@ -141,15 +142,15 @@ function Row({ d, onConfigureSLO }: { d: Deployment; onConfigureSLO: () => void {d.route || "—"} {d.chart} - {d.revision} + {d.revision ?? "—"} - +
{/* Applied is when the chart has created the requirement; whether this release has one at all, the popup asks slo-api. */} - {d.phase === "applied" && ( + {d.phase === "applied" && !isUninstalling(d) && ( @@ -167,7 +168,9 @@ function Row({ d, onConfigureSLO }: { d: Deployment; onConfigureSLO: () => void // "pending-upgrade" on this workload usually means a 20-40 minute model load in // progress, not a failure -- so it reads as in-flight rather than as an error. -function StatusBadge({ status }: { status?: string }) { +function StatusBadge({ d }: { d: Pick }) { + if (isUninstalling(d)) return uninstalling; + const status = d.status; if (!status) return null; if (status === "deployed") return deployed; if (status.startsWith("pending")) return {status}; diff --git a/web/src/modules/swiss/routes/Runs.tsx b/web/src/modules/swiss/routes/Runs.tsx index 5996e82..71305f9 100644 --- a/web/src/modules/swiss/routes/Runs.tsx +++ b/web/src/modules/swiss/routes/Runs.tsx @@ -3,6 +3,7 @@ import { Link, useSearchParams } from "@swiss/lib/host"; import { useQuery } from "@tanstack/react-query"; import { ChevronDown, ChevronRight, Undo2 } from "lucide-react"; import { api, type Revision, type Run } from "@swiss/lib/api"; +import { isUninstalling } from "@swiss/lib/phase"; import { Badge } from "@swiss/components/ui/badge"; import { Button } from "@swiss/components/ui/button"; import { Card } from "@swiss/components/ui/card"; @@ -47,6 +48,12 @@ export function Runs() { queryFn: () => api.revisions(namespace, release), enabled: oneRelease, }); + const status = useQuery({ + queryKey: ["status", namespace, release], + queryFn: () => api.status(namespace, release), + enabled: oneRelease, + }); + const removing = !!status.data && isUninstalling(status.data); const setFilter = (k: string, v: string) => { const next = new URLSearchParams(params); @@ -166,6 +173,7 @@ export function Runs() { revisions={revisions} pending={revs.isPending} error={revs.error} + uninstalling={removing} /> )}
@@ -181,12 +189,14 @@ function RevisionsTab({ revisions, pending, error, + uninstalling, }: { namespace: string; release: string; revisions: Revision[]; pending: boolean; error: unknown; + uninstalling: boolean; }) { if (error) return ; if (pending) return ; @@ -222,6 +232,7 @@ function RevisionsTab({ revision={r} namespace={namespace} release={release} + uninstalling={uninstalling} /> ))} @@ -328,10 +339,12 @@ function RevisionRow({ revision, namespace, release, + uninstalling, }: { revision: Revision; namespace: string; release: string; + uninstalling: boolean; }) { const [open, setOpen] = useState(false); @@ -359,7 +372,7 @@ function RevisionRow({ {revision.current ? ( live ) : ( - + )} @@ -387,19 +400,31 @@ function RollBack({ namespace, release, to, + disabled, }: { namespace: string; release: string; to: number; + disabled?: boolean; }) { + const button = ( + + ); + if (disabled) { + return ( + e.stopPropagation()}> + {button} + + ); + } return ( e.stopPropagation()} > - + {button} ); } diff --git a/web/src/modules/swiss/routes/Upgrade.tsx b/web/src/modules/swiss/routes/Upgrade.tsx index 299e535..b7ddb5b 100644 --- a/web/src/modules/swiss/routes/Upgrade.tsx +++ b/web/src/modules/swiss/routes/Upgrade.tsx @@ -35,6 +35,7 @@ import { Dialog } from "@swiss/components/ui/dialog"; import { gpuCount, unlistedProducts, vendorProducts } from "@swiss/lib/gpu"; import { Button } from "@swiss/components/ui/button"; import { comparison, formatUplift, reportLink, sharedVersion, variantKind, workloadSummary } from "@swiss/lib/catalog"; +import { isUninstalling } from "@swiss/lib/phase"; export function Upgrade() { const { namespace = "", release = "" } = useParams(); @@ -67,6 +68,11 @@ export function Upgrade() { staleTime: Infinity, }); const cluster = useQuery({ queryKey: ["cluster"], queryFn: api.cluster }); + const live = useQuery({ + queryKey: ["status", namespace, release], + queryFn: () => api.status(namespace, release), + refetchInterval: 15_000, + }); const nodes = useQuery({ queryKey: ["nodes"], queryFn: api.nodes }); const catalogs = cluster.data?.catalogs ?? []; const recorded = current.data?.source; @@ -177,6 +183,19 @@ export function Upgrade() { return ; } } + if (live.data && isUninstalling(live.data)) { + return ( +
+ + + + +
This release is uninstalling; wait until its pods are gone.
+ + +
+ ); + } const cur = current.data; const targetVersion = version || keptPin?.version; diff --git a/web/src/modules/swiss/routes/uninstalling.test.tsx b/web/src/modules/swiss/routes/uninstalling.test.tsx new file mode 100644 index 0000000..a31d476 --- /dev/null +++ b/web/src/modules/swiss/routes/uninstalling.test.tsx @@ -0,0 +1,140 @@ +import { type ReactNode } from "react"; +import { describe, expect, it } from "vitest"; +import { renderToStaticMarkup } from "react-dom/server"; +import { MemoryRouter, Route, Routes } from "react-router"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import type { Deployment, Plan, ReleaseStatus } from "@swiss/lib/api"; +import { ModuleProvider } from "@/shell/module"; +import { DeploymentDetail } from "@swiss/routes/DeploymentDetail"; +import { Deployments } from "@swiss/routes/Deployments"; +import { Upgrade } from "@swiss/routes/Upgrade"; + +const module = { id: "swiss", title: "s", basePath: "/swiss", pages: [] }; +const plan = { + apiVersion: "v1", + release: { name: "qwen", namespace: "models" }, + source: { catalogName: "public", model: "qwen", version: "1", variant: "a" }, + chart: { name: "vllm", version: "1" }, + engine: "vllm", + profile: "p", + layers: { form: { sloRequirement: { enabled: true } } }, + hash: "h", +} as Plan; + +const deployment = (over: Partial): Deployment => ({ release: "qwen", namespace: "models", revision: 3, model: "qwen", ...over }); +const release = (over: Partial = {}): ReleaseStatus => ({ + release: "qwen", + namespace: "models", + exists: true, + revision: 3, + helmStatus: "deployed", + pods: [], + ready: 1, + total: 1, + planStatus: { phase: "applied" }, + ...over, +}); + +function client() { + return new QueryClient({ defaultOptions: { queries: { retry: false, retryOnMount: false } } }); +} + +function buttons(html: string) { + return [...html.matchAll(/]*)>([\s\S]*?)<\/button>/g)].map((m) => ({ + label: m[2].replace(/<[^>]+>/g, "").replace(/\s+/g, " ").trim(), + disabled: /(?:^|\s)disabled(?:\s*=|\s|$)/.test(m[1]), + })); +} + +function button(html: string, label: string) { + return buttons(html).find((b) => b.label === label); +} + +function page(c: QueryClient, path: string, element: ReactNode) { + return renderToStaticMarkup( + + + + + + + + + + + , + ); +} + +describe("swiss deployment list", () => { + it("shows uninstalling when the helm release is already gone, and leaves a live release deployed", () => { + const c = client(); + c.setQueryData(["cluster"], { allowDeploy: true, catalog: "u", name: "c", profile: "p", version: "0" }); + c.setQueryData(["deployments", 1], { + cluster: "c", + catalogRef: "r", + page: 1, + perPage: 25, + summary: { total: 2, catalogBehind: 0 }, + deployments: [ + deployment({ release: "going", phase: "uninstalling", status: undefined, revision: undefined, chart: undefined }), + deployment({ release: "steady", phase: "applied", status: "deployed" }), + ], + }); + const html = page(c, "/deployments", ); + const going = html.split(/ row.includes("going")); + const steady = html.split(/ row.includes("steady")); + expect(going).toContain("uninstalling"); + expect(going).not.toContain("SLO"); + expect(steady).toContain("deployed"); + expect(steady).toContain("SLO"); + }); +}); + +describe("swiss deployment detail", () => { + const cases: { name: string; status: ReleaseStatus; enabled: boolean }[] = [ + { + name: "disables upgrade and uninstall while helm is uninstalling", + status: release({ exists: false, ready: 0, total: 1, helmStatus: undefined, planStatus: { phase: "uninstalling" } }), + enabled: false, + }, + { + name: "disables upgrade and uninstall while an LLMService is being deleted", + status: release({ backend: "llmsvc", llmservice: { phase: "uninstalling" } }), + enabled: false, + }, + { + name: "keeps upgrade and uninstall for a serving release", + status: release(), + enabled: true, + }, + ]; + + for (const tc of cases) { + it(tc.name, () => { + const c = client(); + c.setQueryData(["cluster"], { allowDeploy: true, catalog: "u", name: "c", profile: "p", version: "0" }); + c.setQueryData(["status", "models", "qwen"], tc.status); + c.setQueryData(["releasePlan", "models", "qwen"], plan); + c.setQueryData(["objects", "models", "qwen"], { objects: [] }); + const html = page(c, "/deployments/models/qwen", ); + expect(html).toContain(tc.enabled ? "applied" : "uninstalling"); + expect(button(html, "Upgrade")?.disabled).toBe(!tc.enabled); + expect(button(html, "Uninstall…")?.disabled).toBe(!tc.enabled); + if (!tc.enabled) expect(html).toContain("This release is uninstalling; wait until its pods are gone"); + expect(html.includes("Configure SLO")).toBe(tc.enabled); + }); + } +}); + +describe("swiss upgrade page", () => { + it("refuses upgrade and rollback while the release is uninstalling", () => { + const c = client(); + c.setQueryData(["release-plan", "models", "qwen"], plan); + c.setQueryData(["status", "models", "qwen"], release({ exists: false, planStatus: { phase: "uninstalling" }, ready: 0, total: 1 })); + const html = page(c, "/upgrade/models/qwen", ); + expect(html).toContain("This release is uninstalling; wait until its pods are gone"); + expect(html).not.toContain("Compose upgrade"); + expect(html).not.toContain("Review rollback"); + }); +});