From 9addfef205225098f7aed0b46ddbebaec76ca21c Mon Sep 17 00:00:00 2001 From: imneov Date: Wed, 21 Oct 2026 05:44:35 -0700 Subject: [PATCH 1/2] feat(web): configure generation recovery when deploying models Expose existing continuationGateway chart settings in Model Serving deploy and upgrade forms, preserve recorded values, and validate the final plan before review. Fixes #63 --- web/src/modules/docs/content/en-US/deploy.md | 12 +++ web/src/modules/docs/content/zh-CN/deploy.md | 12 +++ .../inferences/components/DeployForm.tsx | 4 +- .../inferences/components/DeploySheet.tsx | 18 +++- .../inferences/components/RecoveryFields.tsx | 35 ++++++++ web/src/modules/inferences/locales/en-US.json | 22 +++++ web/src/modules/inferences/locales/zh-CN.json | 22 +++++ web/src/modules/inferences/recovery.test.tsx | 87 +++++++++++++++++++ web/src/modules/inferences/recovery.ts | 84 ++++++++++++++++++ 9 files changed, 292 insertions(+), 4 deletions(-) create mode 100644 web/src/modules/inferences/components/RecoveryFields.tsx create mode 100644 web/src/modules/inferences/recovery.test.tsx create mode 100644 web/src/modules/inferences/recovery.ts diff --git a/web/src/modules/docs/content/en-US/deploy.md b/web/src/modules/docs/content/en-US/deploy.md index d639202..bd071ad 100644 --- a/web/src/modules/docs/content/en-US/deploy.md +++ b/web/src/modules/docs/content/en-US/deploy.md @@ -34,6 +34,18 @@ Most fields can stay empty and take the site profile's and chart's defaults. **Adaptive concurrency** groups the master switch, floor fraction, absolute floor, start point, idle hold, and TTFT decisions together; its internal knobs are available under **Advanced adaptive options**. Turning the switch off hides and clears the group's entered values; turning it back on does not restore them. **Default peer maximum** stays in **Capacity & failover** and still applies when adaptive concurrency is off; it also influences the static maximum used for adaptive bounds. Review the diff before confirming an upgrade. +## Generation recovery + +Enable **Routing → Generation recovery** in the deploy or upgrade form. Enter the **Actual model** whose weights are deployed, such as `kimi-k3` or `glm-5.3`. This selects the continuation strategy; it is not the client-facing served name. + +- Requires **CART**, **Publish a route**, and `sglang` chart **0.8.11+** or `vllm` chart **0.8.9+**. Composing the plan checks these conditions. +- **Stream idle timeout** is the time without a chunk after streaming starts; empty uses the chart default. +- **Resume interrupted responses** is on by default; off detects and logs interruptions without sending a continuation request. +- **Buffer tool calls** holds arguments until a complete call can be sent together, so interrupted calls can be regenerated. +- Requests flow through `openresty → continuation gateway → CART → inference instance`. The gateway is deployed with the model release; review and confirm the diff before applying. +- Unsupported requests, including structured JSON output, pass through. Verify the actual model and engine combination against the [gateway compatibility documentation](https://github.com/modelsphere/continuation_gateway). +- Upgrades carry the configured recovery settings forward. Turn the switch off, compose and confirm to remove the gateway. Advanced YAML takes precedence over the form; inspect the final plan and diff. + ## 3. Review and confirm **Compose plan** opens the review: diff --git a/web/src/modules/docs/content/zh-CN/deploy.md b/web/src/modules/docs/content/zh-CN/deploy.md index 1d28646..eb0202d 100644 --- a/web/src/modules/docs/content/zh-CN/deploy.md +++ b/web/src/modules/docs/content/zh-CN/deploy.md @@ -33,6 +33,18 @@ **自适应并发** 的总开关、下限比例、绝对下限、起始并发、空闲保持和 TTFT 判断位于同一区域，内部参数可从 **高级自适应参数** 添加。关闭后隐藏本组输入并清空已填写参数，再开启时不会恢复这些旧值。**默认节点并发上限** 留在 **容量与故障切换** 中，关闭自适应并发时也有效；它也影响自适应上下限使用的静态最大值。升级时仍需先检查差异再确认。 +## 生成中断恢复 + +在部署或升级表单的 **路由 → 生成中断恢复** 中开启，填写权重对应的 **实际模型**，例如 `kimi-k3` 或 `glm-5.3`。它用于选择续写策略，不是客户端调用时使用的服务模型名。 + +- 需要启用 **CART**、**发布路由**，并选择 `sglang` chart **0.8.11+** 或 `vllm` chart **0.8.9+**；生成计划时会检查这些条件。 +- **流式输出空闲超时** 控制开始输出后多久没有新片段视为卡顿；留空使用 chart 默认值。 +- **执行中断续写** 默认开启；关闭时网关仅检测并记录中断，不发送续写请求。 +- **缓冲工具调用** 会将参数保留到调用完整生成后一次性发送，以便在中断时重新生成。 +- 请求路径为 `openresty → 恢复网关 → CART → 推理实例`，恢复网关随模型 release 部署；最终仍需预览变更并确认。 +- 结构化 JSON 等不支持的请求直接透传。实际模型与引擎的兼容范围见 [恢复网关说明](https://github.com/modelsphere/continuation_gateway)，启用前应验证实际组合。 +- 升级会带出已配置的恢复参数；关闭后再生成、确认计划可移除网关。高级 YAML 的优先级高于表单，检查最终计划和差异。 + ## 3. 预览并确认 点 **生成计划** 后进入预览： diff --git a/web/src/modules/inferences/components/DeployForm.tsx b/web/src/modules/inferences/components/DeployForm.tsx index 3698bea..c0e745f 100644 --- a/web/src/modules/inferences/components/DeployForm.tsx +++ b/web/src/modules/inferences/components/DeployForm.tsx @@ -58,9 +58,10 @@ export interface DeployFormProps { catalogRoute?: string; // The chart's values and swissd's layers for the Override YAML editor. helmValues?: HelmValuesSource; + recovery?: ReactNode; } -export function DeployForm({ form, onChange, upgrade, submitted, target, cluster, defaults, gpuOptions, gpuListed, gpuResources, capacity, expanded, onExpand, idPrefix, routeOptions, catalogRoute, helmValues }: DeployFormProps) { +export function DeployForm({ form, onChange, upgrade, submitted, target, cluster, defaults, gpuOptions, gpuListed, gpuResources, capacity, expanded, onExpand, idPrefix, routeOptions, catalogRoute, helmValues, recovery }: DeployFormProps) { const t = useT(); const on = effective(form); const f = (k: string) => t(`deploy.fields.${k}`); @@ -187,6 +188,7 @@ export function DeployForm({ form, onChange, upgrade, submitted, target, cluster {section( "routing", <> + {recovery} {sw("exposeRoutedPeer", f("exposeRoutedPeer"))} diff --git a/web/src/modules/inferences/components/DeploySheet.tsx b/web/src/modules/inferences/components/DeploySheet.tsx index 2e84e19..ce9da2e 100644 --- a/web/src/modules/inferences/components/DeploySheet.tsx +++ b/web/src/modules/inferences/components/DeploySheet.tsx @@ -43,6 +43,8 @@ import { DeployForm } from "@/modules/inferences/components/DeployForm"; import type { HelmValuesSource } from "@/modules/inferences/components/OverrideYaml"; import { ExperimentalBadge } from "@/modules/inferences/components/InstallMethodLabel"; import { AdaptivePlanChangedError, composeAdaptivePlan, withAdaptiveConcurrency } from "../adaptive-concurrency"; +import { RecoveryFields } from "./RecoveryFields"; +import { mergeValues, planValues, recoveryError, recoverySettings, withRecovery } from "../recovery"; import { readRouteOptions } from "@/modules/inferences/api"; export type DeployTarget = @@ -138,6 +140,11 @@ function DeploySheetBody({ target, onClose }: { target: DeployTarget; onClose: ( if (current.data && !seedPlan) setSeedPlan(current.data); const deployModel = upgrade ? "" : target.model; const initial = useMemo
(() => (upgrade ? (seedPlan ? formFromPlan(seedPlan) : EMPTY) : { ...EMPTY, serviceId: defaultServiceId(deployModel) }), [upgrade, seedPlan, deployModel]); + const initialRecovery = useMemo(() => recoverySettings( + upgrade && seedPlan ? planValues(seedPlan) : mergeValues(entry.data?.variant.values, profile.data?.profile?.extra), + ), [upgrade, seedPlan, entry.data?.variant.values, profile.data?.profile?.extra]); + const [recoveryPatch, setRecoveryPatch] = useState>>({}); + const recovery = { ...initialRecovery, ...recoveryPatch }; const [form, setForm] = useState(initial); const catalogRoute = upgrade && current.data ? initial.route || initial.serviceId || target.release : undefined; const routeOptions = useQuery({ @@ -149,7 +156,7 @@ function DeploySheetBody({ target, onClose }: { target: DeployTarget; onClose: ( }); useEffect(() => setForm(initial), [initial]); const [seed, setSeed] = useState(null); - const dirty = useDirty({ form, version, variant, moveTo, chartVersion }, { form: seed ? reseed(initial, {}, seed) : initial, version: upgrade ? "" : (target.version ?? ""), variant: upgrade ? "" : (target.variant ?? ""), moveTo: "", chartVersion: "" }); + const dirty = useDirty({ form, version, variant, moveTo, chartVersion, recovery }, { form: seed ? reseed(initial, {}, seed) : initial, version: upgrade ? "" : (target.version ?? ""), variant: upgrade ? "" : (target.variant ?? ""), moveTo: "", chartVersion: "", recovery: initialRecovery }); const [step, setStep] = useState<"form" | "review">("form"); const [submitted, setSubmitted] = useState(false); @@ -200,7 +207,7 @@ function DeploySheetBody({ target, onClose }: { target: DeployTarget; onClose: ( // sheet is open, so saving the overrides and editing them again does not // read the chart again. const valuesForm = { ...form, edits: "" }; - const valuesRequest = upgrade && !recorded ? undefined : withAdaptiveConcurrency(requestFor(valuesForm), valuesForm, routeOptions.data); + const valuesRequest = upgrade && !recorded ? undefined : withAdaptiveConcurrency(withRecovery(requestFor(valuesForm), recovery), valuesForm, routeOptions.data); const helmValues = useQuery({ queryKey: ["helmValues", JSON.stringify(valuesRequest ?? null)], queryFn: () => deployApi.helmValues(valuesRequest!), @@ -219,7 +226,11 @@ function DeploySheetBody({ target, onClose }: { target: DeployTarget; onClose: ( const compose = useMutation({ mutationFn: async () => { try { - return await composeAdaptivePlan(requestFor(form), form, routeOptions.data, deployApi.plan, catalogName); + if (recovery.enabled && recovery.idleTimeout.trim() && (!Number.isFinite(Number(recovery.idleTimeout)) || Number(recovery.idleTimeout) <= 0)) throw new Error(t("deploy.recovery.errors.timeout")); + const proposed = await composeAdaptivePlan(withRecovery(requestFor(form), recovery), form, routeOptions.data, deployApi.plan, catalogName); + const invalid = recoveryError(proposed); + if (invalid) throw new Error(t(`deploy.recovery.errors.${invalid}`)); + return proposed; } catch (error) { if (error instanceof AdaptivePlanChangedError) throw new Error(t("deploy.fields.adaptiveChanged")); throw error; @@ -446,6 +457,7 @@ function DeploySheetBody({ target, onClose }: { target: DeployTarget; onClose: ( onChange={change} upgrade={upgrade} submitted={submitted} + recovery={ setRecoveryPatch((previous) => ({ ...previous, ...patch }))} />} target={targetFields} cluster={cluster.data} defaults={{ serviceId: upgrade ? undefined : defaultServiceId(target.model), localPath: entry.data?.localPath, localPathTemplate: entry.data?.pathTemplate, image: imageOf(v, entry.data?.imageRepository), cache }} diff --git a/web/src/modules/inferences/components/RecoveryFields.tsx b/web/src/modules/inferences/components/RecoveryFields.tsx new file mode 100644 index 0000000..29a6a4b --- /dev/null +++ b/web/src/modules/inferences/components/RecoveryFields.tsx @@ -0,0 +1,35 @@ +import { FieldInput, FloatingField, Switch } from "@modelsphere/ui"; +import { useT } from "../i18n"; +import type { RecoverySettings } from "../recovery"; + +export function RecoveryFields({ settings, onChange }: { settings: RecoverySettings; onChange: (patch: Partial) => void }) { + const t = useT(); + return ( + +

{t("deploy.recovery.requirements")}

+ + onChange({ model: event.target.value })} /> + + + onChange({ idleTimeout: event.target.value })} /> + + + onChange({ resumeEnabled: value })} /> + + + onChange({ bufferToolCalls: value })} /> + +

{t("deploy.recovery.compatibility")}

+ + } + > + onChange({ enabled: value })} /> + + ); +} diff --git a/web/src/modules/inferences/locales/en-US.json b/web/src/modules/inferences/locales/en-US.json index 4833895..5f824f1 100644 --- a/web/src/modules/inferences/locales/en-US.json +++ b/web/src/modules/inferences/locales/en-US.json @@ -419,6 +419,28 @@ "routing": "Routing", "advanced": "Advanced" }, + "recovery": { + "title": "Generation recovery", + "hint": "Deploy a continuation gateway to resume a stalled or interrupted streaming response once.", + "requirements": "Requires CART, a published route, and sglang chart 0.8.11+ or vllm chart 0.8.9+. Advanced YAML overrides these fields.", + "model": "Actual model", + "modelHint": "The model whose weights are deployed, for example kimi-k3 or glm-5.3. This selects the continuation strategy; it is not the client-facing served name.", + "timeout": "Stream idle timeout (seconds)", + "timeoutHint": "Time without a chunk after streaming starts. Empty keeps the chart default.", + "resume": "Resume interrupted responses", + "resumeHint": "Off detects and logs interruptions without sending a continuation request.", + "buffer": "Buffer tool calls", + "bufferHint": "Hold tool-call chunks until complete so interrupted calls can be regenerated. Applies only when resuming is on; arguments arrive together.", + "compatibility": "Model and engine compatibility must be verified. Structured JSON responses and other unsupported requests pass through without recovery.", + "errors": { + "chart": "Generation recovery needs sglang chart 0.8.11+ or vllm chart 0.8.9+. Select a supported chart version.", + "cart": "Generation recovery requires CART. Enable CART or turn recovery off.", + "route": "Generation recovery requires a published route. Publish a route or turn recovery off.", + "model": "Enter the actual model for generation recovery; the client-facing served name is not sufficient.", + "timeout": "The recovery idle timeout must be a finite number greater than zero.", + "engine": "The continuation engine must match the selected sglang or vllm chart. Check Advanced YAML." + } + }, "routeOptions": { "advanced": "Advanced route options", "advancedAdaptive": "Advanced adaptive options", diff --git a/web/src/modules/inferences/locales/zh-CN.json b/web/src/modules/inferences/locales/zh-CN.json index 32161f0..83dab6e 100644 --- a/web/src/modules/inferences/locales/zh-CN.json +++ b/web/src/modules/inferences/locales/zh-CN.json @@ -408,6 +408,28 @@ "routing": "路由", "advanced": "高级" }, + "recovery": { + "title": "生成中断恢复", + "hint": "部署恢复网关，在流式回答卡住或断开时尝试续写一次。", + "requirements": "需要启用 CART、发布路由，并使用 sglang chart 0.8.11+ 或 vllm chart 0.8.9+。高级 YAML 的优先级高于这些字段。", + "model": "实际模型", + "modelHint": "填写部署权重对应的模型，例如 kimi-k3 或 glm-5.3，用于选择续写策略；不是客户端调用时使用的服务模型名。", + "timeout": "流式输出空闲超时（秒）", + "timeoutHint": "开始输出后，多久没有收到新片段视为卡顿。留空使用 chart 默认值。", + "resume": "执行中断续写", + "resumeHint": "关闭时仅检测并记录中断，不发送续写请求。", + "buffer": "缓冲工具调用", + "bufferHint": "工具调用完整生成后再发送，便于在中断时重新生成。仅在执行续写时生效，调用参数会一次性到达。", + "compatibility": "需要验证实际模型与推理引擎的兼容性。结构化 JSON 等不支持的请求会直接透传，不执行恢复。", + "errors": { + "chart": "生成中断恢复需要 sglang chart 0.8.11+ 或 vllm chart 0.8.9+，请选择支持的 chart 版本。", + "cart": "生成中断恢复依赖 CART，请启用 CART 或关闭恢复。", + "route": "生成中断恢复需要发布路由，请发布路由或关闭恢复。", + "model": "请填写生成中断恢复所用的实际模型，不能仅填写客户端调用时使用的服务模型名。", + "timeout": "恢复空闲超时必须是大于零的有限数值。", + "engine": "续写引擎必须与所选 sglang 或 vllm chart 一致，请检查高级 YAML。" + } + }, "routeOptions": { "advanced": "高级路由参数", "advancedAdaptive": "高级自适应参数", diff --git a/web/src/modules/inferences/recovery.test.tsx b/web/src/modules/inferences/recovery.test.tsx new file mode 100644 index 0000000..f1dd80e --- /dev/null +++ b/web/src/modules/inferences/recovery.test.tsx @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; +import { renderToStaticMarkup } from "react-dom/server"; +import type { Plan } from "@swiss/lib/api"; +import { EMPTY, planRequest } from "@swiss/components/DeploySettings"; +import { RecoveryFields } from "./components/RecoveryFields"; +import { mergeValues, planValues, recoveryError, recoverySettings, withRecovery } from "./recovery"; + +const plan = (layers: Plan["layers"], chart = { name: "sglang", version: "0.8.11" }): Plan => ({ + apiVersion: "swiss.modelsphere.dev/v1alpha1", release: { name: "model", namespace: "models" }, + source: { model: "weights", variant: "default" }, chart, engine: chart.name, profile: "site", layers, hash: "hash", +}); +const enabled = { + cart: { enabled: true }, modelRoute: { enabled: true }, + continuationGateway: { enabled: true, config: { continuationModel: "glm-5.3", stallIdleTimeoutSeconds: 30 } }, +}; + +describe("generation recovery deployment", () => { + it("leaves old charts untouched until recovery is configured", () => { + const request = planRequest(EMPTY, { model: "weights" }); + expect(withRecovery(request, recoverySettings())).toBe(request); + expect(recoveryError(plan({ form: request.overrides! }, { name: "sglang", version: "0.8.0" }))).toBeUndefined(); + }); + + it("writes actual model identity, positive timeout and execution settings through the existing plan endpoint", () => { + const form = { ...EMPTY, modelRoute: true, cart: true }; + const settings = { ...recoverySettings(), enabled: true, model: " glm-5.3 ", idleTimeout: "45", bufferToolCalls: false }; + const request = withRecovery(planRequest(form, { model: "arbitrary-client-name" }), settings); + expect(request.overrides?.continuationGateway).toEqual({ enabled: true, config: { + continuationModel: "glm-5.3", continuationEnabled: true, stallIdleTimeoutSeconds: 45, bufferToolCalls: false, + } }); + expect(recoveryError(plan({ form: request.overrides! }))).toBeUndefined(); + }); + + it("seeds upgrades in merge order and preserves gateway values outside the form", () => { + const current = plan({ + catalog: enabled, + site: { continuationGateway: { replicas: 3, config: { stallIdleTimeoutSeconds: 60 } } }, + form: { continuationGateway: { config: { bufferToolCalls: false, continuationEnabled: false } } }, + edit: { continuationGateway: { config: { continuationModel: "kimi-k3" } } }, + }); + const settings = recoverySettings(planValues(current)); + expect(settings).toMatchObject({ enabled: true, configured: true, model: "kimi-k3", idleTimeout: "60", bufferToolCalls: false, resumeEnabled: false }); + const values = mergeValues(planValues(current), withRecovery({}, settings).overrides); + expect(values.continuationGateway).toMatchObject({ replicas: 3, config: { continuationEnabled: false } }); + }); + + it("explicitly disables a previously configured gateway without emitting active settings", () => { + const settings = { ...recoverySettings(enabled), enabled: false }; + expect(withRecovery({}, settings).overrides?.continuationGateway).toEqual({ enabled: false }); + }); + + it.each([ + ["sglang", "0.8.8", "chart"], ["sglang", "0.8.10", "chart"], + ["vllm", "0.8.8", "chart"], ["other", "1.0.0", "chart"], + ["sglang", "0.8.11", undefined], ["sglang", "0.9.22", undefined], ["vllm", "0.8.9", undefined], + ])("checks the resolved %s %s chart", (name, version, error) => { + expect(recoveryError(plan({ form: enabled }, { name, version }))).toBe(error); + }); + + it.each([ + [{ cart: { enabled: false } }, "cart"], + [{ modelRoute: { enabled: false } }, "route"], + [{ continuationGateway: { config: { continuationModel: " " } } }, "model"], + [{ continuationGateway: { config: { stallIdleTimeoutSeconds: 0 } } }, "timeout"], + [{ continuationGateway: { config: { stallIdleTimeoutSeconds: -1 } } }, "timeout"], + [{ continuationGateway: { config: { stallIdleTimeoutSeconds: Infinity } } }, "timeout"], + [{ continuationGateway: { config: { continuationEngine: "vllm" } } }, "engine"], + ])("validates the final plan including higher-priority YAML overrides", (edit, error) => { + expect(recoveryError(plan({ form: enabled, edit }))).toBe(error); + }); + + it("does not replace the chart timeout when the input is empty", () => { + const request = withRecovery({}, { ...recoverySettings(enabled), idleTimeout: "" }); + expect(request.overrides?.continuationGateway).toMatchObject({ config: { continuationModel: "glm-5.3" } }); + expect((request.overrides?.continuationGateway as { config: object }).config).not.toHaveProperty("stallIdleTimeoutSeconds"); + }); + + it("shows model identity and recovery controls only when enabled", () => { + const off = renderToStaticMarkup( {}} />); + expect(off).toContain("生成中断恢复"); + expect(off).not.toContain("实际模型"); + const on = renderToStaticMarkup( {}} />); + expect(on).toContain("实际模型"); + expect(on).toContain('value="glm-5.3"'); + expect(on).toContain("缓冲工具调用"); + }); +}); diff --git a/web/src/modules/inferences/recovery.ts b/web/src/modules/inferences/recovery.ts new file mode 100644 index 0000000..f2fd877 --- /dev/null +++ b/web/src/modules/inferences/recovery.ts @@ -0,0 +1,84 @@ +import type { Plan, PlanRequest } from "@swiss/lib/api"; + +export interface RecoverySettings { + enabled: boolean; + configured: boolean; + model: string; + idleTimeout: string; + bufferToolCalls: boolean; + resumeEnabled: boolean; +} + +type Tree = Record; +const object = (value: unknown): Tree => value && typeof value === "object" && !Array.isArray(value) ? value as Tree : {}; + +export function mergeValues(...layers: (Tree | undefined)[]): Tree { + const result: Tree = {}; + for (const layer of layers) { + for (const [key, value] of Object.entries(layer ?? {})) { + result[key] = value && typeof value === "object" && !Array.isArray(value) + ? mergeValues(object(result[key]), value as Tree) + : value; + } + } + return result; +} + +export function planValues(plan: Plan): Tree { + return mergeValues(...["defaults", "catalog", "site", "derived", "form", "edit"].map((key) => plan.layers[key])); +} + +export function recoverySettings(values: Tree = {}): RecoverySettings { + const gateway = object(values.continuationGateway); + const config = object(gateway.config); + return { + enabled: gateway.enabled === true, + configured: Object.keys(gateway).length > 0, + model: typeof config.continuationModel === "string" ? config.continuationModel : "", + idleTimeout: config.stallIdleTimeoutSeconds == null ? "" : String(config.stallIdleTimeoutSeconds), + bufferToolCalls: typeof config.bufferToolCalls === "boolean" ? config.bufferToolCalls : true, + resumeEnabled: config.continuationEnabled !== false, + }; +} + +export function withRecovery(request: PlanRequest, settings: RecoverySettings): PlanRequest { + if (!settings.enabled && !settings.configured) return request; + const gateway: Tree = { enabled: settings.enabled }; + if (settings.enabled) { + const config: Tree = { + continuationModel: settings.model.trim(), + continuationEnabled: settings.resumeEnabled, + bufferToolCalls: settings.bufferToolCalls, + }; + if (settings.idleTimeout.trim()) config.stallIdleTimeoutSeconds = Number(settings.idleTimeout); + gateway.config = config; + } + return { ...request, overrides: { ...request.overrides, continuationGateway: gateway } }; +} + +export type RecoveryError = "chart" | "cart" | "route" | "model" | "timeout" | "engine"; + +function supportedChart(chart: Plan["chart"]): boolean { + const match = /^(\d+)\.(\d+)\.(\d+)$/.exec(chart.version); + const minimum = chart.name === "sglang" ? [0, 8, 11] : chart.name === "vllm" ? [0, 8, 9] : undefined; + if (!match || !minimum) return false; + const version = match.slice(1).map(Number); + for (let i = 0; i < 3; i++) { + if (version[i] !== minimum[i]) return version[i] > minimum[i]; + } + return true; +} + +export function recoveryError(plan: Plan): RecoveryError | undefined { + const values = planValues(plan); + const gateway = object(values.continuationGateway); + if (gateway.enabled !== true) return; + if (!supportedChart(plan.chart)) return "chart"; + if (object(values.cart).enabled !== true) return "cart"; + if (object(values.modelRoute).enabled !== true) return "route"; + const config = object(gateway.config); + if (typeof config.continuationModel !== "string" || !config.continuationModel.trim()) return "model"; + const timeout = config.stallIdleTimeoutSeconds; + if (timeout !== undefined && (typeof timeout !== "number" || !Number.isFinite(timeout) || timeout <= 0)) return "timeout"; + if (config.continuationEngine !== undefined && config.continuationEngine !== plan.chart.name) return "engine"; +} From ffcbfd31ec969e6ed4ba0bb0e0b4cb8e0e162442 Mon Sep 17 00:00:00 2001 From: imneov Date: Thu, 5 Nov 2026 18:55:37 +0800 Subject: [PATCH 2/2] fix(web): validate and reset recovery overrides on upgrade Add regressions for higher-priority overrides, complete plan layers and timeout removal through validated recomposition. Preserve adaptive concurrency cleanup and reject unrelated changes between compositions. --- docs/console-design.md | 19 +++++ web/src/modules/docs/content/en-US/deploy.md | 4 +- web/src/modules/docs/content/zh-CN/deploy.md | 4 +- .../inferences/adaptive-concurrency.ts | 28 ++----- .../inferences/components/DeploySheet.tsx | 27 +++++-- web/src/modules/inferences/locales/en-US.json | 4 +- web/src/modules/inferences/locales/zh-CN.json | 4 +- .../modules/inferences/plan-composition.ts | 26 ++++++ .../inferences/recovery-compose.test.ts | 79 +++++++++++++++++++ .../modules/inferences/recovery-compose.ts | 28 +++++++ web/src/modules/inferences/recovery.test.tsx | 63 ++++++++++++++- web/src/modules/inferences/recovery.ts | 26 +++++- 12 files changed, 269 insertions(+), 43 deletions(-) create mode 100644 web/src/modules/inferences/plan-composition.ts create mode 100644 web/src/modules/inferences/recovery-compose.test.ts create mode 100644 web/src/modules/inferences/recovery-compose.ts diff --git a/docs/console-design.md b/docs/console-design.md index f4fdad0..47c3230 100644 --- a/docs/console-design.md +++ b/docs/console-design.md @@ -337,6 +337,25 @@ DeploySheet -> GET /api/llm/route-options[?route=] | Text | Per-key locale translations where present; new options retain engine text | | Ownership | Inferences adapts the Swiss metadata without editing the copied upstream module | +### In-flight generation recovery + +The deploy and upgrade forms expose the charts' existing `continuationGateway` values (console #63). + +```text +form -> recovery values -> adaptive compose -> optional timeout reset -> validate -> diff -> apply +nginx -> continuation gateway -> CART -> engine +``` + +| Concern | Decision | +|---|---| +| Controls | Enable gateway, actual model identity, idle timeout, continuation execution and tool-call buffering | +| Upgrade state | Seed from the stored plan once; query refreshes preserve edits | +| Validation | Check all recorded plan layers, chart compatibility, CART, ModelRoute, actual model, timeout and engine | +| Advanced YAML | Preserve precedence; stop when its effective settings differ from the recovery controls | +| Timeout reset | Clearing removes a retained form override by recomposing the validated form without `fromRelease`, pinned to the resolved source and chart | +| Integrity | Recovery and adaptive cleanup share source pinning and normalized plan comparison; reject unrelated changes | +| Ownership | Existing plan API and chart values; copied Swiss sources remain unchanged | + ### Unpublished Monitor integration `modules/inferences/features.ts` holds the single compile-time `ENABLE_MONITOR` flag, false until the Monitor backend is public (console #76). diff --git a/web/src/modules/docs/content/en-US/deploy.md b/web/src/modules/docs/content/en-US/deploy.md index bd071ad..a0b004d 100644 --- a/web/src/modules/docs/content/en-US/deploy.md +++ b/web/src/modules/docs/content/en-US/deploy.md @@ -39,12 +39,12 @@ Most fields can stay empty and take the site profile's and chart's defaults. Enable **Routing → Generation recovery** in the deploy or upgrade form. Enter the **Actual model** whose weights are deployed, such as `kimi-k3` or `glm-5.3`. This selects the continuation strategy; it is not the client-facing served name. - Requires **CART**, **Publish a route**, and `sglang` chart **0.8.11+** or `vllm` chart **0.8.9+**. Composing the plan checks these conditions. -- **Stream idle timeout** is the time without a chunk after streaming starts; empty uses the chart default. +- **Stream idle timeout** is the time without a chunk after streaming starts; clearing removes the stored form timeout override and inherits catalog, site and chart settings. - **Resume interrupted responses** is on by default; off detects and logs interruptions without sending a continuation request. - **Buffer tool calls** holds arguments until a complete call can be sent together, so interrupted calls can be regenerated. - Requests flow through `openresty → continuation gateway → CART → inference instance`. The gateway is deployed with the model release; review and confirm the diff before applying. - Unsupported requests, including structured JSON output, pass through. Verify the actual model and engine combination against the [gateway compatibility documentation](https://github.com/modelsphere/continuation_gateway). -- Upgrades carry the configured recovery settings forward. Turn the switch off, compose and confirm to remove the gateway. Advanced YAML takes precedence over the form; inspect the final plan and diff. +- Upgrades carry the configured recovery settings forward. Turn the switch off, compose and confirm to remove the gateway. Advanced YAML takes precedence over the form; if it overrides the recovery controls, update it before composing the plan. ## 3. Review and confirm diff --git a/web/src/modules/docs/content/zh-CN/deploy.md b/web/src/modules/docs/content/zh-CN/deploy.md index eb0202d..e1573ad 100644 --- a/web/src/modules/docs/content/zh-CN/deploy.md +++ b/web/src/modules/docs/content/zh-CN/deploy.md @@ -38,12 +38,12 @@ 在部署或升级表单的 **路由 → 生成中断恢复** 中开启，填写权重对应的 **实际模型**，例如 `kimi-k3` 或 `glm-5.3`。它用于选择续写策略，不是客户端调用时使用的服务模型名。 - 需要启用 **CART**、**发布路由**，并选择 `sglang` chart **0.8.11+** 或 `vllm` chart **0.8.9+**；生成计划时会检查这些条件。 -- **流式输出空闲超时** 控制开始输出后多久没有新片段视为卡顿；留空使用 chart 默认值。 +- **流式输出空闲超时** 控制开始输出后多久没有新片段视为卡顿；清空会移除已有表单超时覆盖，沿用目录、站点和 chart 配置。 - **执行中断续写** 默认开启；关闭时网关仅检测并记录中断，不发送续写请求。 - **缓冲工具调用** 会将参数保留到调用完整生成后一次性发送，以便在中断时重新生成。 - 请求路径为 `openresty → 恢复网关 → CART → 推理实例`，恢复网关随模型 release 部署；最终仍需预览变更并确认。 - 结构化 JSON 等不支持的请求直接透传。实际模型与引擎的兼容范围见 [恢复网关说明](https://github.com/modelsphere/continuation_gateway)，启用前应验证实际组合。 -- 升级会带出已配置的恢复参数；关闭后再生成、确认计划可移除网关。高级 YAML 的优先级高于表单，检查最终计划和差异。 +- 升级会带出已配置的恢复参数；关闭后再生成、确认计划可移除网关。高级 YAML 的优先级高于表单；如果它覆盖了恢复控件的选择，需先修改高级 YAML 再生成计划。 ## 3. 预览并确认 diff --git a/web/src/modules/inferences/adaptive-concurrency.ts b/web/src/modules/inferences/adaptive-concurrency.ts index 81c3cd5..d925265 100644 --- a/web/src/modules/inferences/adaptive-concurrency.ts +++ b/web/src/modules/inferences/adaptive-concurrency.ts @@ -1,5 +1,6 @@ import type { Form } from "@swiss/components/DeploySettings"; import type { Plan, PlanRequest } from "@swiss/lib/api"; +import { requestFromPlan, samePlanBaseline } from "./plan-composition"; import { routeDirectives, type RouteOptions } from "./routeOptions"; type Tree = Record; @@ -30,14 +31,7 @@ export function withAdaptiveConcurrency(request: PlanRequest, form: Form, option modelRoute.nginx = nginx; overrides.modelRoute = modelRoute; if (!composed) return { ...request, overrides }; - return { - ...request, fromRelease: undefined, - catalog: composed.source.catalogName || catalog || request.catalog, - model: composed.source.model, version: composed.source.version, - variant: composed.source.variant, - release: composed.release.name, namespace: composed.release.namespace, - chartVersion: composed.chart.version, overrides, - }; + return requestFromPlan(request, composed, overrides, catalog); } export class AdaptivePlanChangedError extends Error {} @@ -53,21 +47,9 @@ export async function composeAdaptivePlan(request: PlanRequest, form: Form, opti return proposed; } -function canonical(value: unknown): string { - if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; - if (value && typeof value === "object") return `{${Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, child]) => `${JSON.stringify(key)}:${canonical(child)}`).join(",")}}`; - return JSON.stringify(value) ?? "undefined"; -} - export function sameAdaptiveBaseline(before: Plan, after: Plan, options?: RouteOptions | null): boolean { - const layers = (plan: Plan) => { - const copy = structuredClone(plan.layers ?? {}); - const values = routeValues(copy.form ?? {}); + return samePlanBaseline(before, after, (layers) => { + const values = routeValues(layers.form ?? {}); for (const key of Object.keys(values)) if (isAdaptiveOption(key, options)) delete values[key]; - return canonical(copy); - }; - return before.release.name === after.release.name && before.release.namespace === after.release.namespace - && before.chart.name === after.chart.name && before.chart.version === after.chart.version && before.chart.repo === after.chart.repo && before.chart.path === after.chart.path - && before.engine === after.engine && before.profile === after.profile && layers(before) === layers(after) - && (["catalog", "catalogName", "model", "variant", "version", "hf", "digest"] as const).every((key) => before.source[key] === after.source[key]); + }); } diff --git a/web/src/modules/inferences/components/DeploySheet.tsx b/web/src/modules/inferences/components/DeploySheet.tsx index ce9da2e..e03c216 100644 --- a/web/src/modules/inferences/components/DeploySheet.tsx +++ b/web/src/modules/inferences/components/DeploySheet.tsx @@ -43,8 +43,9 @@ import { DeployForm } from "@/modules/inferences/components/DeployForm"; import type { HelmValuesSource } from "@/modules/inferences/components/OverrideYaml"; import { ExperimentalBadge } from "@/modules/inferences/components/InstallMethodLabel"; import { AdaptivePlanChangedError, composeAdaptivePlan, withAdaptiveConcurrency } from "../adaptive-concurrency"; +import { composeRecoveryPlan, composeRecoveryValuesRequest, RecoveryPlanChangedError } from "../recovery-compose"; import { RecoveryFields } from "./RecoveryFields"; -import { mergeValues, planValues, recoveryError, recoverySettings, withRecovery } from "../recovery"; +import { mergeValues, planValues, recoveryError, recoveryInputError, recoverySettings, withRecovery } from "../recovery"; import { readRouteOptions } from "@/modules/inferences/api"; export type DeployTarget = @@ -206,11 +207,23 @@ function DeploySheetBody({ target, onClose }: { target: DeployTarget; onClose: ( // overrides, which the editor merges itself. Cached per request while the // sheet is open, so saving the overrides and editing them again does not // read the chart again. + const compositionError = (error: unknown) => { + if (error instanceof RecoveryPlanChangedError) return new Error(t("deploy.recovery.errors.changedPlan")); + if (error instanceof AdaptivePlanChangedError) return new Error(t("deploy.fields.adaptiveChanged")); + return error; + }; const valuesForm = { ...form, edits: "" }; const valuesRequest = upgrade && !recorded ? undefined : withAdaptiveConcurrency(withRecovery(requestFor(valuesForm), recovery), valuesForm, routeOptions.data); const helmValues = useQuery({ queryKey: ["helmValues", JSON.stringify(valuesRequest ?? null)], - queryFn: () => deployApi.helmValues(valuesRequest!), + queryFn: async () => { + try { + const request = await composeRecoveryValuesRequest(requestFor(valuesForm), recovery, (next) => composeAdaptivePlan(next, valuesForm, routeOptions.data, deployApi.plan, catalogName), catalogName); + return await deployApi.helmValues(request); + } catch (error) { + throw compositionError(error); + } + }, enabled: false, staleTime: Infinity, // Dropped when the sheet closes: they last one install or upgrade. @@ -226,14 +239,14 @@ function DeploySheetBody({ target, onClose }: { target: DeployTarget; onClose: ( const compose = useMutation({ mutationFn: async () => { try { - if (recovery.enabled && recovery.idleTimeout.trim() && (!Number.isFinite(Number(recovery.idleTimeout)) || Number(recovery.idleTimeout) <= 0)) throw new Error(t("deploy.recovery.errors.timeout")); - const proposed = await composeAdaptivePlan(withRecovery(requestFor(form), recovery), form, routeOptions.data, deployApi.plan, catalogName); - const invalid = recoveryError(proposed); + const inputError = recoveryInputError(recovery); + if (inputError) throw new Error(t(`deploy.recovery.errors.${inputError}`)); + const proposed = await composeRecoveryPlan(requestFor(form), recovery, (request) => composeAdaptivePlan(request, form, routeOptions.data, deployApi.plan, catalogName), catalogName); + const invalid = recoveryError(proposed, recovery); if (invalid) throw new Error(t(`deploy.recovery.errors.${invalid}`)); return proposed; } catch (error) { - if (error instanceof AdaptivePlanChangedError) throw new Error(t("deploy.fields.adaptiveChanged")); - throw error; + throw compositionError(error); } }, onSuccess: (pl) => { diff --git a/web/src/modules/inferences/locales/en-US.json b/web/src/modules/inferences/locales/en-US.json index 5f824f1..07d323d 100644 --- a/web/src/modules/inferences/locales/en-US.json +++ b/web/src/modules/inferences/locales/en-US.json @@ -426,13 +426,15 @@ "model": "Actual model", "modelHint": "The model whose weights are deployed, for example kimi-k3 or glm-5.3. This selects the continuation strategy; it is not the client-facing served name.", "timeout": "Stream idle timeout (seconds)", - "timeoutHint": "Time without a chunk after streaming starts. Empty keeps the chart default.", + "timeoutHint": "Time without a chunk after streaming starts. Clearing removes the form override and inherits catalog, site and chart settings.", "resume": "Resume interrupted responses", "resumeHint": "Off detects and logs interruptions without sending a continuation request.", "buffer": "Buffer tool calls", "bufferHint": "Hold tool-call chunks until complete so interrupted calls can be regenerated. Applies only when resuming is on; arguments arrive together.", "compatibility": "Model and engine compatibility must be verified. Structured JSON responses and other unsupported requests pass through without recovery.", "errors": { + "changedPlan": "The plan changed while clearing the recovery timeout. Generate and review a new plan.", + "overridden": "Advanced YAML overrides the recovery controls. Update its continuationGateway settings before continuing.", "chart": "Generation recovery needs sglang chart 0.8.11+ or vllm chart 0.8.9+. Select a supported chart version.", "cart": "Generation recovery requires CART. Enable CART or turn recovery off.", "route": "Generation recovery requires a published route. Publish a route or turn recovery off.", diff --git a/web/src/modules/inferences/locales/zh-CN.json b/web/src/modules/inferences/locales/zh-CN.json index 83dab6e..9da1ae8 100644 --- a/web/src/modules/inferences/locales/zh-CN.json +++ b/web/src/modules/inferences/locales/zh-CN.json @@ -415,13 +415,15 @@ "model": "实际模型", "modelHint": "填写部署权重对应的模型，例如 kimi-k3 或 glm-5.3，用于选择续写策略；不是客户端调用时使用的服务模型名。", "timeout": "流式输出空闲超时（秒）", - "timeoutHint": "开始输出后，多久没有收到新片段视为卡顿。留空使用 chart 默认值。", + "timeoutHint": "开始输出后，多久没有收到新片段视为卡顿。清空会移除已有表单覆盖，沿用目录、站点及 chart 的配置。", "resume": "执行中断续写", "resumeHint": "关闭时仅检测并记录中断，不发送续写请求。", "buffer": "缓冲工具调用", "bufferHint": "工具调用完整生成后再发送，便于在中断时重新生成。仅在执行续写时生效，调用参数会一次性到达。", "compatibility": "需要验证实际模型与推理引擎的兼容性。结构化 JSON 等不支持的请求会直接透传，不执行恢复。", "errors": { + "changedPlan": "清除恢复超时期间部署计划发生变化，请重新生成并检查计划。", + "overridden": "高级 YAML 覆盖了恢复设置，请先修改其中的 continuationGateway 配置。", "chart": "生成中断恢复需要 sglang chart 0.8.11+ 或 vllm chart 0.8.9+，请选择支持的 chart 版本。", "cart": "生成中断恢复依赖 CART，请启用 CART 或关闭恢复。", "route": "生成中断恢复需要发布路由，请发布路由或关闭恢复。", diff --git a/web/src/modules/inferences/plan-composition.ts b/web/src/modules/inferences/plan-composition.ts new file mode 100644 index 0000000..8c3b774 --- /dev/null +++ b/web/src/modules/inferences/plan-composition.ts @@ -0,0 +1,26 @@ +import type { Plan, PlanRequest } from "@swiss/lib/api"; + +export function requestFromPlan(request: PlanRequest, plan: Plan, overrides: Record, catalog?: string): PlanRequest { + return { + ...request, fromRelease: undefined, + catalog: plan.source.catalogName || catalog || request.catalog, + model: plan.source.model, version: plan.source.version, variant: plan.source.variant, + release: plan.release.name, namespace: plan.release.namespace, + chartVersion: plan.chart.version, overrides, + }; +} + +function canonical(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; + if (value && typeof value === "object") return `{${Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, child]) => `${JSON.stringify(key)}:${canonical(child)}`).join(",")}}`; + return JSON.stringify(value) ?? "undefined"; +} + +export function samePlanBaseline(before: Plan, after: Plan, normalize: (layers: Plan["layers"]) => void): boolean { + const snapshot = (plan: Plan) => { + const layers = structuredClone(plan.layers ?? {}); + normalize(layers); + return canonical({ release: plan.release, source: plan.source, chart: plan.chart, engine: plan.engine, profile: plan.profile, layers }); + }; + return snapshot(before) === snapshot(after); +} diff --git a/web/src/modules/inferences/recovery-compose.test.ts b/web/src/modules/inferences/recovery-compose.test.ts new file mode 100644 index 0000000..fe93a2b --- /dev/null +++ b/web/src/modules/inferences/recovery-compose.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it, vi } from "vitest"; +import type { Plan, PlanRequest } from "@swiss/lib/api"; +import { composeRecoveryPlan, composeRecoveryValuesRequest, RecoveryPlanChangedError } from "./recovery-compose"; +import { EMPTY, planRequest } from "@swiss/components/DeploySettings"; +import { composeAdaptivePlan } from "./adaptive-concurrency"; +import { mergeValues, recoverySettings } from "./recovery"; + +const current: Plan = { + apiVersion: "plan.swiss/v1", release: { name: "model", namespace: "models" }, + source: { model: "weights", variant: "gpu", version: "1.0", catalogName: "private" }, + chart: { name: "sglang", version: "0.9.32" }, engine: "sglang", profile: "site", hash: "original", + layers: { form: { image: { repository: "example.com/engine", tag: "custom" }, continuationGateway: { enabled: true, replicas: 3, config: { continuationModel: "glm-5.3", stallIdleTimeoutSeconds: 45 } } } }, +}; +const cleared = { ...recoverySettings(current.layers.form), idleTimeout: "" }; +const reset = () => { + const copy = structuredClone(current); + delete (copy.layers.form.continuationGateway as { config: { stallIdleTimeoutSeconds?: number } }).config.stallIdleTimeoutSeconds; + return copy; +}; + +describe("recovery timeout reset", () => { + it("removes a timeout retained by fromRelease without dropping unrelated settings", async () => { + const compose = vi.fn<(request: PlanRequest) => Promise>().mockResolvedValueOnce(current).mockResolvedValueOnce(reset()); + const result = await composeRecoveryPlan({ fromRelease: "model" }, cleared, compose); + expect(compose).toHaveBeenCalledTimes(2); + const request = compose.mock.calls[1][0]; + expect(request).toMatchObject({ model: "weights", catalog: "private", release: "model", namespace: "models", chartVersion: "0.9.32" }); + expect(request.fromRelease).toBeUndefined(); + expect(request.overrides?.image).toEqual(current.layers.form.image); + expect(request.overrides?.continuationGateway).toEqual({ enabled: true, replicas: 3, config: { continuationModel: "glm-5.3" } }); + expect(result).toEqual(reset()); + expect((current.layers.form.continuationGateway as { config: object }).config).toHaveProperty("stallIdleTimeoutSeconds", 45); + }); + + it("uses one compose for fresh installs and unchanged explicit timeouts", async () => { + for (const [request, settings] of [[{}, cleared], [{ fromRelease: "model" }, recoverySettings(current.layers.form)]] as const) { + const compose = vi.fn<(request: PlanRequest) => Promise>().mockResolvedValue(current); + await composeRecoveryPlan(request, settings, compose); + expect(compose).toHaveBeenCalledTimes(1); + } + }); + + it("rejects catalog, chart or unrelated configuration changes during reset", async () => { + const changedSource = reset(); changedSource.source.version = "2.0"; + const changedChart = reset(); changedChart.chart.version = "0.9.33"; + const changedImage = reset(); changedImage.layers.form.image = { repository: "example.com/other" }; + for (const changed of [changedSource, changedChart, changedImage, current]) { + const compose = vi.fn<(request: PlanRequest) => Promise>().mockResolvedValueOnce(current).mockResolvedValueOnce(changed); + await expect(composeRecoveryPlan({ fromRelease: "model" }, cleared, compose)).rejects.toBeInstanceOf(RecoveryPlanChangedError); + } + }); +}); + + +it("clears both inherited adaptive fields and the timeout without changing other values", async () => { + const form = { ...EMPTY, cart: true, modelRoute: true, adaptiveCc: false }; + const request = planRequest(form, { model: "weights", fromRelease: "model" }); + const retained = structuredClone(current); + retained.layers.form = mergeValues(request.overrides, retained.layers.form, { modelRoute: { nginx: { values: { adaptive_cc_min: "80" } } } }); + const adaptiveCleaned = structuredClone(retained); + delete (adaptiveCleaned.layers.form.modelRoute as { nginx: { values: Record } }).nginx.values.adaptive_cc_min; + const allCleaned = structuredClone(adaptiveCleaned); + delete (allCleaned.layers.form.continuationGateway as { config: { stallIdleTimeoutSeconds?: number } }).config.stallIdleTimeoutSeconds; + const compose = vi.fn<(request: PlanRequest) => Promise>().mockResolvedValueOnce(retained).mockResolvedValueOnce(adaptiveCleaned).mockResolvedValueOnce(allCleaned); + const result = await composeRecoveryPlan(request, cleared, (next) => composeAdaptivePlan(next, form, null, compose)); + expect(compose).toHaveBeenCalledTimes(3); + expect(compose.mock.calls[2][0].overrides).toEqual(allCleaned.layers.form); + expect(result).toEqual(allCleaned); +}); + +it("previews the cleared upgrade timeout with pinned source and GPU options intact", async () => { + const compose = vi.fn<(request: PlanRequest) => Promise>().mockResolvedValueOnce(current).mockResolvedValueOnce(reset()); + const request = await composeRecoveryValuesRequest({ fromRelease: "model", forceGpuProducts: true, editsYAML: "" }, cleared, compose); + expect(compose).toHaveBeenCalledTimes(2); + expect(request.fromRelease).toBeUndefined(); + expect(request).toMatchObject({ model: "weights", catalog: "private", version: "1.0", variant: "gpu", chartVersion: "0.9.32", forceGpuProducts: true, editsYAML: "" }); + expect(request.overrides).toEqual(reset().layers.form); + expect(request.overrides?.continuationGateway).not.toHaveProperty("config.stallIdleTimeoutSeconds"); +}); diff --git a/web/src/modules/inferences/recovery-compose.ts b/web/src/modules/inferences/recovery-compose.ts new file mode 100644 index 0000000..d397546 --- /dev/null +++ b/web/src/modules/inferences/recovery-compose.ts @@ -0,0 +1,28 @@ +import type { Plan, PlanRequest } from "@swiss/lib/api"; +import { requestFromPlan, samePlanBaseline } from "./plan-composition"; +import { withRecovery, type RecoverySettings } from "./recovery"; + +type Tree = Record; +const object = (value: unknown): Tree => value && typeof value === "object" && !Array.isArray(value) ? value as Tree : {}; +const gatewayConfig = (form: Tree) => object(object(form.continuationGateway).config); +const clearTimeoutOverride = (layers: Plan["layers"]) => { delete gatewayConfig(layers.form ?? {}).stallIdleTimeoutSeconds; }; + +export class RecoveryPlanChangedError extends Error {} + +export async function composeRecoveryPlan(request: PlanRequest, settings: RecoverySettings, compose: (request: PlanRequest) => Promise, catalog?: string): Promise { + const prepared = withRecovery(request, settings); + const proposed = await compose(prepared); + if (!request.fromRelease || !settings.enabled || settings.idleTimeout.trim() || !Object.hasOwn(gatewayConfig(proposed.layers.form ?? {}), "stallIdleTimeoutSeconds")) return proposed; + // fromRelease retains omitted map keys; recompose the validated form to remove this override. + const overrides = structuredClone(proposed.layers.form); + clearTimeoutOverride({ form: overrides }); + const cleaned = await compose(requestFromPlan(prepared, proposed, overrides, catalog)); + if (!samePlanBaseline(proposed, cleaned, clearTimeoutOverride) || Object.hasOwn(gatewayConfig(cleaned.layers.form ?? {}), "stallIdleTimeoutSeconds")) throw new RecoveryPlanChangedError(); + return cleaned; +} + +// Preview the same pinned form that deployment uses, including removed upgrade overrides. +export async function composeRecoveryValuesRequest(request: PlanRequest, settings: RecoverySettings, compose: (request: PlanRequest) => Promise, catalog?: string): Promise { + const plan = await composeRecoveryPlan(request, settings, compose, catalog); + return requestFromPlan(request, plan, plan.layers.form, catalog); +} diff --git a/web/src/modules/inferences/recovery.test.tsx b/web/src/modules/inferences/recovery.test.tsx index f1dd80e..a519b1b 100644 --- a/web/src/modules/inferences/recovery.test.tsx +++ b/web/src/modules/inferences/recovery.test.tsx @@ -1,9 +1,10 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { renderToStaticMarkup } from "react-dom/server"; -import type { Plan } from "@swiss/lib/api"; +import type { Plan, PlanRequest } from "@swiss/lib/api"; import { EMPTY, planRequest } from "@swiss/components/DeploySettings"; +import { composeAdaptivePlan } from "./adaptive-concurrency"; import { RecoveryFields } from "./components/RecoveryFields"; -import { mergeValues, planValues, recoveryError, recoverySettings, withRecovery } from "./recovery"; +import { mergeValues, planValues, recoveryError, recoveryInputError, recoverySettings, withRecovery } from "./recovery"; const plan = (layers: Plan["layers"], chart = { name: "sglang", version: "0.8.11" }): Plan => ({ apiVersion: "swiss.modelsphere.dev/v1alpha1", release: { name: "model", namespace: "models" }, @@ -85,3 +86,59 @@ describe("generation recovery deployment", () => { expect(on).toContain("缓冲工具调用"); }); }); + +describe("recovery review regressions", () => { + it("includes later plan layers when validating the final recovery configuration", () => { + expect(recoveryError(plan({ form: enabled, autoscale: { cart: { enabled: false } } }))).toBe("cart"); + }); + + it("rejects advanced YAML that overrides the operator's recovery choice", () => { + const expected = { ...recoverySettings(enabled), enabled: false }; + expect(recoveryError(plan({ form: enabled }), expected)).toBe("overridden"); + expect(recoveryError(plan({ form: enabled }), { ...expected, enabled: true, model: "kimi-k3" })).toBe("overridden"); + }); + + it("rejects disabled gateway values retained on a chart without that section", () => { + expect(recoveryError(plan({ form: { continuationGateway: { enabled: false } } }, { name: "sglang", version: "0.8.0" }))).toBe("chart"); + }); + + it("keeps intentional detection-only mode valid", () => { + const settings = { ...recoverySettings(enabled), resumeEnabled: false }; + const request = withRecovery({}, settings); + expect(recoveryError(plan({ form: { ...enabled, ...request.overrides } }), settings)).toBeUndefined(); + }); +}); + + +describe("recovery inputs and advanced overrides", () => { + it("rejects invalid input before requesting a plan", () => { + expect(recoveryInputError({ ...recoverySettings(), enabled: true })).toBe("model"); + for (const idleTimeout of ["0", "-1", "invalid", "Infinity"]) { + expect(recoveryInputError({ ...recoverySettings(enabled), idleTimeout })).toBe("timeout"); + } + expect(recoveryInputError({ ...recoverySettings(enabled), idleTimeout: "" })).toBeUndefined(); + }); + + it("checks model, timeout and both execution switches after advanced YAML", () => { + const expected = recoverySettings(enabled); + for (const config of [{ continuationModel: "kimi-k3" }, { stallIdleTimeoutSeconds: 60 }, { bufferToolCalls: false }, { continuationEnabled: false }]) { + expect(recoveryError(plan({ form: enabled, edit: { continuationGateway: { config } } }), expected)).toBe("overridden"); + } + expect(recoveryError(plan({ form: enabled }), { ...expected, idleTimeout: "30.00" })).toBeUndefined(); + }); +}); + + +it("preserves recovery through both adaptive upgrade compositions", async () => { + const form = { ...EMPTY, cart: true, modelRoute: true, adaptiveCc: false }; + const settings = recoverySettings(enabled); + const request = withRecovery(planRequest(form, { model: "weights", fromRelease: "model" }), settings); + const first = plan({ form: mergeValues(request.overrides, { modelRoute: { nginx: { values: { adaptive_cc_min: "80" } } } }) }); + const cleaned = plan({ form: request.overrides! }); + const compose = vi.fn<(request: PlanRequest) => Promise>().mockResolvedValueOnce(first).mockResolvedValueOnce(cleaned); + const result = await composeAdaptivePlan(request, form, null, compose); + expect(compose).toHaveBeenCalledTimes(2); + expect(compose.mock.calls[1][0].overrides?.continuationGateway).toEqual(request.overrides?.continuationGateway); + expect(compose.mock.calls[1][0].fromRelease).toBeUndefined(); + expect(recoveryError(result, settings)).toBeUndefined(); +}); diff --git a/web/src/modules/inferences/recovery.ts b/web/src/modules/inferences/recovery.ts index f2fd877..4141e5e 100644 --- a/web/src/modules/inferences/recovery.ts +++ b/web/src/modules/inferences/recovery.ts @@ -1,4 +1,5 @@ import type { Plan, PlanRequest } from "@swiss/lib/api"; +import { layersOf } from "@swiss/components/Provenance"; export interface RecoverySettings { enabled: boolean; @@ -25,7 +26,7 @@ export function mergeValues(...layers: (Tree | undefined)[]): Tree { } export function planValues(plan: Plan): Tree { - return mergeValues(...["defaults", "catalog", "site", "derived", "form", "edit"].map((key) => plan.layers[key])); + return mergeValues(...layersOf(plan).map((key) => plan.layers[key])); } export function recoverySettings(values: Tree = {}): RecoverySettings { @@ -56,7 +57,7 @@ export function withRecovery(request: PlanRequest, settings: RecoverySettings): return { ...request, overrides: { ...request.overrides, continuationGateway: gateway } }; } -export type RecoveryError = "chart" | "cart" | "route" | "model" | "timeout" | "engine"; +export type RecoveryError = "chart" | "cart" | "route" | "model" | "timeout" | "engine" | "overridden"; function supportedChart(chart: Plan["chart"]): boolean { const match = /^(\d+)\.(\d+)\.(\d+)$/.exec(chart.version); @@ -69,11 +70,28 @@ function supportedChart(chart: Plan["chart"]): boolean { return true; } -export function recoveryError(plan: Plan): RecoveryError | undefined { +export function recoveryInputError(settings: RecoverySettings): "model" | "timeout" | undefined { + if (!settings.enabled) return; + if (!settings.model.trim()) return "model"; + if (settings.idleTimeout.trim() && (!Number.isFinite(Number(settings.idleTimeout)) || Number(settings.idleTimeout) <= 0)) return "timeout"; +} + +function matchesRecovery(actual: RecoverySettings, expected: RecoverySettings): boolean { + if (actual.enabled !== expected.enabled) return false; + if (!expected.enabled) return true; + return actual.model.trim() === expected.model.trim() + && actual.resumeEnabled === expected.resumeEnabled + && actual.bufferToolCalls === expected.bufferToolCalls + && (!expected.idleTimeout.trim() || Number(actual.idleTimeout) === Number(expected.idleTimeout)); +} + +export function recoveryError(plan: Plan, expected?: RecoverySettings): RecoveryError | undefined { const values = planValues(plan); const gateway = object(values.continuationGateway); + const actual = recoverySettings(values); + if (actual.configured && !supportedChart(plan.chart)) return "chart"; + if (expected && !matchesRecovery(actual, expected)) return "overridden"; if (gateway.enabled !== true) return; - if (!supportedChart(plan.chart)) return "chart"; if (object(values.cart).enabled !== true) return "cart"; if (object(values.modelRoute).enabled !== true) return "route"; const config = object(gateway.config);