diff --git a/README.md b/README.md index 981a46f..676ae8b 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Console is a Go backend-for-frontend with a React UI compiled into the same bina - **Identity and access control.** Users, roles and login history stored as Kubernetes CRDs; OAuth2 password login with HS256 tokens; Kubernetes-RBAC-style roles that gate both pages and backend APIs. The seeded administrator is asked to set a password on first login. - **Model deployment.** With [Swiss](https://github.com/modelsphere/swiss) on the cluster, browse the model catalog and deploy, upgrade and uninstall models, with a diff before every change and the GPU nodes they run on. +- **Canary experiments.** When the Canary backend is configured, each experiment arm selects its own v2 model catalog, then a model and variant within it. The catalog selector defaults to `upstreamv2` when available, otherwise the supported environment default or first supported catalog. Changing catalogs clears the arm's previous model, variant, chart pin and YAML overrides; v1 catalogs cannot be selected. - **Playground.** Streaming chat with the models deployed in Model Serving, full sampling parameters, per-answer TTFT, tokens/s and cache hit rate, reasoning output, and "view code" for cURL, Python and Node.js. - **OpenAI-compatible router.** Programs call `/v1` with API keys that administrators issue — with expiry and optional per-model scope. Keys are stored hashed; usage is exported as Prometheus metrics. - **Gateway discovery.** The inference entrypoint, routes and gateway key are read from the cluster (Swiss's site profile or the route ConfigMap) and followed as they change — no URLs or keys copied into configuration. diff --git a/web/src/modules/canary/lib/api.ts b/web/src/modules/canary/lib/api.ts index 365d019..efa4534 100644 --- a/web/src/modules/canary/lib/api.ts +++ b/web/src/modules/canary/lib/api.ts @@ -3,6 +3,7 @@ import type { ChartVersionsSource, CatalogIndexV2, CatalogVariant, + CatalogInfo, LLMCanary, ProductionTarget, } from "../types"; @@ -25,13 +26,14 @@ export interface CanaryApi { } export interface CatalogApi { - index(): Promise; + list(): Promise; + index(catalog?: string): Promise; variant( model: string, id: string, version?: string, ): Promise; - variants(): Promise; + variants(catalog?: string): Promise; chartVersions(source: ChartVersionsSource): Promise; } @@ -66,6 +68,9 @@ export const fixtureCanaryApi: CanaryApi = { }; export const fixtureCatalogApi: CatalogApi = { + async list() { + return [{ name: catalogIndexV2.name, schema: "v2", default: true }]; + }, async index() { if (catalogIndexV2.schema !== "v2") { throw new Error("catalog schema v1 is not supported"); diff --git a/web/src/modules/canary/lib/catalog.test.ts b/web/src/modules/canary/lib/catalog.test.ts new file mode 100644 index 0000000..2102c30 --- /dev/null +++ b/web/src/modules/canary/lib/catalog.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; +import { changeArmCatalog, defaultCatalog } from "./catalog"; +import type { ArmDraft } from "../types"; + +describe("arm catalog selection", () => { + it("prefers upstreamv2 even when another catalog is the environment default", () => { + expect(defaultCatalog([ + { name: "upstream", schema: "v1" }, + { name: "inner", schema: "v2", default: true }, + { name: "upstreamv2", schema: "v2" }, + ])).toBe("upstreamv2"); + }); + + it("uses a supported environment default or first supported catalog when upstreamv2 is absent", () => { + expect(defaultCatalog([{ name: "old", schema: "v1", default: true }, { name: "inner", schema: "v2" }])).toBe("inner"); + expect(defaultCatalog([{ name: "first", schema: "v2" }, { name: "inner", schema: "v2", default: true }])).toBe("inner"); + expect(defaultCatalog([{ name: "old", schema: "v1" }])).toBe(""); + expect(defaultCatalog([])).toBe(""); + }); + + it("clears source-specific configuration even when both catalogs contain the same model and variant names", () => { + const arm: ArmDraft = { + name: "baseline", replicaCount: 2, pricingKey: "custom", + source: { catalog: "upstreamv2", model: "same-model", variant: "same-variant", version: "1.0.0", digest: "sha256:old" }, + chart: { name: "sglang", version: "0.8.0" }, + layers: { catalog: { model: { gpus: 8 } }, form: { replicaCount: 2 }, edit: { image: { tag: "old" } } }, + }; + const switched = changeArmCatalog(arm, "inner"); + expect(switched.source).toEqual({ catalog: "inner", model: "", variant: "", version: "" }); + expect(switched.chart).toEqual({ name: "", version: "" }); + expect(switched.layers).toEqual({ form: { replicaCount: 2 } }); + expect(switched).toMatchObject({ name: "baseline", replicaCount: 2, pricingKey: "custom" }); + expect(arm.source?.catalog).toBe("upstreamv2"); + expect(arm.layers.edit).toBeDefined(); + }); +}); diff --git a/web/src/modules/canary/lib/catalog.ts b/web/src/modules/canary/lib/catalog.ts new file mode 100644 index 0000000..d82d4ae --- /dev/null +++ b/web/src/modules/canary/lib/catalog.ts @@ -0,0 +1,16 @@ +import type { ArmDraft, CatalogInfo } from "../types"; + +export function defaultCatalog(catalogs: CatalogInfo[]): string { + const supported = catalogs.filter((c) => c.schema !== "v1"); + return (supported.find((c) => c.name === "upstreamv2") ?? + supported.find((c) => c.default) ?? supported[0])?.name ?? ""; +} + +export function changeArmCatalog(arm: ArmDraft, catalog: string): ArmDraft { + return { + ...arm, + source: { catalog, model: "", variant: "", version: "" }, + chart: { name: "", version: "" }, + layers: { form: { replicaCount: arm.replicaCount } }, + }; +} diff --git a/web/src/modules/canary/lib/live-api.test.ts b/web/src/modules/canary/lib/live-api.test.ts index 841298f..bbc9b55 100644 --- a/web/src/modules/canary/lib/live-api.test.ts +++ b/web/src/modules/canary/lib/live-api.test.ts @@ -4,6 +4,30 @@ import { registerCanaryLocales } from "../locales"; import i18next from "i18next"; describe("live transport", () => { + it("loads available catalogs and reads models from the named catalog through authenticated Swiss transport", async () => { + const catalogs = [{ name: "upstreamv2", schema: "v2", default: true }, { name: "inner", schema: "v2" }]; + const fetcher = vi.fn(async (path: RequestInfo | URL, _init?: RequestInit) => + new Response(JSON.stringify(String(path).endsWith("/cluster") ? { catalogs } : { schema: "v2", name: "inner & test", models: [] }))); + const api = createLiveApi(fetcher, "/custom/canary"); + expect(await api.catalog.list()).toEqual(catalogs); + expect(await api.catalog.index("inner & test")).toMatchObject({ name: "inner & test", models: [] }); + expect(fetcher.mock.calls.map(([path]) => String(path))).toEqual([ + "/api/deploy/cluster", "/api/deploy/catalog?catalog=inner+%26+test", + ]); + expect(fetcher.mock.calls.every(([, init]) => init?.credentials === "same-origin")).toBe(true); + }); + it("preserves independently selected arm catalogs through prepare", async () => { + const fetcher = vi.fn(async (_path: RequestInfo | URL, _init?: RequestInit) => new Response("{}")); + const { emptyDraft } = await import("./emit"); + const draft = emptyDraft(); + draft.arms = ["upstreamv2", "inner"].map((catalog, i) => ({ + name: `arm-${i}`, replicaCount: 1, layers: {}, chart: { name: "sglang", version: "0.8.0" }, + source: { catalog, model: "same-model", variant: "same-variant", version: "1.0.0" }, + })); + await createLiveApi(fetcher).experiments.prepare(draft); + const payload = JSON.parse(String(fetcher.mock.calls[0][1]?.body)); + expect(payload.arms.map((a: { source: { catalog: string } }) => a.source.catalog)).toEqual(["upstreamv2", "inner"]); + }); it("patches only weights and the expected generation through authenticated transport", async () => { const fetcher = vi.fn(async (_input: RequestInfo | URL, _init?: RequestInit) => new Response("{}")); await createLiveApi(fetcher).experiments.updateTraffic("a/b", { production: 90, canary: 10, expectedGeneration: 4 }); diff --git a/web/src/modules/canary/lib/live-api.ts b/web/src/modules/canary/lib/live-api.ts index 50a4db0..8c0c06b 100644 --- a/web/src/modules/canary/lib/live-api.ts +++ b/web/src/modules/canary/lib/live-api.ts @@ -6,6 +6,7 @@ import type { ChartVersionsSource, CatalogIndexV2, CatalogVariant, + CatalogInfo, LLMCanary, ProductionTarget, Experiment, @@ -29,8 +30,9 @@ export interface CanaryClient { get(namespace: string, name: string): Promise; }; catalog: { - index(): Promise; - variants(): Promise; + list(): Promise; + index(catalog?: string): Promise; + variants(catalog?: string): Promise; chartVersions(source: ChartVersionsSource): Promise; }; swiss: { deployments(): Promise }; @@ -164,7 +166,15 @@ export function createLiveApi( ), }; const catalog = { - index: () => request("/catalog"), + list: async () => (await request<{ catalogs?: CatalogInfo[] }>("/cluster", "GET", undefined, "/api/deploy")).catalogs ?? [], + async index(name?: string): Promise { + const index = await request( + name ? `/catalog?${new URLSearchParams({ catalog: name })}` : "/catalog", + "GET", undefined, name ? "/api/deploy" : base, + ); + if (index.schema !== "v2") throw new Error("catalog schema v2 is required"); + return index; + }, async chartVersions(source: ChartVersionsSource): Promise { const query = new URLSearchParams({ variant: source.variant, @@ -182,23 +192,8 @@ export function createLiveApi( throw new CanaryApiError("Invalid Swiss chart versions response", undefined, "badResponse"); return { ...data, versions: [...new Set(data.versions)] }; }, - async variants(): Promise { - const index = await catalog.index(); - if (index.schema !== "v2") - throw new Error("catalog schema v2 is required"); - return index.models.flatMap((m) => - m.variants.map((v) => ({ - ...v, - apiVersion: "catalog.swiss/v2" as const, - kind: "Variant" as const, - model: m.name, - engine: v.engine as CatalogVariant["engine"], - chart: v.chart as CatalogVariant["chart"], - // Chart constraints are resolved by Swiss at prepare, never guessed by the UI. - resolvedChartVersion: "", - values: {}, - })), - ); + async variants(name?: string): Promise { + return variantsFromIndex(await catalog.index(name)); }, }; return { @@ -232,3 +227,16 @@ export function createLiveApi( }, }; } + +export function variantsFromIndex(index: CatalogIndexV2): CatalogVariant[] { + return index.models.flatMap((m) => m.variants.map((v) => ({ + ...v, + apiVersion: "catalog.swiss/v2" as const, + kind: "Variant" as const, + model: m.name, + engine: v.engine as CatalogVariant["engine"], + chart: v.chart as CatalogVariant["chart"], + resolvedChartVersion: "", + values: {}, + }))); +} diff --git a/web/src/modules/canary/locales.ts b/web/src/modules/canary/locales.ts index dfa7267..2bc3907 100644 --- a/web/src/modules/canary/locales.ts +++ b/web/src/modules/canary/locales.ts @@ -49,11 +49,23 @@ export const en = { environmentUnset: "Not configured (environment default)", environmentDefaultPort: "Use environment default port", armsHint: - "Each arm takes one catalog schema v2 variant: chart and values. The traffic split, Gateway, and HTTPRoute are the following steps.", + "Choose a catalog, then a model and variant for each arm. The default catalog is upstreamv2 when available.", addArm: "Add arm", removeArm: "Remove", armName: "Arm name", model: "Model", + catalog: "Model catalog", + catalogHint: "Changing the catalog clears this arm's model, variant, chart version and YAML overrides. Canary requires catalog schema v2.", + selectCatalog: "Select a model catalog", + selectModel: "Select a model", + searchModels: "Search models", + catalogV1Unsupported: "v1, unsupported for Canary", + catalogLoading: "Loading models…", + catalogsLoading: "Loading model catalogs…", + catalogsEmpty: "No v2 model catalogs are configured. Ask an administrator to configure one.", + catalogFailed: "Could not load models from this catalog.", + catalogRetry: "Retry loading catalog", + catalogEmpty: "No models are available. Select another model catalog.", variant: "Variant @ version", chart: "Chart", chartPin: "Chart {name}", @@ -205,11 +217,23 @@ export const zh = { environmentUnset: "未配置（使用环境默认行为）", environmentDefaultPort: "使用环境默认端口", armsHint: - "每个组别选择一个目录 schema v2 变体，只决定 chart 和 values。流量拆分、Gateway 和 HTTPRoute 在后面的步骤里填写。", + "每个组别先选模型目录，再选该目录中的模型和变体。默认使用 upstreamv2（如已配置）。", addArm: "添加组别", removeArm: "删除", armName: "组别名称", model: "模型", + catalog: "模型目录", + catalogHint: "切换目录会清空当前组别的模型、变体、chart 版本及 YAML 覆盖配置。北极星实验需要 schema v2 目录。", + selectCatalog: "请选择模型目录", + selectModel: "请选择模型", + searchModels: "搜索模型", + catalogV1Unsupported: "v1，不支持北极星实验", + catalogLoading: "正在加载模型…", + catalogsLoading: "正在加载模型目录…", + catalogsEmpty: "尚未配置 v2 模型目录，请联系管理员配置。", + catalogFailed: "无法加载该目录中的模型。", + catalogRetry: "重新加载目录", + catalogEmpty: "没有可用模型，请选择其他模型目录。", variant: "变体 @ 版本", chart: "Chart", chartPin: "Chart {name}", diff --git a/web/src/modules/canary/pages/CanaryCreate.tsx b/web/src/modules/canary/pages/CanaryCreate.tsx index a267405..40e95b0 100644 --- a/web/src/modules/canary/pages/CanaryCreate.tsx +++ b/web/src/modules/canary/pages/CanaryCreate.tsx @@ -22,6 +22,9 @@ import { cn, } from "@modelsphere/ui"; import { useCanaryApi, useCanaryWrite } from "../lib/provider"; +import { variantsFromIndex } from "../lib/live-api"; +import { changeArmCatalog, defaultCatalog } from "../lib/catalog"; +import type { CatalogInfo } from "../types"; import { usePluginPath } from "../lib/host"; import { emptyDraft, emitManifests, layersFromVariant } from "../lib/emit"; import { applyProductionTarget, findProductionTarget } from "../lib/target"; @@ -132,22 +135,31 @@ export function CanaryCreate() { ); } }, [config.data]); - const catalog = useQuery({ - queryKey: ["canary", api.mode, "catalog"], - queryFn: () => api.catalog.index(), + const catalogs = useQuery({ + queryKey: ["canary", api.mode, "catalogs"], + queryFn: () => api.catalog.list(), + staleTime: 60_000, }); - const variants = useQuery({ - queryKey: ["canary", api.mode, "variants"], - queryFn: () => api.catalog.variants(), + const catalogName = defaultCatalog(catalogs.data ?? []); + const armCatalogs = useQueries({ + queries: draft.arms.map((arm) => { + const name = arm.source?.catalog || catalogName; + return { + queryKey: ["canary", api.mode, "catalog", name], + queryFn: () => api.catalog.index(name), + enabled: !!name && !!catalogs.data?.some((c) => c.name === name && c.schema !== "v1"), + staleTime: 60_000, + retry: false, + }; + }), }); // Keep queries in the wizard so readiness remains valid when arm editors unmount. const chartVersions = useQueries({ queries: draft.arms.map((arm) => ({ - queryKey: ["canary", api.mode, "chart-versions", arm.source?.catalog ?? catalog.data?.name, + queryKey: ["canary", api.mode, "chart-versions", arm.source?.catalog, arm.source?.model, arm.source?.variant, arm.source?.version], - queryFn: () => api.catalog.chartVersions({ ...arm.source!, - catalog: arm.source?.catalog ?? catalog.data?.name }), - enabled: !!catalog.data && !!arm.source?.model && !!arm.source?.variant && !!arm.source?.version, + queryFn: () => api.catalog.chartVersions(arm.source!), + enabled: !!arm.source?.catalog && !!arm.source?.model && !!arm.source?.variant && !!arm.source?.version, staleTime: 60_000, retry: false, })), @@ -169,7 +181,7 @@ export function CanaryCreate() { }, [draft, chartVersions]); const chartVersionsReady = draft.arms.every((arm, index) => { const result = chartVersions[index]; - return result?.isSuccess && result.data.chart === arm.chart.name && + return armCatalogs[index]?.isSuccess && result?.isSuccess && result.data.chart === arm.chart.name && isExactChartVersion(arm.chart.version) && result.data.versions.includes(arm.chart.version); }); const deployments = useQuery({ @@ -311,21 +323,12 @@ export function CanaryCreate() { } function blankArm(name: string): ArmDraft { - const first = variants.data?.[0]; - const base: ArmDraft = { + return changeArmCatalog({ name, - chart: { name: "sglang", version: "0.8.0" }, + chart: { name: "", version: "" }, replicaCount: 1, - layers: { - form: { replicaCount: 1 }, - derived: { - scaler: { enabled: false }, - cart: { enabled: true }, - sloRequirement: { enabled: false }, - }, - }, - }; - return first ? applyVariant(base, first, 1, catalog.data?.name) : base; + layers: {}, + }, catalogName); } return ( @@ -339,15 +342,13 @@ export function CanaryCreate() { {(error || config.error || - catalog.error || - variants.error || + catalogs.error || deployments.error) && ( + {catalogs.isPending &&

{t("create.catalogsLoading")}

} + {catalogs.error && } + {catalogs.isSuccess && !catalogName &&

{t("create.catalogsEmpty")}

} { setDraft((d) => ({ ...d, arms })); @@ -487,15 +492,20 @@ export function CanaryCreate() { renderRow={(arm, _index, patchArm) => ( { void (catalogs.error ? catalogs.refetch() : armCatalogs[_index]?.refetch()); }} + models={armCatalogs[_index]?.data?.models ?? []} + variants={armCatalogs[_index]?.data ? variantsFromIndex(armCatalogs[_index].data) : []} versions={chartVersions[_index]?.data} versionsLoading={chartVersions[_index]?.isFetching ?? false} versionsError={chartVersions[_index]?.error} retryVersions={() => { void chartVersions[_index]?.refetch(); }} onChange={(next) => { if ( + next.source?.catalog !== arm.source?.catalog || next.source?.model !== arm.source?.model || next.source?.variant !== arm.source?.variant ) { @@ -873,6 +883,10 @@ export function CanaryCreate() { function ArmFields({ arm, + catalogs, + catalogLoading, + catalogError, + retryCatalog, models, variants, catalogName, @@ -886,6 +900,10 @@ function ArmFields({ onEdit, }: { arm: ArmDraft; + catalogs: CatalogInfo[]; + catalogLoading: boolean; + catalogError: Error | null | undefined; + retryCatalog: () => void; models: { name: string; displayName?: string; defaultVariant?: string }[]; variants: CatalogVariant[]; catalogName?: string; @@ -899,12 +917,12 @@ function ArmFields({ onEdit: (text: string) => void; }) { const { t } = useTranslation("canary"); - const model = arm.source?.model ?? models[0]?.name ?? ""; + const model = arm.source?.model ?? ""; const modelVariants = variants.filter((v) => v.model === model); const selected = modelVariants.find( (v) => v.id === arm.source?.variant && v.version === arm.source?.version, - ) ?? modelVariants.find((v) => v.id === arm.source?.variant) ?? modelVariants[0]; + ) ?? modelVariants.find((v) => v.id === arm.source?.variant); const availableVersions = versions?.versions ?? []; const versionError = versionsLoading || versionsError || !versions ? undefined @@ -965,6 +983,47 @@ function ArmFields({

{t("economics.bindingHint")}

+ + ({ + value: c.name, + label: c.schema === "v1" ? `${c.name} (${t("create.catalogV1Unsupported")})` : c.name, + disabled: c.schema === "v1", + }))} + onValueChange={(name) => { + if (name !== catalogName) onChange(changeArmCatalog(arm, name)); + }} + /> + + + { + if (nextModel === arm.source?.model) return; + const preferred = models.find((m) => m.name === nextModel)?.defaultVariant; + const v = variants.find((item) => item.model === nextModel && item.id === preferred) ?? + variants.find((item) => item.model === nextModel); + if (v) onChange(applyVariant(arm, v, arm.replicaCount, catalogName)); + }} + options={models.map((m) => ({ value: m.name, label: m.displayName ?? m.name }))} + /> + + {catalogLoading &&

{t("create.catalogLoading")}

} + {catalogError &&
+

{t("create.catalogFailed")}

+ + +
} + {!catalogLoading && !catalogError && !models.length &&

{t("create.catalogEmpty")}

} - - { - // Re-selecting the current value still fires Base UI's - // onValueChange; rebuilding the layers would drop `edit` while - // the YAML textarea keeps showing it. - if (nextModel === arm.source?.model) return; - const v = variants.find((item) => item.model === nextModel); - if (v) - onChange( - applyVariant({ ...arm, name: arm.name }, v, arm.replicaCount, catalogName), - ); - }} - options={models.map((m) => ({ - value: m.name, - label: m.displayName ?? m.name, - }))} - /> - { if (value === `${arm.source?.variant}@${arm.source?.version}`) { return; diff --git a/web/src/modules/canary/types.ts b/web/src/modules/canary/types.ts index 8e3cc97..24cd20c 100644 --- a/web/src/modules/canary/types.ts +++ b/web/src/modules/canary/types.ts @@ -196,6 +196,12 @@ export interface CatalogIndexV2 { models: CatalogIndexModel[]; } +export interface CatalogInfo { + name: string; + schema?: string; + default?: boolean; +} + export interface ProductionTarget { kind: "HelmRelease"; name: string;