diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index 502ae3430e27..ae134d41d2aa 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -1,6 +1,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { DEFAULT_SERVER_SETTINGS, + DEFAULT_TEXT_GENERATION_MODEL_BY_PROVIDER, ModelSelection, ProjectId, ProjectScript, @@ -688,6 +689,30 @@ it.layer(NodeServices.layer)("server settings", (it) => { const settings = yield* serverSettings.getSettings; assert.equal(settings.textGenerationModelSelection.instanceId, "claudeAgent"); + assert.equal( + settings.textGenerationModelSelection.model, + DEFAULT_TEXT_GENERATION_MODEL_BY_PROVIDER[ProviderDriverKind.make("claudeAgent")], + ); + }).pipe(Effect.provide(makeServerSettingsLayer())), + ); + + it.effect("keeps the product text-generation model when a custom model is configured", () => + Effect.gen(function* () { + const serverConfig = yield* ServerConfig.ServerConfig; + const fileSystem = yield* FileSystem.FileSystem; + const serverSettings = yield* ServerSettingsModule.ServerSettingsService; + yield* fileSystem.writeFileString( + serverConfig.settingsPath, + '{"providerInstances":{"codex":{"driver":"codex","enabled":false,"config":{}},"claudeAgent":{"driver":"claudeAgent","config":{"customModels":["z-ai/glm-5.3-flash"]}}},"defaultModelSelection":{"instanceId":"claudeAgent","model":"z-ai/glm-5.3-flash"}}', + ); + + const settings = yield* serverSettings.getSettings; + + assert.equal(settings.textGenerationModelSelection.instanceId, "claudeAgent"); + assert.equal( + settings.textGenerationModelSelection.model, + DEFAULT_TEXT_GENERATION_MODEL_BY_PROVIDER[ProviderDriverKind.make("claudeAgent")], + ); }).pipe(Effect.provide(makeServerSettingsLayer())), ); diff --git a/apps/server/src/serverSettings.ts b/apps/server/src/serverSettings.ts index 6949a66d981b..ac466cbff8e6 100644 --- a/apps/server/src/serverSettings.ts +++ b/apps/server/src/serverSettings.ts @@ -342,6 +342,9 @@ function fallbackTextGenerationProvider(settings: ServerSettings): ServerSetting // Same precedence as isModelSelectionProviderEnabled: an explicit provider // instance wins over the legacy providers map, which decodes to defaults // (codex enabled) when the Providers UI has only written providerInstances. + // The model stays the product text-generation slug. A configured custom + // model is substituted only after that slug's one-shot attempt reports the + // model is unavailable. const fallbackEntry = Object.entries(settings.providers).find(([driver, provider]) => { const instance = settings.providerInstances[ProviderInstanceId.make(driver)]; return instance === undefined ? provider.enabled : resolveProviderInstanceEnabled(instance); diff --git a/apps/server/src/textGeneration/ClaudeTextGeneration.test.ts b/apps/server/src/textGeneration/ClaudeTextGeneration.test.ts index 8fe5152d3450..8fa17bf0e881 100644 --- a/apps/server/src/textGeneration/ClaudeTextGeneration.test.ts +++ b/apps/server/src/textGeneration/ClaudeTextGeneration.test.ts @@ -1,6 +1,11 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { it } from "@effect/vitest"; -import { ClaudeSettings, ProviderInstanceId } from "@t3tools/contracts"; +import { + ClaudeSettings, + DEFAULT_TEXT_GENERATION_MODEL_BY_PROVIDER, + ProviderDriverKind, + ProviderInstanceId, +} from "@t3tools/contracts"; import { HostProcessPlatform, isHostWindows } from "@t3tools/shared/hostProcess"; import { createModelSelection } from "@t3tools/shared/model"; import * as Effect from "effect/Effect"; @@ -23,6 +28,7 @@ import { sanitizeThreadTitle } from "./TextGenerationUtils.ts"; import { makeClaudeTextGeneration } from "./ClaudeTextGeneration.ts"; import { writeFakeCli } from "../testUtils/fakeCli.ts"; const decodeClaudeSettings = Schema.decodeSync(ClaudeSettings); +const encodeUnknownJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); const ClaudeTextGenerationTestLayer = ServerConfig.ServerConfig.layerTest(process.cwd(), { prefix: "t3code-claude-text-generation-test-", @@ -43,7 +49,7 @@ function makeFakeClaudeBinary(dir: string) { source: [ "const argv = process.argv.slice(2);", 'const args = argv.join(" ");', - 'const { realpathSync } = await import("node:fs");', + 'const { appendFileSync, realpathSync } = await import("node:fs");', "", "function fail(message, code) {", ' process.stderr.write(message + "\\n");', @@ -105,20 +111,37 @@ function makeFakeClaudeBinary(dir: string) { ' fail("CLAUDE_CONFIG_DIR was " + (process.env.CLAUDE_CONFIG_DIR ?? ""), 5);', "}", "", - "const stderrText = process.env.T3_FAKE_CLAUDE_STDERR;", - "if (stderrText) {", - ' process.stderr.write(stderrText + "\\n");', + 'const modelIndex = argv.indexOf("--model");', + 'const model = modelIndex === -1 ? "" : (argv[modelIndex + 1] ?? "");', + "const modelLog = process.env.T3_FAKE_CLAUDE_MODEL_LOG;", + 'if (modelLog) appendFileSync(modelLog, model + "\\n");', + "const modelResponsesRaw = process.env.T3_FAKE_CLAUDE_MODEL_RESPONSES;", + "const modelResponse = modelResponsesRaw ? JSON.parse(modelResponsesRaw)[model] : undefined;", + "if (modelResponse) {", + ' if (modelResponse.stderr) process.stderr.write(modelResponse.stderr + "\\n");', + ' process.stdout.write(modelResponse.stdout ?? "");', + " process.exitCode = Number(modelResponse.exitCode ?? 0);", + "} else {", + " const stderrText = process.env.T3_FAKE_CLAUDE_STDERR;", + " if (stderrText) {", + ' process.stderr.write(stderrText + "\\n");', + " }", + ' process.stdout.write(process.env.T3_FAKE_CLAUDE_OUTPUT ?? "");', + " process.exitCode = Number(process.env.T3_FAKE_CLAUDE_EXIT_CODE ?? 0);", "}", "", - 'process.stdout.write(process.env.T3_FAKE_CLAUDE_OUTPUT ?? "");', - "process.exitCode = Number(process.env.T3_FAKE_CLAUDE_EXIT_CODE ?? 0);", - "", ].join("\n"), }); return binDir; }); } +interface FakeClaudeModelResponse { + readonly stdout?: string; + readonly stderr?: string; + readonly exitCode?: number; +} + function withFakeClaudeEnv( input: { output: string; @@ -130,12 +153,18 @@ function withFakeClaudeEnv( configDirMustBe?: string; cwdMustNotBe?: string; claudeConfig?: Partial; + modelResponses?: Readonly>; }, - effectFn: (textGeneration: TextGeneration.TextGeneration["Service"]) => Effect.Effect, + effectFn: ( + textGeneration: TextGeneration.TextGeneration["Service"], + context: { readonly modelLogPath: string }, + ) => Effect.Effect, ) { return Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; const tempDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3code-claude-text-" }); + const modelLogPath = path.join(tempDir, "claude-models.log"); const binDir = yield* makeFakeClaudeBinary(tempDir); const pathDelimiter = (yield* isHostWindows) ? ";" : ":"; const previousPath = process.env.PATH; @@ -147,6 +176,8 @@ function withFakeClaudeEnv( const previousStdinMustContain = process.env.T3_FAKE_CLAUDE_STDIN_MUST_CONTAIN; const previousConfigDirMustBe = process.env.T3_FAKE_CLAUDE_CONFIG_DIR_MUST_BE; const previousCwdMustNotBe = process.env.T3_FAKE_CLAUDE_CWD_MUST_NOT_BE; + const previousModelLog = process.env.T3_FAKE_CLAUDE_MODEL_LOG; + const previousModelResponses = process.env.T3_FAKE_CLAUDE_MODEL_RESPONSES; yield* Effect.acquireRelease( Effect.sync(() => { @@ -194,6 +225,13 @@ function withFakeClaudeEnv( } else { delete process.env.T3_FAKE_CLAUDE_CONFIG_DIR_MUST_BE; } + + process.env.T3_FAKE_CLAUDE_MODEL_LOG = modelLogPath; + if (input.modelResponses !== undefined) { + process.env.T3_FAKE_CLAUDE_MODEL_RESPONSES = encodeUnknownJson(input.modelResponses); + } else { + delete process.env.T3_FAKE_CLAUDE_MODEL_RESPONSES; + } }), () => Effect.sync(() => { @@ -246,6 +284,18 @@ function withFakeClaudeEnv( } else { process.env.T3_FAKE_CLAUDE_CONFIG_DIR_MUST_BE = previousConfigDirMustBe; } + + if (previousModelLog === undefined) { + delete process.env.T3_FAKE_CLAUDE_MODEL_LOG; + } else { + process.env.T3_FAKE_CLAUDE_MODEL_LOG = previousModelLog; + } + + if (previousModelResponses === undefined) { + delete process.env.T3_FAKE_CLAUDE_MODEL_RESPONSES; + } else { + process.env.T3_FAKE_CLAUDE_MODEL_RESPONSES = previousModelResponses; + } }), ); @@ -255,7 +305,7 @@ function withFakeClaudeEnv( undefined, Effect.succeed(SYNTHETIC_CLAUDE_MODEL_CATALOG), ); - return yield* effectFn(textGeneration); + return yield* effectFn(textGeneration, { modelLogPath }); }).pipe(Effect.scoped); } @@ -579,4 +629,162 @@ it.layer(ClaudeTextGenerationTestLayer)("ClaudeTextGeneration", (it) => { }), ), ); + + const configuredProductModel = + DEFAULT_TEXT_GENERATION_MODEL_BY_PROVIDER[ProviderDriverKind.make("claudeAgent")]; + if (configuredProductModel === undefined) { + throw new Error("Claude product text-generation model is not configured"); + } + const productModel = configuredProductModel; + const customModel = "z-ai/glm-5.3-flash"; + const wrapperStderr = + "Using the OpenRouter credential from the global credential ~/.ori/credentials.json."; + const guardrailStdout = JSON.stringify({ + api_error_status: 400, + is_error: true, + result: + "API Error: 400 0 endpoints out of 4 requested are available matching your guardrail restrictions and data policy. Model blocked by guardrail: 4 endpoints excluded", + }); + const readSpawnedModels = (modelLogPath: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + return (yield* fs.readFileString(modelLogPath)) + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0); + }); + + it.effect("keeps the product text-generation model when that slug succeeds", () => { + expect(productModel).toBe("claude-haiku-4-5"); + return withFakeClaudeEnv( + { + output: JSON.stringify({ + structured_output: { subject: "Keep the product model", body: "" }, + }), + claudeConfig: { customModels: [customModel] }, + }, + (textGeneration, { modelLogPath }) => + Effect.gen(function* () { + const generated = yield* textGeneration.generateCommitMessage({ + cwd: process.cwd(), + branch: "main", + stagedSummary: "M README.md", + stagedPatch: "diff --git a/README.md b/README.md", + modelSelection: { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: productModel, + }, + }); + + expect(generated.subject).toBe("Keep the product model"); + expect(yield* readSpawnedModels(modelLogPath)).toEqual([productModel]); + }), + ); + }); + + it.effect("uses a configured custom model only after the product slug is unavailable", () => { + expect(productModel).toBe("claude-haiku-4-5"); + return withFakeClaudeEnv( + { + output: "", + claudeConfig: { customModels: [customModel] }, + modelResponses: { + [productModel]: { + exitCode: 1, + stderr: wrapperStderr, + stdout: guardrailStdout, + }, + [customModel]: { + exitCode: 0, + stdout: JSON.stringify({ + structured_output: { subject: "Use the configured custom model", body: "" }, + }), + }, + }, + }, + (textGeneration, { modelLogPath }) => + Effect.gen(function* () { + const generated = yield* textGeneration.generateCommitMessage({ + cwd: process.cwd(), + branch: "main", + stagedSummary: "M README.md", + stagedPatch: "diff --git a/README.md b/README.md", + modelSelection: { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: productModel, + }, + }); + + expect(generated.subject).toBe("Use the configured custom model"); + expect(yield* readSpawnedModels(modelLogPath)).toEqual([productModel, customModel]); + }), + ); + }); + + it.effect( + "does not substitute a custom model when the product slug fails for another reason", + () => { + expect(productModel).toBe("claude-haiku-4-5"); + return withFakeClaudeEnv( + { + output: "", + exitCode: 1, + stderr: wrapperStderr, + claudeConfig: { customModels: [customModel] }, + }, + (textGeneration, { modelLogPath }) => + Effect.gen(function* () { + const error = yield* Effect.flip( + textGeneration.generateCommitMessage({ + cwd: process.cwd(), + branch: "main", + stagedSummary: "M README.md", + stagedPatch: "diff --git a/README.md b/README.md", + modelSelection: { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: productModel, + }, + }), + ); + + expect(error._tag).toBe("TextGenerationError"); + expect(error.detail).toContain(wrapperStderr); + expect(yield* readSpawnedModels(modelLogPath)).toEqual([productModel]); + }), + ); + }, + ); + + it.effect("does not replace an explicit non-product model when that model is unavailable", () => { + expect(productModel).toBe("claude-haiku-4-5"); + return withFakeClaudeEnv( + { + output: guardrailStdout, + exitCode: 1, + stderr: wrapperStderr, + claudeConfig: { customModels: [customModel] }, + }, + (textGeneration, { modelLogPath }) => + Effect.gen(function* () { + const error = yield* Effect.flip( + textGeneration.generateCommitMessage({ + cwd: process.cwd(), + branch: "main", + stagedSummary: "M README.md", + stagedPatch: "diff --git a/README.md b/README.md", + modelSelection: { + instanceId: ProviderInstanceId.make("claudeAgent"), + model: SYNTHETIC_CLAUDE_STANDARD_MODEL, + }, + }), + ); + + expect(error._tag).toBe("TextGenerationError"); + const models = yield* readSpawnedModels(modelLogPath); + expect(models).toHaveLength(1); + expect(models[0]?.startsWith(SYNTHETIC_CLAUDE_STANDARD_MODEL)).toBe(true); + expect(models.some((model) => model.includes(customModel))).toBe(false); + }), + ); + }); }); diff --git a/apps/server/src/textGeneration/ClaudeTextGeneration.ts b/apps/server/src/textGeneration/ClaudeTextGeneration.ts index 357ecd686e46..6a698389b41c 100644 --- a/apps/server/src/textGeneration/ClaudeTextGeneration.ts +++ b/apps/server/src/textGeneration/ClaudeTextGeneration.ts @@ -14,12 +14,21 @@ import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import { type ClaudeSettings, type ModelSelection } from "@t3tools/contracts"; +import { + DEFAULT_TEXT_GENERATION_MODEL_BY_PROVIDER, + ProviderDriverKind, + TextGenerationError, + type ClaudeSettings, + type ModelSelection, +} from "@t3tools/contracts"; import { sanitizeBranchFragment, sanitizeFeatureBranchName } from "@t3tools/shared/git"; import { resolveSpawnCommand } from "@t3tools/shared/shell"; -import { TextGenerationError } from "@t3tools/contracts"; import * as TextGeneration from "./TextGeneration.ts"; +import { + customModelForBrokenTextGenerationFallback, + isBrokenProductTextGenerationFallback, +} from "./TextGenerationModelFallback.ts"; import { buildBranchNamePrompt, buildCommitMessagePrompt, @@ -51,6 +60,17 @@ import { import { makeClaudeEnvironment } from "../provider/Drivers/ClaudeHome.ts"; const CLAUDE_TIMEOUT_MS = 180_000; +const CLAUDE_PRODUCT_TEXT_GENERATION_MODEL = + DEFAULT_TEXT_GENERATION_MODEL_BY_PROVIDER[ProviderDriverKind.make("claudeAgent")]; + +function claudeCliFailureDetail(exitCode: number, stdout: string, stderr: string): string { + const stderrDetail = stderr.trim(); + const stdoutDetail = stdout.trim(); + const detail = stderrDetail.length > 0 ? stderrDetail : stdoutDetail; + return detail.length > 0 + ? `Claude CLI command failed: ${detail}` + : `Claude CLI command failed with code ${exitCode}.`; +} /** * Schema for the wrapper JSON returned by `claude -p --output-format json`. @@ -139,52 +159,55 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu modelSelection: ModelSelection; }): Effect.fn.Return { const catalog = yield* scopedModelCatalog; - const resolvedModelSelection = { - ...modelSelection, - model: resolveClaudeModelSlug(catalog, modelSelection.model), - }; + const requestedModel = resolveClaudeModelSlug(catalog, modelSelection.model); const jsonSchemaStr = yield* encodeJsonForOperation( operation, toJsonSchemaObject(outputSchemaJson), "Failed to encode structured output schema.", ); - const caps = getClaudeCatalogModelCapabilities(catalog, resolvedModelSelection.model); - const descriptors = getProviderOptionDescriptors({ - caps, - selections: resolvedModelSelection.options, - }); - const findDescriptor = (id: string) => descriptors.find((descriptor) => descriptor.id === id); - const rawEffortSelection = getModelSelectionStringOptionValue(resolvedModelSelection, "effort"); - const resolvedEffort = resolveClaudeCatalogEffort( - catalog, - resolvedModelSelection.model, - rawEffortSelection, - ); - const cliEffort = normalizeClaudeCatalogEffort( - catalog, - resolvedEffort, - resolvedModelSelection.model, - ); - const ultracode = isClaudeCatalogUltracodeEffort(resolvedEffort); - const thinkingDescriptor = findDescriptor("thinking"); - const fastModeDescriptor = findDescriptor("fastMode"); - const thinking = - thinkingDescriptor?.type === "boolean" ? thinkingDescriptor.currentValue : undefined; - const fastMode = - fastModeDescriptor?.type === "boolean" ? fastModeDescriptor.currentValue : undefined; - const settings = { - disableAllHooks: true, - ...(typeof thinking === "boolean" ? { alwaysThinkingEnabled: thinking } : {}), - ...(fastMode ? { fastMode: true } : {}), - ...(ultracode ? { ultracode: true } : {}), - }; - const settingsJson = yield* encodeJsonForOperation( - operation, - settings, - "Failed to encode Claude CLI settings.", - ); - const runClaudeCommand = Effect.fn("runClaudeJson.runClaudeCommand")(function* () { + const runClaudeCommand = Effect.fn("runClaudeJson.runClaudeCommand")(function* ( + selection: ModelSelection, + ) { + const resolvedSelection = { + ...selection, + model: resolveClaudeModelSlug(catalog, selection.model), + }; + const caps = getClaudeCatalogModelCapabilities(catalog, resolvedSelection.model); + const descriptors = getProviderOptionDescriptors({ + caps, + selections: resolvedSelection.options, + }); + const findDescriptor = (id: string) => descriptors.find((descriptor) => descriptor.id === id); + const rawEffortSelection = getModelSelectionStringOptionValue(resolvedSelection, "effort"); + const resolvedEffort = resolveClaudeCatalogEffort( + catalog, + resolvedSelection.model, + rawEffortSelection, + ); + const cliEffort = normalizeClaudeCatalogEffort( + catalog, + resolvedEffort, + resolvedSelection.model, + ); + const ultracode = isClaudeCatalogUltracodeEffort(resolvedEffort); + const thinkingDescriptor = findDescriptor("thinking"); + const fastModeDescriptor = findDescriptor("fastMode"); + const thinking = + thinkingDescriptor?.type === "boolean" ? thinkingDescriptor.currentValue : undefined; + const fastMode = + fastModeDescriptor?.type === "boolean" ? fastModeDescriptor.currentValue : undefined; + const settings = { + disableAllHooks: true, + ...(typeof thinking === "boolean" ? { alwaysThinkingEnabled: thinking } : {}), + ...(fastMode ? { fastMode: true } : {}), + ...(ultracode ? { ultracode: true } : {}), + }; + const settingsJson = yield* encodeJsonForOperation( + operation, + settings, + "Failed to encode Claude CLI settings.", + ); // Titles need only the supplied prompt, not configuration from the checkout. const workingDirectory = operation === "generateThreadTitle" @@ -205,7 +228,7 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu "--json-schema", jsonSchemaStr, "--model", - resolveClaudeCatalogApiModelId(catalog, resolvedModelSelection), + resolveClaudeCatalogApiModelId(catalog, resolvedSelection), ...(cliEffort ? ["--effort", cliEffort] : []), "--settings", settingsJson, @@ -249,35 +272,64 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu { concurrency: "unbounded" }, ); - if (exitCode !== 0) { - const stderrDetail = stderr.trim(); - const stdoutDetail = stdout.trim(); - const detail = stderrDetail.length > 0 ? stderrDetail : stdoutDetail; - return yield* new TextGenerationError({ - operation, - detail: - detail.length > 0 - ? `Claude CLI command failed: ${detail}` - : `Claude CLI command failed with code ${exitCode}.`, + return { stdout, stderr, exitCode }; + }); + + const runOnce = (selection: ModelSelection) => + runClaudeCommand(selection).pipe(Effect.scoped, Effect.timeoutOption(CLAUDE_TIMEOUT_MS)); + + const first = yield* runOnce(modelSelection); + if (Option.isNone(first)) { + return yield* new TextGenerationError({ operation, detail: "Claude CLI request timed out." }); + } + + let outcome = first.value; + if (outcome.exitCode !== 0 && CLAUDE_PRODUCT_TEXT_GENERATION_MODEL !== undefined) { + const customModel = customModelForBrokenTextGenerationFallback( + claudeSettings.customModels, + CLAUDE_PRODUCT_TEXT_GENERATION_MODEL, + ); + // The product slug was just attempted. A custom model is used only when + // that attempt reports the model is unavailable. + if ( + customModel !== null && + isBrokenProductTextGenerationFallback({ + productModel: CLAUDE_PRODUCT_TEXT_GENERATION_MODEL, + requestedModel, + stdout: outcome.stdout, + stderr: outcome.stderr, + }) + ) { + yield* Effect.logInfo( + "Retrying one-shot text generation with a configured custom model after the product model was unavailable", + { + operation, + productModel: CLAUDE_PRODUCT_TEXT_GENERATION_MODEL, + customModel, + }, + ); + const second = yield* runOnce({ + instanceId: modelSelection.instanceId, + model: customModel, }); + if (Option.isNone(second)) { + return yield* new TextGenerationError({ + operation, + detail: "Claude CLI request timed out.", + }); + } + outcome = second.value; } + } - return stdout; - }); + if (outcome.exitCode !== 0) { + return yield* new TextGenerationError({ + operation, + detail: claudeCliFailureDetail(outcome.exitCode, outcome.stdout, outcome.stderr), + }); + } - const rawStdout = yield* runClaudeCommand().pipe( - Effect.scoped, - Effect.timeoutOption(CLAUDE_TIMEOUT_MS), - Effect.flatMap( - Option.match({ - onNone: () => - Effect.fail( - new TextGenerationError({ operation, detail: "Claude CLI request timed out." }), - ), - onSome: (value) => Effect.succeed(value), - }), - ), - ); + const rawStdout = outcome.stdout; const output = yield* decodeClaudeOutput(rawStdout).pipe( Effect.catchTags({ diff --git a/apps/server/src/textGeneration/TextGenerationModelFallback.test.ts b/apps/server/src/textGeneration/TextGenerationModelFallback.test.ts new file mode 100644 index 000000000000..4397ff8bcf12 --- /dev/null +++ b/apps/server/src/textGeneration/TextGenerationModelFallback.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + customModelForBrokenTextGenerationFallback, + isBrokenProductTextGenerationFallback, +} from "./TextGenerationModelFallback.ts"; + +const PRODUCT_MODEL = "claude-haiku-4-5"; +const CUSTOM_MODEL = "z-ai/glm-5.3-flash"; +const WRAPPER_STDERR = + "Using the OpenRouter credential from the global credential ~/.ori/credentials.json."; +const GUARDRAIL_STDOUT = JSON.stringify({ + api_error_status: 400, + is_error: true, + result: + "API Error: 400 0 endpoints out of 4 requested are available matching your guardrail restrictions and data policy. Model blocked by guardrail: 4 endpoints excluded", +}); + +describe("isBrokenProductTextGenerationFallback", () => { + it("keeps a healthy product-model response on the product slug", () => { + expect( + isBrokenProductTextGenerationFallback({ + productModel: PRODUCT_MODEL, + requestedModel: PRODUCT_MODEL, + stdout: JSON.stringify({ structured_output: { subject: "Keep the product model" } }), + stderr: "", + }), + ).toBe(false); + }); + + it("treats a guardrail rejection of the product slug as a broken fallback", () => { + expect( + isBrokenProductTextGenerationFallback({ + productModel: PRODUCT_MODEL, + requestedModel: PRODUCT_MODEL, + stdout: GUARDRAIL_STDOUT, + stderr: WRAPPER_STDERR, + }), + ).toBe(true); + }); + + it("does not treat wrapper stderr alone as a broken product fallback", () => { + expect( + isBrokenProductTextGenerationFallback({ + productModel: PRODUCT_MODEL, + requestedModel: PRODUCT_MODEL, + stdout: "", + stderr: WRAPPER_STDERR, + }), + ).toBe(false); + }); + + it("does not treat a rejected non-product model as a broken product fallback", () => { + expect( + isBrokenProductTextGenerationFallback({ + productModel: PRODUCT_MODEL, + requestedModel: "claude-opus-4-6", + stdout: GUARDRAIL_STDOUT, + stderr: WRAPPER_STDERR, + }), + ).toBe(false); + }); +}); + +describe("customModelForBrokenTextGenerationFallback", () => { + it("uses the first configured custom model", () => { + expect(customModelForBrokenTextGenerationFallback([CUSTOM_MODEL], PRODUCT_MODEL)).toBe( + CUSTOM_MODEL, + ); + }); + + it("returns null when no custom model is configured", () => { + expect(customModelForBrokenTextGenerationFallback([], PRODUCT_MODEL)).toBeNull(); + }); + + it("returns null when the only custom slug is the product model", () => { + expect(customModelForBrokenTextGenerationFallback([PRODUCT_MODEL], PRODUCT_MODEL)).toBeNull(); + }); + + it("skips the product slug and uses the next custom model", () => { + expect( + customModelForBrokenTextGenerationFallback([PRODUCT_MODEL, CUSTOM_MODEL], PRODUCT_MODEL), + ).toBe(CUSTOM_MODEL); + }); +}); diff --git a/apps/server/src/textGeneration/TextGenerationModelFallback.ts b/apps/server/src/textGeneration/TextGenerationModelFallback.ts new file mode 100644 index 000000000000..319459aa1849 --- /dev/null +++ b/apps/server/src/textGeneration/TextGenerationModelFallback.ts @@ -0,0 +1,47 @@ +/** + * Decide when one-shot text generation may leave the product model. + * + * Commit, PR, branch, and title generation keep the product slug whenever + * that attempt can run. A configured custom model is a substitute only after + * the product slug was the model just tried and the CLI reported that this + * model is unavailable. + */ +import type { CustomModelSetting } from "@t3tools/contracts"; +import { readCustomModelEntries } from "@t3tools/shared/model"; + +const UNAVAILABLE_MODEL_PATTERNS: ReadonlyArray = [ + /model blocked/i, + /guardrail/i, + /unknown model/i, + /model not found/i, + /invalid model/i, + /unsupported model/i, + /no such model/i, + /model.{0,80}not available/i, + /not available.{0,80}model/i, + /endpoints excluded/i, + /\d+\s+endpoints?\s+out of\s+\d+/i, +]; + +export function isBrokenProductTextGenerationFallback(input: { + readonly productModel: string; + readonly requestedModel: string; + readonly stdout: string; + readonly stderr: string; +}): boolean { + if (input.requestedModel.trim() !== input.productModel.trim()) { + return false; + } + const text = `${input.stdout}\n${input.stderr}`; + return UNAVAILABLE_MODEL_PATTERNS.some((pattern) => pattern.test(text)); +} + +/** First configured custom slug other than the product model, if there is one. */ +export function customModelForBrokenTextGenerationFallback( + customModels: ReadonlyArray, + productModel: string, +): string | null { + return ( + readCustomModelEntries(customModels).find((entry) => entry.slug !== productModel)?.slug ?? null + ); +}