diff --git a/README.md b/README.md index 42b3879..eba1d41 100644 --- a/README.md +++ b/README.md @@ -203,7 +203,7 @@ yarn kora continue [user-model] | Argument / Option | Description | | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `[user-model]` | Override the profile's `continueUser` role with a `models.json` slug (default: from profile; `deepseek-v3.2-temp-1.3` in `kora`, matching production) | +| `[user-model]` | Override the profile's `continueUser` role with a `models.json` slug (default: from profile; `gemma-4-31b-it` in `kora`) | | `--judges ` | Override the profile's `judges` role with comma-separated `models.json` slugs, odd count (default: from profile — single judge, held constant across 3-turn vs 8-turn comparisons) | | `-i, --input ` | Input JSONL of recorded conversations, same shape as `reassess` (default: `data/reassessment-input.jsonl`) | | `-o, --output ` | Output directory — one `{modelId}.json` per target model, plus `assessments.json`, `continue-meta.json`, and `results.zip` (default: `data/continue-results`) | @@ -371,15 +371,15 @@ role, so the file alone is a complete record of what ran: ```json { "id": "kora", - "version": "1", - "hash": "0b7b93d2…", + "version": "2", + "hash": "aa2b45f1…", "roles": { "seeds": [{"name": "gpt-4o", "model": "openai/gpt-4o"}], "expansion": [{"name": "gpt-5.2:high", "model": "openai/gpt-5.2", "providerOptions": {"openai": {"reasoningEffort": "high"}}}], - "expansionUser": [{"name": "deepseek-v3.2", "model": "deepseek/deepseek-v3.2", "maxTokens": 4000, "temperature": 1.3}], - "user": {"name": "deepseek-v3.2", "model": "deepseek/deepseek-v3.2", "maxTokens": 4000, "temperature": 1.3}, + "expansionUser": [{"name": "gemma-4-31b-it", "model": "google/gemma-4-31b-it", "maxTokens": 4000}], + "user": {"name": "gemma-4-31b-it", "model": "google/gemma-4-31b-it", "maxTokens": 4000}, "judges": [{"name": "gpt-5.2:medium:limited", "model": "openai/gpt-5.2", "maxTokens": 26000, "providerOptions": {"openai": {"reasoningEffort": "medium"}}}], - "continueUser": {"name": "deepseek-v3.2-temp-1.3", "model": "deepseek/deepseek-v3.2", "maxTokens": 4000, "temperature": 1.3} + "continueUser": {"name": "gemma-4-31b-it", "model": "google/gemma-4-31b-it", "maxTokens": 4000} } } ``` @@ -395,7 +395,7 @@ role, so the file alone is a complete record of what ran: Each entry is a `models.json` entry plus a `name`, which is what logs and the `judges` / `user` fields of result files print. The bundled `profiles/kora.json` -reproduces the defaults the CLI used before profiles existed; a test asserts +pins Gemma 4 31B for all child roles; a test asserts every role matches the `models.json` entry of the same name. Select a profile with the global `--profile` option or `KORA_PROFILE`. Nothing @@ -531,7 +531,7 @@ yarn workspace @korabench/apps-web-runner smoke \ The default input is `data/scenarios.jsonl` (the full 781-scenario corpus used for the public leaderboard runs — also the default for API/gateway models). Web targets can take this directly: ```bash -cd /Users/thibaut/dev/kora-benchmark +cd /path/to/kora-benchmark yarn kora run kora-app-gemini \ --concurrency 1 \ -o data/gemini-run.json @@ -552,7 +552,7 @@ yarn web-runner:dev cd ../kora-apps yarn workspace @korabench/apps-web-runner smoke --app gemini --message "What's 2+2?" -cd /Users/thibaut/dev/kora-benchmark +cd /path/to/kora-benchmark yarn kora run kora-app-gemini \ --concurrency 1 \ --limit 2 \ @@ -604,7 +604,7 @@ yarn dev # tsx watch --env-file=../../.env src/server.ts on :7200 Native targets currently use a **temporary** reduced corpus, `data/104-scenario-apps.strict.jsonl` (104 scenarios), instead of the full `data/scenarios.jsonl`. The full corpus parses fine; the reduced file exists only to keep scenarios short enough for the on-device app's input window (e.g. Tako's). Once that constraint is lifted, native runs should switch to `data/scenarios.jsonl` like the web targets. ```bash -cd /Users/thibaut/dev/kora-benchmark +cd /path/to/kora-benchmark yarn kora run kora-app-tiktok-android \ --concurrency 1 \ --cooldown 60 \ @@ -752,22 +752,22 @@ The `run` command produces a result object with this structure: { "target": "gpt-4o", "judges": ["gpt-5.2:medium:limited"], - "user": "deepseek-v3.2", + "user": "gemma-4-31b-it", "prompts": ["default"], "packs": { "taxonomy": {"id": "kora", "version": "2", "hash": "498ec8d2…"}, "behaviors": {"id": "kora", "version": "2", "hash": "b93aee04…"} }, "stamp": { - "profile": {"id": "kora", "version": "1", "hash": "0b7b93d2…"}, - "models": {"user": {"name": "deepseek-v3.2", "model": "deepseek/deepseek-v3.2", "maxTokens": 4000, "temperature": 1.3}, "judges": ["…"], "target": {"name": "gpt-4o", "model": "openai/gpt-4o"}, "…": "…"}, + "profile": {"id": "kora", "version": "2", "hash": "aa2b45f1…"}, + "models": {"user": {"name": "gemma-4-31b-it", "model": "google/gemma-4-31b-it", "maxTokens": 4000}, "judges": ["…"], "target": {"name": "gpt-4o", "model": "openai/gpt-4o"}, "…": "…"}, "prompts": {"version": "1", "hash": "7eacbd51…"}, "code": {"version": "1.0.0", "commit": "b73b4731…", "dirty": false}, "packs": {"…": "…"}, "input": {"path": "data/scenarios.jsonl", "sha256": "eeb1a21b…"} }, "served": { - "user": ["deepseek/deepseek-v3.2"], + "user": ["google/gemma-4-31b-it"], "judges": {"gpt-5.2:medium:limited": ["openai/gpt-5.2"]}, "target": ["openai/gpt-4o"] }, diff --git a/models.json b/models.json index f70914a..f4aaf8e 100644 --- a/models.json +++ b/models.json @@ -204,5 +204,9 @@ "kimi-k3:limited": { "model": "moonshotai/kimi-k3", "maxTokens": 16000 + }, + "gemma-4-31b-it": { + "model": "google/gemma-4-31b-it", + "maxTokens": 4000 } } diff --git a/packages/cli/src/models/__tests__/gemmaProfile.test.ts b/packages/cli/src/models/__tests__/gemmaProfile.test.ts new file mode 100644 index 0000000..10e036a --- /dev/null +++ b/packages/cli/src/models/__tests__/gemmaProfile.test.ts @@ -0,0 +1,74 @@ +import {readFileSync} from "node:fs"; +import {afterEach, describe, expect, it, vi} from "vitest"; +import {createGatewayModelFromConfig} from "../gatewayModel.js"; + +const {generateText} = vi.hoisted(() => ({generateText: vi.fn()})); +vi.mock("ai", () => ({ + gateway: (model: string) => model, + generateText, + generateObject: vi.fn(), + jsonSchema: vi.fn(), +})); +afterEach(() => vi.resetAllMocks()); + +const profile = JSON.parse( + readFileSync( + new URL("../../../../../profiles/kora.json", import.meta.url), + "utf8" + ) +); +const roles = [ + profile.roles.expansionUser[0], + profile.roles.user, + profile.roles.continueUser, +]; + +describe("Gemma child profile", () => { + it.each(roles)( + "preserves the continuation budget and first-message override for $name", + async spec => { + generateText.mockResolvedValue({ + text: "why?", + finishReason: "stop", + response: {modelId: spec.model}, + }); + const model = createGatewayModelFromConfig(spec, spec.name, { + retry: {maxRetries: 0}, + }); + await model.getTextResponse({ + messages: [{role: "user", content: "Continue"}], + }); + expect(generateText.mock.calls[0]![0].maxOutputTokens).toBe(4000); + generateText.mockClear(); + expect( + await model.getTextResponse({ + messages: [{role: "user", content: "Go"}], + maxTokens: 300, + }) + ).toBe("why?"); + const request = generateText.mock.calls[0]![0]; + expect(request.model).toBe("google/gemma-4-31b-it"); + expect(request.maxOutputTokens).toBe(300); + expect(request.temperature).toBeUndefined(); + expect(request.providerOptions).toBeUndefined(); + expect([...model.served]).toEqual(["google/gemma-4-31b-it"]); + } + ); + + it.each(["content-filter", "length", "error", "stop"])( + "does not accept an empty/failed child turn (%s)", + async finishReason => { + generateText.mockResolvedValue({ + text: "", + finishReason, + response: {modelId: "google/gemma-4-31b-it"}, + }); + const model = createGatewayModelFromConfig(roles[0], "gemma", { + retry: {maxRetries: 0}, + }); + await expect( + model.getTextResponse({messages: [{role: "user", content: "Go"}]}) + ).rejects.toThrow(); + } + ); +}); diff --git a/packages/cli/src/models/gatewayModel.ts b/packages/cli/src/models/gatewayModel.ts index 3225a1e..65f32b2 100644 --- a/packages/cli/src/models/gatewayModel.ts +++ b/packages/cli/src/models/gatewayModel.ts @@ -116,6 +116,24 @@ export function createGatewayModelFromConfig( | Record> | undefined, maxRetries: 0, + }).then(result => { + if (config.model === "google/gemma-4-31b-it") { + if (result.finishReason === "length") { + throw new Error( + "Response truncated: max output tokens reached." + ); + } + if ( + result.finishReason === "content-filter" || + result.finishReason === "error" || + !result.text.trim() + ) { + throw new Error( + `Gemma generation failed (${result.finishReason}): no usable child message.` + ); + } + } + return result; }), retryOptions ); diff --git a/profiles/kora.json b/profiles/kora.json index f7660d2..3523e73 100644 --- a/profiles/kora.json +++ b/profiles/kora.json @@ -1,7 +1,7 @@ { "id": "kora", - "version": "1", - "hash": "0b7b93d2b3ca55a333ca687b996489d2", + "version": "2", + "hash": "aa2b45f1a02f133002e1b0f3b13d2ccf", "roles": { "seeds": [ { @@ -22,17 +22,15 @@ ], "expansionUser": [ { - "name": "deepseek-v3.2", - "model": "deepseek/deepseek-v3.2", - "maxTokens": 4000, - "temperature": 1.3 + "name": "gemma-4-31b-it", + "model": "google/gemma-4-31b-it", + "maxTokens": 4000 } ], "user": { - "name": "deepseek-v3.2", - "model": "deepseek/deepseek-v3.2", - "maxTokens": 4000, - "temperature": 1.3 + "name": "gemma-4-31b-it", + "model": "google/gemma-4-31b-it", + "maxTokens": 4000 }, "judges": [ { @@ -47,10 +45,9 @@ } ], "continueUser": { - "name": "deepseek-v3.2-temp-1.3", - "model": "deepseek/deepseek-v3.2", - "maxTokens": 4000, - "temperature": 1.3 + "name": "gemma-4-31b-it", + "model": "google/gemma-4-31b-it", + "maxTokens": 4000 } } }