From dafffd0b295ed75e3b2ea0982a74b15116f3a317 Mon Sep 17 00:00:00 2001 From: Clem Date: Mon, 28 Sep 2026 11:58:21 +0200 Subject: [PATCH 1/5] Use Gemma 4 31B for child simulation in kora v2 --- README.md | 42 ++++++++---- models.json | 4 ++ .../src/models/__tests__/gemmaProfile.test.ts | 66 +++++++++++++++++++ packages/cli/src/models/gatewayModel.ts | 18 +++++ profiles/kora.json | 25 ++++--- 5 files changed, 130 insertions(+), 25 deletions(-) create mode 100644 packages/cli/src/models/__tests__/gemmaProfile.test.ts diff --git a/README.md b/README.md index 42b3879..7f95b56 100644 --- a/README.md +++ b/README.md @@ -203,7 +203,7 @@ yarn kora continue [user-model] | Argument / Option | Description | | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `[user-model]` | Override the profile's `continueUser` role with a `models.json` slug (default: from profile; `deepseek-v3.2-temp-1.3` in `kora`, matching production) | +| `[user-model]` | Override the profile's `continueUser` role with a `models.json` slug (default: from profile; `gemma-4-31b-it` in `kora`) | | `--judges ` | Override the profile's `judges` role with comma-separated `models.json` slugs, odd count (default: from profile — single judge, held constant across 3-turn vs 8-turn comparisons) | | `-i, --input ` | Input JSONL of recorded conversations, same shape as `reassess` (default: `data/reassessment-input.jsonl`) | | `-o, --output ` | Output directory — one `{modelId}.json` per target model, plus `assessments.json`, `continue-meta.json`, and `results.zip` (default: `data/continue-results`) | @@ -371,15 +371,15 @@ role, so the file alone is a complete record of what ran: ```json { "id": "kora", - "version": "1", - "hash": "0b7b93d2…", + "version": "2", + "hash": "3399846e…", "roles": { "seeds": [{"name": "gpt-4o", "model": "openai/gpt-4o"}], "expansion": [{"name": "gpt-5.2:high", "model": "openai/gpt-5.2", "providerOptions": {"openai": {"reasoningEffort": "high"}}}], - "expansionUser": [{"name": "deepseek-v3.2", "model": "deepseek/deepseek-v3.2", "maxTokens": 4000, "temperature": 1.3}], - "user": {"name": "deepseek-v3.2", "model": "deepseek/deepseek-v3.2", "maxTokens": 4000, "temperature": 1.3}, + "expansionUser": [{"name": "gemma-4-31b-it", "model": "google/gemma-4-31b-it", "maxTokens": 300}], + "user": {"name": "gemma-4-31b-it", "model": "google/gemma-4-31b-it", "maxTokens": 300}, "judges": [{"name": "gpt-5.2:medium:limited", "model": "openai/gpt-5.2", "maxTokens": 26000, "providerOptions": {"openai": {"reasoningEffort": "medium"}}}], - "continueUser": {"name": "deepseek-v3.2-temp-1.3", "model": "deepseek/deepseek-v3.2", "maxTokens": 4000, "temperature": 1.3} + "continueUser": {"name": "gemma-4-31b-it", "model": "google/gemma-4-31b-it", "maxTokens": 300} } } ``` @@ -395,13 +395,33 @@ role, so the file alone is a complete record of what ran: Each entry is a `models.json` entry plus a `name`, which is what logs and the `judges` / `user` fields of result files print. The bundled `profiles/kora.json` -reproduces the defaults the CLI used before profiles existed; a test asserts +pins Gemma 4 31B for all child roles; a test asserts every role matches the `models.json` entry of the same name. Select a profile with the global `--profile` option or `KORA_PROFILE`. Nothing in `models.json` is consulted for a profile role: the registry only serves the target model and the command-line overrides below. +### Gemma child model (kora@2) + +The `expansionUser`, `user`, and `continueUser` roles use +`google/gemma-4-31b-it` through AI Gateway, with a 300-token cap and **no +explicit temperature or reasoning options**, matching Thibaut's +[refusal battery](https://github.com/korabench/research/tree/main/refusal_battery). +That study observed Novita as the upstream; the model slug is not a provider +pin. Check the gateway's resolved provider before a production rollout. +The seed, scenario-expansion and judge models and all prompts are unchanged. +DeepSeek entries remain in `models.json` for explicit overrides and old profiles. + +**Existing corpora do not change.** `run` still uses each scenario's stored +`firstUserMessage`. Selecting this profile with an old corpus produces a mixed +run (old opener, Gemma continuations); do not label it entirely Gemma-generated. +`expand-scenarios` generates Gemma openers for new scenarios. This change does +not regenerate any existing first messages or historical results. Use separate +output paths for new runs; resume stamps reject a changed profile. When +reassessing old transcripts, explicitly select the original child model/profile: +`reassess` labels the source child but does not regenerate it. + ### Testing a model configuration (local profiles) To try a different judge, user simulator or expansion model, copy the example @@ -752,22 +772,22 @@ The `run` command produces a result object with this structure: { "target": "gpt-4o", "judges": ["gpt-5.2:medium:limited"], - "user": "deepseek-v3.2", + "user": "gemma-4-31b-it", "prompts": ["default"], "packs": { "taxonomy": {"id": "kora", "version": "2", "hash": "498ec8d2…"}, "behaviors": {"id": "kora", "version": "2", "hash": "b93aee04…"} }, "stamp": { - "profile": {"id": "kora", "version": "1", "hash": "0b7b93d2…"}, - "models": {"user": {"name": "deepseek-v3.2", "model": "deepseek/deepseek-v3.2", "maxTokens": 4000, "temperature": 1.3}, "judges": ["…"], "target": {"name": "gpt-4o", "model": "openai/gpt-4o"}, "…": "…"}, + "profile": {"id": "kora", "version": "2", "hash": "3399846e…"}, + "models": {"user": {"name": "gemma-4-31b-it", "model": "google/gemma-4-31b-it", "maxTokens": 300}, "judges": ["…"], "target": {"name": "gpt-4o", "model": "openai/gpt-4o"}, "…": "…"}, "prompts": {"version": "1", "hash": "7eacbd51…"}, "code": {"version": "1.0.0", "commit": "b73b4731…", "dirty": false}, "packs": {"…": "…"}, "input": {"path": "data/scenarios.jsonl", "sha256": "eeb1a21b…"} }, "served": { - "user": ["deepseek/deepseek-v3.2"], + "user": ["google/gemma-4-31b-it"], "judges": {"gpt-5.2:medium:limited": ["openai/gpt-5.2"]}, "target": ["openai/gpt-4o"] }, diff --git a/models.json b/models.json index f70914a..31bbbf5 100644 --- a/models.json +++ b/models.json @@ -204,5 +204,9 @@ "kimi-k3:limited": { "model": "moonshotai/kimi-k3", "maxTokens": 16000 + }, + "gemma-4-31b-it": { + "model": "google/gemma-4-31b-it", + "maxTokens": 300 } } diff --git a/packages/cli/src/models/__tests__/gemmaProfile.test.ts b/packages/cli/src/models/__tests__/gemmaProfile.test.ts new file mode 100644 index 0000000..1ebb9b3 --- /dev/null +++ b/packages/cli/src/models/__tests__/gemmaProfile.test.ts @@ -0,0 +1,66 @@ +import {readFileSync} from "node:fs"; +import {afterEach, describe, expect, it, vi} from "vitest"; +import {createGatewayModelFromConfig} from "../gatewayModel.js"; + +const {generateText} = vi.hoisted(() => ({generateText: vi.fn()})); +vi.mock("ai", () => ({ + gateway: (model: string) => model, + generateText, + generateObject: vi.fn(), + jsonSchema: vi.fn(), +})); +afterEach(() => vi.resetAllMocks()); + +const profile = JSON.parse( + readFileSync( + new URL("../../../../../profiles/kora.json", import.meta.url), + "utf8" + ) +); +const roles = [ + profile.roles.expansionUser[0], + profile.roles.user, + profile.roles.continueUser, +]; + +describe("Gemma child profile", () => { + it.each(roles)("uses the study settings for $name", async spec => { + generateText.mockResolvedValue({ + text: "why?", + finishReason: "stop", + response: {modelId: spec.model}, + }); + const model = createGatewayModelFromConfig(spec, spec.name, { + retry: {maxRetries: 0}, + }); + expect( + await model.getTextResponse({ + messages: [{role: "user", content: "Go"}], + maxTokens: 300, + }) + ).toBe("why?"); + const request = generateText.mock.calls[0]![0]; + expect(request.model).toBe("google/gemma-4-31b-it"); + expect(request.maxOutputTokens).toBe(300); + expect(request.temperature).toBeUndefined(); + expect(request.providerOptions).toBeUndefined(); + expect([...model.served]).toEqual(["google/gemma-4-31b-it"]); + }); + + it.each(["content-filter", "length", "error", "stop"])( + "does not accept an empty/failed child turn (%s)", + async finishReason => { + generateText.mockResolvedValue({ + text: "", + finishReason, + response: {modelId: "google/gemma-4-31b-it"}, + }); + const model = createGatewayModelFromConfig(roles[0], "gemma", { + retry: {maxRetries: 0}, + }); + await expect( + model.getTextResponse({messages: [{role: "user", content: "Go"}]}) + ).rejects.toThrow(); + } + ); +}); diff --git a/packages/cli/src/models/gatewayModel.ts b/packages/cli/src/models/gatewayModel.ts index 3225a1e..65f32b2 100644 --- a/packages/cli/src/models/gatewayModel.ts +++ b/packages/cli/src/models/gatewayModel.ts @@ -116,6 +116,24 @@ export function createGatewayModelFromConfig( | Record> | undefined, maxRetries: 0, + }).then(result => { + if (config.model === "google/gemma-4-31b-it") { + if (result.finishReason === "length") { + throw new Error( + "Response truncated: max output tokens reached." + ); + } + if ( + result.finishReason === "content-filter" || + result.finishReason === "error" || + !result.text.trim() + ) { + throw new Error( + `Gemma generation failed (${result.finishReason}): no usable child message.` + ); + } + } + return result; }), retryOptions ); diff --git a/profiles/kora.json b/profiles/kora.json index f7660d2..9868646 100644 --- a/profiles/kora.json +++ b/profiles/kora.json @@ -1,7 +1,7 @@ { "id": "kora", - "version": "1", - "hash": "0b7b93d2b3ca55a333ca687b996489d2", + "version": "2", + "hash": "3399846e01e6561355cbdcdb56cc3e4a", "roles": { "seeds": [ { @@ -22,17 +22,15 @@ ], "expansionUser": [ { - "name": "deepseek-v3.2", - "model": "deepseek/deepseek-v3.2", - "maxTokens": 4000, - "temperature": 1.3 + "name": "gemma-4-31b-it", + "model": "google/gemma-4-31b-it", + "maxTokens": 300 } ], "user": { - "name": "deepseek-v3.2", - "model": "deepseek/deepseek-v3.2", - "maxTokens": 4000, - "temperature": 1.3 + "name": "gemma-4-31b-it", + "model": "google/gemma-4-31b-it", + "maxTokens": 300 }, "judges": [ { @@ -47,10 +45,9 @@ } ], "continueUser": { - "name": "deepseek-v3.2-temp-1.3", - "model": "deepseek/deepseek-v3.2", - "maxTokens": 4000, - "temperature": 1.3 + "name": "gemma-4-31b-it", + "model": "google/gemma-4-31b-it", + "maxTokens": 300 } } } From 332fcee20ba3b6e4013cb26c19282d763b29f287 Mon Sep 17 00:00:00 2001 From: Clem Date: Mon, 28 Sep 2026 12:16:06 +0200 Subject: [PATCH 2/5] Make Gemma README documentation self-contained for public readers --- README.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 7f95b56..b7abbd0 100644 --- a/README.md +++ b/README.md @@ -406,10 +406,9 @@ target model and the command-line overrides below. The `expansionUser`, `user`, and `continueUser` roles use `google/gemma-4-31b-it` through AI Gateway, with a 300-token cap and **no -explicit temperature or reasoning options**, matching Thibaut's -[refusal battery](https://github.com/korabench/research/tree/main/refusal_battery). -That study observed Novita as the upstream; the model slug is not a provider -pin. Check the gateway's resolved provider before a production rollout. +explicit temperature or reasoning options**. The upstream provider is selected +by AI Gateway and is not pinned by this profile. Check the gateway's resolved +provider before a production rollout. The seed, scenario-expansion and judge models and all prompts are unchanged. DeepSeek entries remain in `models.json` for explicit overrides and old profiles. @@ -551,7 +550,7 @@ yarn workspace @korabench/apps-web-runner smoke \ The default input is `data/scenarios.jsonl` (the full 781-scenario corpus used for the public leaderboard runs — also the default for API/gateway models). Web targets can take this directly: ```bash -cd /Users/thibaut/dev/kora-benchmark +cd /path/to/kora-benchmark yarn kora run kora-app-gemini \ --concurrency 1 \ -o data/gemini-run.json @@ -572,7 +571,7 @@ yarn web-runner:dev cd ../kora-apps yarn workspace @korabench/apps-web-runner smoke --app gemini --message "What's 2+2?" -cd /Users/thibaut/dev/kora-benchmark +cd /path/to/kora-benchmark yarn kora run kora-app-gemini \ --concurrency 1 \ --limit 2 \ @@ -624,7 +623,7 @@ yarn dev # tsx watch --env-file=../../.env src/server.ts on :7200 Native targets currently use a **temporary** reduced corpus, `data/104-scenario-apps.strict.jsonl` (104 scenarios), instead of the full `data/scenarios.jsonl`. The full corpus parses fine; the reduced file exists only to keep scenarios short enough for the on-device app's input window (e.g. Tako's). Once that constraint is lifted, native runs should switch to `data/scenarios.jsonl` like the web targets. ```bash -cd /Users/thibaut/dev/kora-benchmark +cd /path/to/kora-benchmark yarn kora run kora-app-tiktok-android \ --concurrency 1 \ --cooldown 60 \ From ffb137d4ea048267c463905b6b624acf7e391918 Mon Sep 17 00:00:00 2001 From: Clem Date: Mon, 28 Sep 2026 13:57:21 +0200 Subject: [PATCH 3/5] Describe the Gemma model change without release branding --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b7abbd0..5596864 100644 --- a/README.md +++ b/README.md @@ -402,7 +402,7 @@ Select a profile with the global `--profile` option or `KORA_PROFILE`. Nothing in `models.json` is consulted for a profile role: the registry only serves the target model and the command-line overrides below. -### Gemma child model (kora@2) +### Gemma child model The `expansionUser`, `user`, and `continueUser` roles use `google/gemma-4-31b-it` through AI Gateway, with a 300-token cap and **no From 56805f3de18cac327d6244c898d856e871b7d2f8 Mon Sep 17 00:00:00 2001 From: Clem Date: Tue, 29 Sep 2026 14:08:30 +0200 Subject: [PATCH 4/5] Remove Gemma rollout section from README --- README.md | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/README.md b/README.md index 5596864..d6361e9 100644 --- a/README.md +++ b/README.md @@ -402,25 +402,6 @@ Select a profile with the global `--profile` option or `KORA_PROFILE`. Nothing in `models.json` is consulted for a profile role: the registry only serves the target model and the command-line overrides below. -### Gemma child model - -The `expansionUser`, `user`, and `continueUser` roles use -`google/gemma-4-31b-it` through AI Gateway, with a 300-token cap and **no -explicit temperature or reasoning options**. The upstream provider is selected -by AI Gateway and is not pinned by this profile. Check the gateway's resolved -provider before a production rollout. -The seed, scenario-expansion and judge models and all prompts are unchanged. -DeepSeek entries remain in `models.json` for explicit overrides and old profiles. - -**Existing corpora do not change.** `run` still uses each scenario's stored -`firstUserMessage`. Selecting this profile with an old corpus produces a mixed -run (old opener, Gemma continuations); do not label it entirely Gemma-generated. -`expand-scenarios` generates Gemma openers for new scenarios. This change does -not regenerate any existing first messages or historical results. Use separate -output paths for new runs; resume stamps reject a changed profile. When -reassessing old transcripts, explicitly select the original child model/profile: -`reassess` labels the source child but does not regenerate it. - ### Testing a model configuration (local profiles) To try a different judge, user simulator or expansion model, copy the example From c25c7eaf1f9b71161d395e0e46bd8ac67a3db163 Mon Sep 17 00:00:00 2001 From: Clem Date: Tue, 29 Sep 2026 14:14:36 +0200 Subject: [PATCH 5/5] Preserve existing child message token limits with Gemma --- README.md | 12 ++--- models.json | 2 +- .../src/models/__tests__/gemmaProfile.test.ts | 50 +++++++++++-------- profiles/kora.json | 8 +-- 4 files changed, 40 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index d6361e9..eba1d41 100644 --- a/README.md +++ b/README.md @@ -372,14 +372,14 @@ role, so the file alone is a complete record of what ran: { "id": "kora", "version": "2", - "hash": "3399846e…", + "hash": "aa2b45f1…", "roles": { "seeds": [{"name": "gpt-4o", "model": "openai/gpt-4o"}], "expansion": [{"name": "gpt-5.2:high", "model": "openai/gpt-5.2", "providerOptions": {"openai": {"reasoningEffort": "high"}}}], - "expansionUser": [{"name": "gemma-4-31b-it", "model": "google/gemma-4-31b-it", "maxTokens": 300}], - "user": {"name": "gemma-4-31b-it", "model": "google/gemma-4-31b-it", "maxTokens": 300}, + "expansionUser": [{"name": "gemma-4-31b-it", "model": "google/gemma-4-31b-it", "maxTokens": 4000}], + "user": {"name": "gemma-4-31b-it", "model": "google/gemma-4-31b-it", "maxTokens": 4000}, "judges": [{"name": "gpt-5.2:medium:limited", "model": "openai/gpt-5.2", "maxTokens": 26000, "providerOptions": {"openai": {"reasoningEffort": "medium"}}}], - "continueUser": {"name": "gemma-4-31b-it", "model": "google/gemma-4-31b-it", "maxTokens": 300} + "continueUser": {"name": "gemma-4-31b-it", "model": "google/gemma-4-31b-it", "maxTokens": 4000} } } ``` @@ -759,8 +759,8 @@ The `run` command produces a result object with this structure: "behaviors": {"id": "kora", "version": "2", "hash": "b93aee04…"} }, "stamp": { - "profile": {"id": "kora", "version": "2", "hash": "3399846e…"}, - "models": {"user": {"name": "gemma-4-31b-it", "model": "google/gemma-4-31b-it", "maxTokens": 300}, "judges": ["…"], "target": {"name": "gpt-4o", "model": "openai/gpt-4o"}, "…": "…"}, + "profile": {"id": "kora", "version": "2", "hash": "aa2b45f1…"}, + "models": {"user": {"name": "gemma-4-31b-it", "model": "google/gemma-4-31b-it", "maxTokens": 4000}, "judges": ["…"], "target": {"name": "gpt-4o", "model": "openai/gpt-4o"}, "…": "…"}, "prompts": {"version": "1", "hash": "7eacbd51…"}, "code": {"version": "1.0.0", "commit": "b73b4731…", "dirty": false}, "packs": {"…": "…"}, diff --git a/models.json b/models.json index 31bbbf5..f4aaf8e 100644 --- a/models.json +++ b/models.json @@ -207,6 +207,6 @@ }, "gemma-4-31b-it": { "model": "google/gemma-4-31b-it", - "maxTokens": 300 + "maxTokens": 4000 } } diff --git a/packages/cli/src/models/__tests__/gemmaProfile.test.ts b/packages/cli/src/models/__tests__/gemmaProfile.test.ts index 1ebb9b3..10e036a 100644 --- a/packages/cli/src/models/__tests__/gemmaProfile.test.ts +++ b/packages/cli/src/models/__tests__/gemmaProfile.test.ts @@ -24,28 +24,36 @@ const roles = [ ]; describe("Gemma child profile", () => { - it.each(roles)("uses the study settings for $name", async spec => { - generateText.mockResolvedValue({ - text: "why?", - finishReason: "stop", - response: {modelId: spec.model}, - }); - const model = createGatewayModelFromConfig(spec, spec.name, { - retry: {maxRetries: 0}, - }); - expect( + it.each(roles)( + "preserves the continuation budget and first-message override for $name", + async spec => { + generateText.mockResolvedValue({ + text: "why?", + finishReason: "stop", + response: {modelId: spec.model}, + }); + const model = createGatewayModelFromConfig(spec, spec.name, { + retry: {maxRetries: 0}, + }); await model.getTextResponse({ - messages: [{role: "user", content: "Go"}], - maxTokens: 300, - }) - ).toBe("why?"); - const request = generateText.mock.calls[0]![0]; - expect(request.model).toBe("google/gemma-4-31b-it"); - expect(request.maxOutputTokens).toBe(300); - expect(request.temperature).toBeUndefined(); - expect(request.providerOptions).toBeUndefined(); - expect([...model.served]).toEqual(["google/gemma-4-31b-it"]); - }); + messages: [{role: "user", content: "Continue"}], + }); + expect(generateText.mock.calls[0]![0].maxOutputTokens).toBe(4000); + generateText.mockClear(); + expect( + await model.getTextResponse({ + messages: [{role: "user", content: "Go"}], + maxTokens: 300, + }) + ).toBe("why?"); + const request = generateText.mock.calls[0]![0]; + expect(request.model).toBe("google/gemma-4-31b-it"); + expect(request.maxOutputTokens).toBe(300); + expect(request.temperature).toBeUndefined(); + expect(request.providerOptions).toBeUndefined(); + expect([...model.served]).toEqual(["google/gemma-4-31b-it"]); + } + ); it.each(["content-filter", "length", "error", "stop"])( "does not accept an empty/failed child turn (%s)", diff --git a/profiles/kora.json b/profiles/kora.json index 9868646..3523e73 100644 --- a/profiles/kora.json +++ b/profiles/kora.json @@ -1,7 +1,7 @@ { "id": "kora", "version": "2", - "hash": "3399846e01e6561355cbdcdb56cc3e4a", + "hash": "aa2b45f1a02f133002e1b0f3b13d2ccf", "roles": { "seeds": [ { @@ -24,13 +24,13 @@ { "name": "gemma-4-31b-it", "model": "google/gemma-4-31b-it", - "maxTokens": 300 + "maxTokens": 4000 } ], "user": { "name": "gemma-4-31b-it", "model": "google/gemma-4-31b-it", - "maxTokens": 300 + "maxTokens": 4000 }, "judges": [ { @@ -47,7 +47,7 @@ "continueUser": { "name": "gemma-4-31b-it", "model": "google/gemma-4-31b-it", - "maxTokens": 300 + "maxTokens": 4000 } } }