From d775b9c3e685ab2dbf1d54704c2dd84ebfb66747 Mon Sep 17 00:00:00 2001 From: Nick Charney Kaye Date: Tue, 28 Jul 2026 11:34:41 +0000 Subject: [PATCH] Disable model reasoning for NPC turns; give structured output headroom MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First real-hardware conversation testing (Windows, bundled CPU llama-server, Qwen3-4B starter model) showed every turn degrading to the "I'm not sure what to say right now" fallback: Qwen3 spends hundreds of tokens on chain-of-thought before the JSON, the 512-token default budget truncates the object, parse + repair both fail, and the player never hears the model at all — at 2-4x the latency. - The managed llama-server now starts with LLAMA_ARG_REASONING=off (env var, not CLI flag: older user-installed binaries ignore unknown env vars but would refuse to start on an unknown flag). NPC turns want fast, in-character JSON, not visible thinking. - Turn requests get max_tokens=1024: rubric-rich turn output regularly lands in the 300-600 token range and the default left no headroom. Verified on the target machine: with these two changes the same scenario turn returns a real in-character Japanese reply from the starter model (no fallback), and the engine autostart path brings the model back after a core restart. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01X4GogqwrcPc5M5VkM6wMWT --- .../convsim_core/runtime/sidecar.py | 10 ++++ .../convsim_core/services/turn_pipeline.py | 6 +++ services/convsim-core/tests/test_sidecar.py | 47 +++++++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/services/convsim-core/convsim_core/runtime/sidecar.py b/services/convsim-core/convsim_core/runtime/sidecar.py index 6a2e217d..ee07a000 100644 --- a/services/convsim-core/convsim_core/runtime/sidecar.py +++ b/services/convsim-core/convsim_core/runtime/sidecar.py @@ -265,12 +265,22 @@ async def start( log_fh.write(header) log_fh.flush() + # NPC turns need fast, in-character JSON — not visible chain-of-thought. + # Reasoning-family models (Qwen3, DeepSeek-R1 distils) otherwise spend + # hundreds of tokens thinking, blowing the turn token budget (truncated + # JSON → fallback utterance) and multiplying latency. LLAMA_ARG_REASONING + # is honoured by current llama-server builds and silently ignored by + # older ones, unlike the equivalent CLI flag which would fail startup. + env = dict(os.environ) + env.setdefault("LLAMA_ARG_REASONING", "off") + try: self._process = await asyncio.create_subprocess_exec( *cmd, stdout=log_fh, stderr=subprocess.STDOUT, creationflags=CREATE_NO_WINDOW, + env=env, ) except OSError as exc: self._state = SidecarState.CRASHED diff --git a/services/convsim-core/convsim_core/services/turn_pipeline.py b/services/convsim-core/convsim_core/services/turn_pipeline.py index 3059c75e..0e3b8537 100644 --- a/services/convsim-core/convsim_core/services/turn_pipeline.py +++ b/services/convsim-core/convsim_core/services/turn_pipeline.py @@ -463,6 +463,12 @@ async def process_turn( messages=messages, json_schema=NPC_TURN_OUTPUT_SCHEMA, scripted_turn_index=turn_number, + # Structured turn output (utterance + deltas + rubric observations) + # regularly lands in the 300-600 token range; the 512 default left no + # headroom and a truncated JSON object degrades to the fallback + # utterance. Reasoning is disabled at the engine level, so the extra + # budget costs nothing when unused. + max_tokens=1024, ) logger.debug( "Calling runtime %s for session %s turn %d (estimated %d tokens, truncated=%s)", diff --git a/services/convsim-core/tests/test_sidecar.py b/services/convsim-core/tests/test_sidecar.py index bb568756..78dbd8a1 100644 --- a/services/convsim-core/tests/test_sidecar.py +++ b/services/convsim-core/tests/test_sidecar.py @@ -1037,3 +1037,50 @@ async def test_structured_output_via_fake_server(tmp_path): finally: await sidecar.stop() + + +@pytest.mark.asyncio +async def test_start_disables_model_reasoning_via_env(tmp_path, monkeypatch): + """The spawned llama-server must run with LLAMA_ARG_REASONING=off. + + Reasoning-family models (Qwen3, R1 distils) otherwise burn the turn token + budget on chain-of-thought: the structured turn JSON gets truncated, every + turn degrades to the fallback utterance, and latency multiplies. The env + var (unlike the CLI flag) is ignored by older llama-server binaries, so + user-installed engines keep starting. + """ + from convsim_core.runtime.sidecar import LlamaCppSidecar + + captured: dict = {} + + async def _fake_exec(*args, **kwargs): + captured["env"] = kwargs.get("env") + raise OSError("stop after capture") + + monkeypatch.setattr("asyncio.create_subprocess_exec", _fake_exec) + sidecar = LlamaCppSidecar(log_dir=str(tmp_path)) + with pytest.raises(RuntimeError): + await sidecar.start("/tmp/model.gguf", executable="/tmp/llama-server") + + assert captured["env"] is not None + assert captured["env"].get("LLAMA_ARG_REASONING") == "off" + + +@pytest.mark.asyncio +async def test_start_respects_existing_reasoning_env(tmp_path, monkeypatch): + """An explicit user override of LLAMA_ARG_REASONING wins over the default.""" + from convsim_core.runtime.sidecar import LlamaCppSidecar + + captured: dict = {} + + async def _fake_exec(*args, **kwargs): + captured["env"] = kwargs.get("env") + raise OSError("stop after capture") + + monkeypatch.setenv("LLAMA_ARG_REASONING", "on") + monkeypatch.setattr("asyncio.create_subprocess_exec", _fake_exec) + sidecar = LlamaCppSidecar(log_dir=str(tmp_path)) + with pytest.raises(RuntimeError): + await sidecar.start("/tmp/model.gguf", executable="/tmp/llama-server") + + assert captured["env"].get("LLAMA_ARG_REASONING") == "on"