|
| 1 | +"""v1.5-C (M8 proof point): per-agent LLM dispatch contract. |
| 2 | +
|
| 3 | +Pins the contract that ``runtime.graph._build_agent_nodes`` resolves |
| 4 | +``skill.model`` per-skill, so apps can route different agents through |
| 5 | +different providers without touching the framework. |
| 6 | +
|
| 7 | +The live demonstration of this contract — intake on Ollama Cloud |
| 8 | +gpt-oss while downstream agents follow ``llm.default`` — lives in |
| 9 | +``examples/incident_management/skills/intake/config.yaml`` (the |
| 10 | +``model: gpt_oss_cheap`` line) and is exercised by |
| 11 | +``tests/test_integration_driver_s1.py`` when the appropriate API |
| 12 | +keys are set. |
| 13 | +
|
| 14 | +These tests run without keys: they intercept ``runtime.graph.get_llm`` |
| 15 | +and assert the model name passed for each skill matches the skill's |
| 16 | +``model`` field (or the ``LLMConfig.default`` fallback when ``model`` |
| 17 | +is None). |
| 18 | +""" |
| 19 | +from __future__ import annotations |
| 20 | + |
| 21 | +from unittest.mock import patch |
| 22 | + |
| 23 | +from runtime.config import ( |
| 24 | + AppConfig, |
| 25 | + LLMConfig, |
| 26 | + MCPConfig, |
| 27 | + OrchestratorConfig, |
| 28 | + Paths, |
| 29 | + RuntimeConfig, |
| 30 | +) |
| 31 | +from runtime.mcp_loader import ToolRegistry |
| 32 | +from runtime.skill import RouteRule, Skill |
| 33 | + |
| 34 | + |
| 35 | +def _stub_app_cfg() -> AppConfig: |
| 36 | + """Minimal AppConfig with two named models — the framework picks |
| 37 | + between them by ``skill.model`` only.""" |
| 38 | + llm_cfg = LLMConfig.stub() |
| 39 | + return AppConfig( |
| 40 | + llm=llm_cfg, |
| 41 | + mcp=MCPConfig(servers=[]), |
| 42 | + paths=Paths(skills_dir="config/skills", incidents_dir="/tmp"), |
| 43 | + runtime=RuntimeConfig(state_class=None), |
| 44 | + orchestrator=OrchestratorConfig(), |
| 45 | + ) |
| 46 | + |
| 47 | + |
| 48 | +def test_build_agent_nodes_passes_skill_model_to_get_llm(): |
| 49 | + """The framework must call ``get_llm(cfg.llm, skill.model, ...)`` for |
| 50 | + every responsive skill. Without this, per-agent provider swaps |
| 51 | + silently collapse to the default model. |
| 52 | +
|
| 53 | + We fully mock ``get_llm`` to capture the (role, model_name) tuple |
| 54 | + per skill — this isolates the test from the LLMConfig.models |
| 55 | + registry shape, which is what the production code resolves the |
| 56 | + name through downstream. |
| 57 | + """ |
| 58 | + from runtime.graph import _build_agent_nodes |
| 59 | + from runtime.llm import StubChatModel |
| 60 | + |
| 61 | + skills = { |
| 62 | + "intake": Skill( |
| 63 | + name="intake", |
| 64 | + description="d", |
| 65 | + kind="responsive", |
| 66 | + model="gpt_oss_cheap", |
| 67 | + routes=[RouteRule(when="default", next="triage")], |
| 68 | + system_prompt="x", |
| 69 | + ), |
| 70 | + "triage": Skill( |
| 71 | + name="triage", |
| 72 | + description="d", |
| 73 | + kind="responsive", |
| 74 | + model=None, # falls back to llm.default downstream |
| 75 | + routes=[RouteRule(when="default", next="__end__")], |
| 76 | + system_prompt="x", |
| 77 | + ), |
| 78 | + } |
| 79 | + |
| 80 | + captured: list[tuple[str, str | None]] = [] |
| 81 | + |
| 82 | + def _fake_get_llm(cfg, model_name, *, role, **kwargs): |
| 83 | + captured.append((role, model_name)) |
| 84 | + return StubChatModel(role=role) |
| 85 | + |
| 86 | + cfg = _stub_app_cfg() |
| 87 | + with patch("runtime.graph.get_llm", side_effect=_fake_get_llm): |
| 88 | + nodes = _build_agent_nodes( |
| 89 | + cfg=cfg, |
| 90 | + skills=skills, |
| 91 | + store=None, # type: ignore[arg-type] — _build_agent_nodes |
| 92 | + # only forwards ``store`` to make_agent_node, never reads it |
| 93 | + # itself; tests of the dispatch contract leave it None. |
| 94 | + registry=ToolRegistry(entries={}), |
| 95 | + ) |
| 96 | + |
| 97 | + # Both skills produced a node. |
| 98 | + assert set(nodes.keys()) == {"intake", "triage"} |
| 99 | + |
| 100 | + # Per-skill model resolution: intake got its override, triage got |
| 101 | + # None (which get_llm resolves to llm.default downstream). |
| 102 | + by_role = dict(captured) |
| 103 | + assert by_role.get("intake") == "gpt_oss_cheap", ( |
| 104 | + f"intake should resolve to its skill.model override; got {by_role!r}" |
| 105 | + ) |
| 106 | + assert by_role.get("triage") is None, ( |
| 107 | + f"triage skill.model was None; should pass None through so " |
| 108 | + f"get_llm falls back to llm.default; got {by_role!r}" |
| 109 | + ) |
| 110 | + |
| 111 | + |
| 112 | +def test_intake_skill_yaml_has_per_agent_override_uncommented(): |
| 113 | + """The intake skill config must carry ``model: gpt_oss_cheap`` — the |
| 114 | + v1.5-C deliverable. A human flipping this back to a comment is |
| 115 | + intentional (e.g. forcing all-default for a benchmark run); the |
| 116 | + test fails if that happens silently in a refactor. |
| 117 | + """ |
| 118 | + import yaml |
| 119 | + from pathlib import Path |
| 120 | + |
| 121 | + cfg_path = Path( |
| 122 | + "examples/incident_management/skills/intake/config.yaml" |
| 123 | + ) |
| 124 | + parsed = yaml.safe_load(cfg_path.read_text(encoding="utf-8")) |
| 125 | + assert parsed.get("model") == "gpt_oss_cheap", ( |
| 126 | + f"intake skill must declare ``model: gpt_oss_cheap`` per the " |
| 127 | + f"v1.5-C M8 proof point; got {parsed.get('model')!r}. If " |
| 128 | + f"intentionally rolling back, remove this test guard too." |
| 129 | + ) |
0 commit comments