diff --git a/README.md b/README.md index 3839220..8f60a74 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,10 @@ Automatic selection uses the smallest effective local profile: - Balanced: `gemma4:e4b-it-qat` for Apple Silicon with 16 GB or more - Advanced: explicit opt-in only; it is never selected automatically +The same profiles set the Ollama context window explicitly: 16K for Fast and +32K for Balanced or manual Advanced. Set a positive +`CHROMEAI_OLLAMA_NUM_CTX` value only when an explicit local override is needed. + An explicit `CHROMEAI_OLLAMA_MODEL` override remains authoritative. If that model is unavailable locally, SelectPilot reports the missing model instead of silently substituting another model. diff --git a/launchd/com.chromeai.nano.plist b/launchd/com.chromeai.nano.plist index 8ef8571..56b69c5 100644 --- a/launchd/com.chromeai.nano.plist +++ b/launchd/com.chromeai.nano.plist @@ -25,6 +25,7 @@ CHROMEAI_OLLAMA_BASE_URL__OLLAMA_BASE_URL__ CHROMEAI_OLLAMA_MODEL__OLLAMA_MODEL__ CHROMEAI_OLLAMA_EMBED_MODEL__OLLAMA_EMBED_MODEL__ + CHROMEAI_OLLAMA_NUM_CTX__OLLAMA_NUM_CTX__ CHROMEAI_BINARY_HASH__BINARY_HASH__ diff --git a/scripts/bootstrap-macos-local.sh b/scripts/bootstrap-macos-local.sh index 6466cb7..ccf4895 100755 --- a/scripts/bootstrap-macos-local.sh +++ b/scripts/bootstrap-macos-local.sh @@ -62,6 +62,7 @@ print(json.dumps({ "reason": recommendation["reason"], "generation_model": runtime_profile.generation_model, "embedding_model": runtime_profile.embedding_model, + "num_ctx": runtime_profile.num_ctx, "command": commands["command"], })) PY @@ -74,7 +75,7 @@ import shlex import sys payload = json.loads(sys.argv[1]) -for key in ("selected_profile", "generation_model", "embedding_model", "reason"): +for key in ("selected_profile", "generation_model", "embedding_model", "num_ctx", "reason"): print(f"{key.upper()}={shlex.quote(str(payload[key]))}") PY )" @@ -123,6 +124,7 @@ fi CHROMEAI_OLLAMA_MODEL="$GEN_MODEL" \ CHROMEAI_OLLAMA_EMBED_MODEL="$EMBED_MODEL" \ +CHROMEAI_OLLAMA_NUM_CTX="$NUM_CTX" \ "$ROOT/scripts/install-macos-local.sh" STATUS_LAUNCHAGENT="ok" @@ -140,6 +142,7 @@ Profile: $SELECTED_PROFILE Reason: $REASON Generation model: $GEN_MODEL Embedding model: $EMBED_MODEL +Context window: $NUM_CTX tokens Next recommended command: pnpm benchmark:local diff --git a/scripts/install-macos-local.sh b/scripts/install-macos-local.sh index a3d1317..36c4a67 100755 --- a/scripts/install-macos-local.sh +++ b/scripts/install-macos-local.sh @@ -8,6 +8,7 @@ HASH="$(shasum -a 256 "$ROOT/server/nano_server.py" | awk '{print $1}')" OLLAMA_BASE_URL="${CHROMEAI_OLLAMA_BASE_URL:-http://127.0.0.1:11434}" OLLAMA_MODEL="${CHROMEAI_OLLAMA_MODEL:-gemma4:e2b-it-qat}" OLLAMA_EMBED_MODEL="${CHROMEAI_OLLAMA_EMBED_MODEL:-nomic-embed-text-v2-moe:latest}" +OLLAMA_NUM_CTX="${CHROMEAI_OLLAMA_NUM_CTX:-16384}" RUN_DIR="${CHROMEAI_RUN_DIR:-${HOME}/Library/Application Support/SelectPilot/run}" LOG_DIR="${CHROMEAI_LOG_DIR:-${HOME}/Library/Logs/SelectPilot}" @@ -22,6 +23,7 @@ sed \ -e "s|__OLLAMA_BASE_URL__|$OLLAMA_BASE_URL|g" \ -e "s|__OLLAMA_MODEL__|$OLLAMA_MODEL|g" \ -e "s|__OLLAMA_EMBED_MODEL__|$OLLAMA_EMBED_MODEL|g" \ + -e "s|__OLLAMA_NUM_CTX__|$OLLAMA_NUM_CTX|g" \ "$TEMPLATE" > "$DEST" launchctl unload "$DEST" 2>/dev/null || true @@ -37,7 +39,8 @@ Next steps: 2. Current Ollama base URL: $OLLAMA_BASE_URL 3. Current Ollama model: $OLLAMA_MODEL 4. Local bridge URL: http://127.0.0.1:8083 - 5. Run dir: $RUN_DIR - 6. Log dir: $LOG_DIR - 7. Run 'pnpm benchmark:local' to validate latency on this machine. + 5. Context window: $OLLAMA_NUM_CTX tokens + 6. Run dir: $RUN_DIR + 7. Log dir: $LOG_DIR + 8. Run 'pnpm benchmark:local' to validate latency on this machine. EOF diff --git a/server/ollama_client.py b/server/ollama_client.py index 9d62762..d911be7 100644 --- a/server/ollama_client.py +++ b/server/ollama_client.py @@ -70,6 +70,7 @@ class OllamaConfig: base_url: str model: str embed_model: str + num_ctx: int timeout_seconds: float @@ -77,28 +78,41 @@ class OllamaError(RuntimeError): pass +def _positive_int(value: str | int, name: str) -> int: + try: + parsed = int(value) + except (TypeError, ValueError) as exc: + raise OllamaError(f"{name} must be a positive integer") from exc + if parsed <= 0: + raise OllamaError(f"{name} must be a positive integer") + return parsed + + class OllamaClient: def __init__(self, config: OllamaConfig | None = None): if config is None: default_generation_model = "gemma4:e2b-it-qat" default_embed_model = "nomic-embed-text-v2-moe:latest" + default_num_ctx = 16_384 runtime_profile = os.environ.get("CHROMEAI_RUNTIME_PROFILE", "auto") - try: - from runtime_profiles import get_runtime_profile, recommend_runtime_profile + from runtime_profiles import get_runtime_profile, recommend_runtime_profile - recommendation = recommend_runtime_profile() - resolved_profile = recommendation["recommended_profile"] if runtime_profile == "auto" else runtime_profile - profile = get_runtime_profile(resolved_profile) - default_generation_model = profile.generation_model - default_embed_model = profile.embedding_model - except Exception: - pass + recommendation = recommend_runtime_profile() + resolved_profile = recommendation["recommended_profile"] if runtime_profile == "auto" else runtime_profile + profile = get_runtime_profile(resolved_profile) + default_generation_model = profile.generation_model + default_embed_model = profile.embedding_model + default_num_ctx = profile.num_ctx config = OllamaConfig( base_url=_normalize_base_url(os.environ.get("CHROMEAI_OLLAMA_BASE_URL", "http://127.0.0.1:11434")), model=os.environ.get("CHROMEAI_OLLAMA_MODEL", default_generation_model), embed_model=os.environ.get("CHROMEAI_OLLAMA_EMBED_MODEL", default_embed_model), + num_ctx=_positive_int( + os.environ.get("CHROMEAI_OLLAMA_NUM_CTX", default_num_ctx), + "CHROMEAI_OLLAMA_NUM_CTX", + ), timeout_seconds=float(os.environ.get("CHROMEAI_OLLAMA_TIMEOUT_SECONDS", "30")), ) self.config = config @@ -216,6 +230,12 @@ def active_generation_model(self, models: list[str] | None = None) -> str: def active_embedding_model(self, models: list[str] | None = None) -> str: return self.config.embed_model + def _generation_options(self, temperature: float) -> dict[str, Any]: + return { + "temperature": temperature, + "num_ctx": self.config.num_ctx, + } + def health(self) -> dict[str, Any]: try: all_models = self._model_names() @@ -226,6 +246,7 @@ def health(self) -> dict[str, Any]: "base_url": self.config.base_url, "requested_model": self.config.model, "requested_embed_model": self.config.embed_model, + "num_ctx": self.config.num_ctx, "active_model": self.config.model, "active_embed_model": self.config.embed_model, "timeout_seconds": self.config.timeout_seconds, @@ -249,6 +270,7 @@ def health(self) -> dict[str, Any]: "base_url": self.config.base_url, "requested_model": self.config.model, "requested_embed_model": self.config.embed_model, + "num_ctx": self.config.num_ctx, "active_model": active_model, "active_embed_model": active_embed_model, "timeout_seconds": self.config.timeout_seconds, @@ -304,7 +326,7 @@ def summarize(self, text: str, title: str | None = None, url: str | None = None, "system": "You write precise summaries for selected text in a browser side panel.", "stream": False, "format": schema, - "options": {"temperature": 0.2}, + "options": self._generation_options(0.2), } response = self._request_json("/api/generate", payload) raw_response = str(response.get("response", "")).strip() @@ -367,7 +389,7 @@ def agent(self, prompt: str, context: dict[str, Any] | None = None) -> dict[str, "system": "You are a practical browser copilot that rewrites and structures selected text locally.", "stream": False, "format": schema, - "options": {"temperature": 0.2}, + "options": self._generation_options(0.2), } response = self._request_json("/api/generate", payload) raw_response = str(response.get("response", "")).strip() @@ -475,7 +497,7 @@ def extract( "system": "You generate clean structured extraction results for highlighted browser text.", "stream": False, "format": preset.schema, - "options": {"temperature": 0.1}, + "options": self._generation_options(0.1), } response = self._request_json("/api/generate", payload) raw_response = str(response.get("response", "")).strip() diff --git a/server/runtime_profiles.py b/server/runtime_profiles.py index eb45d02..e08cb72 100644 --- a/server/runtime_profiles.py +++ b/server/runtime_profiles.py @@ -15,6 +15,7 @@ class RuntimeProfile: description: str generation_model: str embedding_model: str + num_ctx: int target_latency: str intended_for: str is_default_auto: bool = False @@ -27,6 +28,7 @@ class RuntimeProfile: description="Memory-safe local profile for structured extraction and low-latency summaries.", generation_model="gemma4:e2b-it-qat", embedding_model="nomic-embed-text-v2-moe:latest", + num_ctx=16_384, target_latency="1-4s", intended_for="Selected-text extraction, action briefs, and quick summaries.", is_default_auto=True, @@ -37,6 +39,7 @@ class RuntimeProfile: description="Efficient local profile for higher-quality browser transforms on 16 GB or more.", generation_model="gemma4:e4b-it-qat", embedding_model="nomic-embed-text-v2-moe:latest", + num_ctx=32_768, target_latency="2-6s", intended_for="Daily use when you want better quality without drifting into heavy models.", ), @@ -46,6 +49,7 @@ class RuntimeProfile: description="Manual opt-in profile for stronger reasoning on larger machines.", generation_model="qwen2.5:7b", embedding_model="nomic-embed-text-v2-moe:latest", + num_ctx=32_768, target_latency="4-10s", intended_for="Heavier rewrite and ask flows when latency budget is less important.", ), @@ -131,4 +135,5 @@ def build_bootstrap_commands(profile_key: str, project_root: str | Path) -> dict "command": command, "generation_model": profile.generation_model, "embedding_model": profile.embedding_model, + "num_ctx": profile.num_ctx, } diff --git a/tests/server/test_runtime_profiles.py b/tests/server/test_runtime_profiles.py index d1a61ec..fe16668 100644 --- a/tests/server/test_runtime_profiles.py +++ b/tests/server/test_runtime_profiles.py @@ -11,7 +11,7 @@ sys.path.insert(0, str(SERVER_DIR)) from runtime_profiles import build_bootstrap_commands, get_runtime_profile, recommend_runtime_profile # noqa: E402 -from ollama_client import OllamaClient # noqa: E402 +from ollama_client import OllamaClient, OllamaConfig, OllamaError # noqa: E402 class RuntimeProfileTests(unittest.TestCase): @@ -19,6 +19,7 @@ def test_unknown_profile_falls_back_to_fast(self) -> None: profile = get_runtime_profile("missing") self.assertEqual(profile.key, "fast") self.assertEqual(profile.generation_model, "gemma4:e2b-it-qat") + self.assertEqual(profile.num_ctx, 16_384) def test_low_memory_hardware_uses_memory_safe_gemma_profile(self) -> None: recommendation = recommend_runtime_profile( @@ -35,6 +36,7 @@ def test_16_gb_apple_silicon_uses_balanced_gemma_profile(self) -> None: profile = get_runtime_profile(recommendation["recommended_profile"]) self.assertEqual(profile.key, "balanced") self.assertEqual(profile.generation_model, "gemma4:e4b-it-qat") + self.assertEqual(profile.num_ctx, 32_768) def test_intel_hardware_remains_on_memory_safe_profile(self) -> None: recommendation = recommend_runtime_profile( @@ -51,12 +53,17 @@ def test_auto_recommendation_prefers_balanced_on_large_machines(self) -> None: def test_bootstrap_command_contains_profile(self) -> None: command = build_bootstrap_commands("balanced", ROOT) self.assertIn("--profile balanced", command["command"]) + self.assertEqual(command["num_ctx"], 32_768) + + def test_advanced_profile_has_explicit_manual_context_window(self) -> None: + self.assertEqual(get_runtime_profile("advanced").num_ctx, 32_768) def test_ollama_client_uses_explicit_profile_without_model_fallback(self) -> None: with patch.dict("os.environ", {"CHROMEAI_RUNTIME_PROFILE": "balanced"}, clear=True): client = OllamaClient() self.assertEqual(client.config.model, "gemma4:e4b-it-qat") + self.assertEqual(client.config.num_ctx, 32_768) self.assertEqual(client.active_generation_model(["qwen2.5:0.5b"]), "gemma4:e4b-it-qat") def test_explicit_model_override_remains_authoritative(self) -> None: @@ -65,12 +72,45 @@ def test_explicit_model_override_remains_authoritative(self) -> None: { "CHROMEAI_RUNTIME_PROFILE": "balanced", "CHROMEAI_OLLAMA_MODEL": "custom-local:model", + "CHROMEAI_OLLAMA_NUM_CTX": "24576", }, clear=True, ): client = OllamaClient() self.assertEqual(client.config.model, "custom-local:model") + self.assertEqual(client.config.num_ctx, 24_576) + + def test_invalid_context_override_fails_explicitly(self) -> None: + with patch.dict("os.environ", {"CHROMEAI_OLLAMA_NUM_CTX": "0"}, clear=True): + with self.assertRaisesRegex(OllamaError, "must be a positive integer"): + OllamaClient() + + def test_every_generation_request_includes_context_window(self) -> None: + client = OllamaClient( + OllamaConfig( + base_url="http://127.0.0.1:11434", + model="test:model", + embed_model="test:embed", + num_ctx=32_768, + timeout_seconds=1, + ) + ) + responses = [ + {"model": "test:model", "response": '{"summary":"S","bullets":[],"action_items":[],"title":"T","tags":[]}'}, + {"model": "test:model", "response": '{"reasoning":[],"markdown":"M","json":{}}'}, + {"model": "test:model", "response": '{}'}, + ] + with patch.object(client, "_model_names", return_value=["test:model"]), patch.object( + client, "_request_json", side_effect=responses + ) as request_json: + client.summarize("Text") + client.agent("Prompt") + client.extract("Text", preset_key="action_brief") + + generation_payloads = [call.args[1] for call in request_json.call_args_list] + self.assertEqual([payload["options"]["num_ctx"] for payload in generation_payloads], [32_768] * 3) + self.assertEqual([payload["options"]["temperature"] for payload in generation_payloads], [0.2, 0.2, 0.1]) if __name__ == "__main__":