diff --git a/docs/docs/api/modules/RLM.md b/docs/docs/api/modules/RLM.md index cb5033a309..cf2250b3e6 100644 --- a/docs/docs/api/modules/RLM.md +++ b/docs/docs/api/modules/RLM.md @@ -36,10 +36,12 @@ print(result.answer) RLM relies on [Deno](https://deno.land/) and [Pyodide](https://pyodide.org/) to create a local WASM sandbox for secure Python execution. -You can install Deno with: `curl -fsSL https://deno.land/install.sh | sh` on macOS and Linux. See the [Deno Installation Docs](https://docs.deno.com/runtime/getting_started/installation/) for more details. Make sure to accept the prompt when it asks to add it to your shell profile. +You can install Deno with: `brew install deno` on MacOS or `curl -fsSL https://deno.land/install.sh | sh` on MacOS and Linux. See the [Deno Installation Docs](https://docs.deno.com/runtime/getting_started/installation/) for more details. Make sure to accept the prompt when it asks to add it to your shell profile. After you have installed Deno, **Make sure to restart your shell.** +Deno may get confused by existing `package.json` files it happens to find; use the environment variable `DENO_NO_PACKAGE_JSON=1` to ignore `package.json` entirely and resolve `npm:pyodide` from its own cache, which is what DSPy expects. + Then you can run `dspy.RLM`. Users have reported issues with the Deno cache not being found by DSPy. We are actively investigating these issues, and your feedback is greatly appreciated. diff --git a/dspy/clients/openai.py b/dspy/clients/openai.py index f687249d76..065abd8ee3 100644 --- a/dspy/clients/openai.py +++ b/dspy/clients/openai.py @@ -100,23 +100,23 @@ def finetune( return model @staticmethod - def does_job_exist(job_id: str) -> bool: + def does_job_exist(job_id: str | None) -> bool: + if job_id is None: + return False try: - # TODO(nit): This call may fail for other reasons. We should check - # the error message to ensure that the job does not exist. openai.fine_tuning.jobs.retrieve(job_id) return True - except Exception: + except openai.NotFoundError: return False @staticmethod - def does_file_exist(file_id: str) -> bool: + def does_file_exist(file_id: str | None) -> bool: + if file_id is None: + return False try: - # TODO(nit): This call may fail for other reasons. We should check - # the error message to ensure that the file does not exist. openai.files.retrieve(file_id) return True - except Exception: + except openai.NotFoundError: return False @staticmethod diff --git a/dspy/primitives/python_interpreter.py b/dspy/primitives/python_interpreter.py index b700dfffc3..22ad6ce330 100644 --- a/dspy/primitives/python_interpreter.py +++ b/dspy/primitives/python_interpreter.py @@ -12,6 +12,7 @@ import json import keyword import logging +import math import os import subprocess import threading @@ -480,6 +481,10 @@ def _serialize_value(self, value: Any) -> str: elif isinstance(value, bool): # Must check bool before int since bool is a subclass of int return "True" if value else "False" + elif isinstance(value, float) and not math.isfinite(value): + # str(inf/-inf/nan) returns bare words ("inf", "nan") that are not valid + # Python literals; wrap them so they evaluate correctly in the sandbox. + return f"float('{value}')" elif isinstance(value, (int, float)): return str(value) elif isinstance(value, (list, tuple)): diff --git a/dspy/teleprompt/utils.py b/dspy/teleprompt/utils.py index 9fe6a1eaef..3cdfb63d3f 100644 --- a/dspy/teleprompt/utils.py +++ b/dspy/teleprompt/utils.py @@ -137,8 +137,7 @@ def get_program_with_highest_avg_score(param_score_dict, fully_evaled_param_comb return program, mean, key, params - # If no valid program is found, we return the last valid one that we found - return program, mean, key, params + raise ValueError("No valid program found in param_score_dict") def calculate_last_n_proposed_quality( diff --git a/pyproject.toml b/pyproject.toml index f733e0bda7..8e5d893feb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,13 +21,13 @@ classifiers = [ "Programming Language :: Python :: 3" ] dependencies = [ - "openai>=0.28.1", + "openai>=1.66.2", "regex>=2023.10.3", "orjson>=3.9.0", "tqdm>=4.66.1", "requests>=2.31.0", "pydantic>=2.0", - "litellm>=1.64.0", + "litellm>=1.65.8", "diskcache>=5.6.0", "json-repair>=0.54.2", "tenacity>=8.2.3", # undeclared runtime dep of litellm, needed for exponential_backoff_retry @@ -44,7 +44,7 @@ mcp = ["mcp; python_version >= '3.10'"] langchain = ["langchain_core"] optuna = ["optuna>=3.4.0"] numpy = ["numpy>=1.26.0"] -litellm = ["litellm>=1.64.0"] +litellm = ["litellm>=1.65.8"] dev = [ "pytest>=6.2.5", "pytest-mock>=3.12.0", @@ -55,8 +55,8 @@ dev = [ "datamodel_code_generator>=0.26.3", "build>=1.0.3", "numpy>=1.26.0", - "litellm>=1.64.0; sys_platform == 'win32' or python_version == '3.14'", - "litellm[proxy]>=1.64.0; sys_platform != 'win32' and python_version < '3.14'", # Remove 3.14 condition once uvloop supports + "litellm>=1.65.8; sys_platform == 'win32' or python_version == '3.14'", + "litellm[proxy]>=1.65.8; sys_platform != 'win32' and python_version < '3.14'", # Remove 3.14 condition once uvloop supports ] test_extras = [ "mcp; python_version >= '3.10'", diff --git a/tests/clients/test_lm.py b/tests/clients/test_lm.py index c3d7ee2b11..db70b379fd 100644 --- a/tests/clients/test_lm.py +++ b/tests/clients/test_lm.py @@ -396,6 +396,11 @@ def test_reasoning_model_token_parameter(): assert lm.kwargs["max_tokens"] == 1000 +def test_lm_supports_reasoning_with_litellm_capability_api(): + lm = dspy.LM("anthropic/claude-3-7-sonnet-20250219") + assert lm.supports_reasoning is True + + @pytest.mark.parametrize("model_name", ["openai/o1", "openai/gpt-5-nano", "openai/gpt-5-mini"]) def test_reasoning_model_requirements(model_name): # Should raise assertion error if temperature or max_tokens requirements not met diff --git a/tests/clients/test_openai_provider.py b/tests/clients/test_openai_provider.py new file mode 100644 index 0000000000..677c5cd6f4 --- /dev/null +++ b/tests/clients/test_openai_provider.py @@ -0,0 +1,41 @@ +from types import SimpleNamespace +from unittest.mock import Mock + +import httpx +import openai +import pytest + +from dspy.clients.openai import OpenAIProvider + + +@pytest.mark.parametrize( + ("method_name", "resource_name"), + [ + ("does_job_exist", "fine_tuning"), + ("does_file_exist", "files"), + ], +) +def test_provider_existence_checks_only_handle_not_found(monkeypatch, method_name, resource_name): + retrieve = Mock() + resource = SimpleNamespace(retrieve=retrieve) + if resource_name == "fine_tuning": + resource = SimpleNamespace(jobs=resource) + monkeypatch.setitem(openai.__dict__, resource_name, resource) + exists = getattr(OpenAIProvider, method_name) + + assert exists("resource-id") is True + retrieve.assert_called_once_with("resource-id") + + response = httpx.Response(404, request=httpx.Request("GET", "https://api.openai.com/resource")) + retrieve.side_effect = openai.NotFoundError("not found", response=response, body=None) + assert exists("missing-id") is False + + response = httpx.Response(401, request=httpx.Request("GET", "https://api.openai.com/resource")) + retrieve.side_effect = openai.AuthenticationError("unauthorized", response=response, body=None) + with pytest.raises(openai.AuthenticationError): + exists("private-id") + + retrieve.reset_mock() + retrieve.side_effect = None + assert exists(None) is False + retrieve.assert_not_called() diff --git a/tests/primitives/test_python_interpreter.py b/tests/primitives/test_python_interpreter.py index 3e523afc0c..15d1ae3dc0 100644 --- a/tests/primitives/test_python_interpreter.py +++ b/tests/primitives/test_python_interpreter.py @@ -31,6 +31,23 @@ def test_user_variable_definitions(): assert result == 5, "User variable assignment should work" +def test_non_finite_float_variables(): + """Regression test: inf/-inf/nan variables must be injected as valid Python literals. + + str(float("inf")) is the bare word "inf", which is not a valid Python name, so + injecting it as `x = inf` previously raised NameError in the sandbox. + """ + with PythonInterpreter() as interpreter: + inf_code = "result = 1 if x == float('inf') else 0\nresult" + assert interpreter.execute(inf_code, variables={"x": float("inf")}) == 1 + + neg_inf_code = "result = 1 if x == float('-inf') else 0\nresult" + assert interpreter.execute(neg_inf_code, variables={"x": float("-inf")}) == 1 + + nan_code = "import math\nresult = 1 if math.isnan(x) else 0\nresult" + assert interpreter.execute(nan_code, variables={"x": float("nan")}) == 1 + + def test_rejects_python_keywords_as_variable_names(): """Test that Python keywords are rejected as variable names.""" with PythonInterpreter() as interpreter: diff --git a/tests/utils/resources/mcp_server.py b/tests/utils/resources/mcp_server.py index 0361262112..c40ad737de 100644 --- a/tests/utils/resources/mcp_server.py +++ b/tests/utils/resources/mcp_server.py @@ -21,7 +21,7 @@ def add(a: int, b: int) -> int: @mcp.tool() -def hello(names: list[str]) -> str: +def hello(names: list[str]) -> list[str]: """Greet people""" return [f"Hello, {name}!" for name in names] diff --git a/uv.lock b/uv.lock index 485357e162..1e2598185b 100644 --- a/uv.lock +++ b/uv.lock @@ -846,16 +846,16 @@ requires-dist = [ { name = "json-repair", specifier = ">=0.54.2" }, { name = "langchain-core", marker = "extra == 'langchain'" }, { name = "langchain-core", marker = "extra == 'test-extras'" }, - { name = "litellm", specifier = ">=1.64.0" }, - { name = "litellm", marker = "(python_full_version == '3.14.*' and extra == 'dev') or (sys_platform == 'win32' and extra == 'dev')", specifier = ">=1.64.0" }, - { name = "litellm", marker = "extra == 'litellm'", specifier = ">=1.64.0" }, - { name = "litellm", extras = ["proxy"], marker = "python_full_version < '3.14' and sys_platform != 'win32' and extra == 'dev'", specifier = ">=1.64.0" }, + { name = "litellm", specifier = ">=1.65.8" }, + { name = "litellm", marker = "(python_full_version == '3.14.*' and extra == 'dev') or (sys_platform == 'win32' and extra == 'dev')", specifier = ">=1.65.8" }, + { name = "litellm", marker = "extra == 'litellm'", specifier = ">=1.65.8" }, + { name = "litellm", extras = ["proxy"], marker = "python_full_version < '3.14' and sys_platform != 'win32' and extra == 'dev'", specifier = ">=1.65.8" }, { name = "mcp", marker = "python_full_version >= '3.10' and extra == 'mcp'" }, { name = "mcp", marker = "python_full_version >= '3.10' and extra == 'test-extras'" }, { name = "numpy", marker = "extra == 'dev'", specifier = ">=1.26.0" }, { name = "numpy", marker = "extra == 'numpy'", specifier = ">=1.26.0" }, { name = "numpy", marker = "extra == 'test-extras'", specifier = ">=1.26.0" }, - { name = "openai", specifier = ">=0.28.1" }, + { name = "openai", specifier = ">=1.66.2" }, { name = "optuna", marker = "extra == 'optuna'", specifier = ">=3.4.0" }, { name = "optuna", marker = "extra == 'test-extras'", specifier = ">=3.4.0" }, { name = "orjson", specifier = ">=3.9.0" },