Skip to content

Commit 68ee872

Browse files
author
Sebastian Braun
committed
Merge branch 'fix/issue-229-retry-timeout' into integration
2 parents ff54396 + 9180f4d commit 68ee872

2 files changed

Lines changed: 254 additions & 4 deletions

File tree

‎openkb/agent/compiler.py‎

Lines changed: 124 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,71 @@
260260
# ---------------------------------------------------------------------------
261261

262262

263+
def _should_retry_exception(exc: Exception) -> bool:
264+
"""Determine whether an exception is retryable (transient error).
265+
266+
Returns True for temporary API/network errors that may succeed on retry:
267+
- Timeout (client-side or server-side)
268+
- APIError 5xx (server errors)
269+
- RateLimitError (429)
270+
- ConnectionError / ServiceUnavailableError
271+
272+
Returns False for permanent errors that won't be fixed by retry:
273+
- TruncatedResponseError (model hit max_tokens)
274+
- ValueError, TypeError (malformed input/output)
275+
- AuthenticationError (credentials issue)
276+
- BadRequestError (invalid parameters)
277+
- Unknown error types (conservative approach)
278+
"""
279+
exc_type_name = type(exc).__name__
280+
281+
# ===== RETRYABLE (transient errors) =====
282+
283+
# Timeout (network/gateway timeout)
284+
if "Timeout" in exc_type_name:
285+
return True
286+
287+
# Generic API errors (5xx range, but not 4xx)
288+
if "APIError" in exc_type_name:
289+
# Don't retry if it's a BadRequest/Invalid error (4xx)
290+
if "Invalid" not in exc_type_name and "BadRequest" not in exc_type_name:
291+
return True
292+
293+
# Rate limiting (429)
294+
if "RateLimitError" in exc_type_name or "Rate" in exc_type_name:
295+
return True
296+
297+
# Connection errors
298+
if "ConnectionError" in exc_type_name:
299+
return True
300+
301+
# Service unavailable
302+
if "ServiceUnavailable" in exc_type_name:
303+
return True
304+
305+
# ===== NOT RETRYABLE (permanent errors) =====
306+
307+
# Model hit max_tokens limit
308+
if isinstance(exc, TruncatedResponseError):
309+
return False
310+
311+
# Content validation failures
312+
if "ValueError" in exc_type_name or "TypeError" in exc_type_name:
313+
return False
314+
315+
# Authentication failures
316+
if "Auth" in exc_type_name or "Permission" in exc_type_name:
317+
return False
318+
319+
# Bad parameters/requests
320+
if "BadRequest" in exc_type_name or "Invalid" in exc_type_name:
321+
return False
322+
323+
# ===== UNKNOWN: Conservative approach =====
324+
# Don't retry errors we don't recognize
325+
return False
326+
327+
263328
def _cached_text(text: str) -> list[dict]:
264329
"""Wrap a text payload into a content-block list with an Anthropic
265330
ephemeral cache_control marker.
@@ -406,7 +471,11 @@ def _llm_call(
406471
bundle=None,
407472
**kwargs,
408473
) -> str:
409-
"""Single LLM call with animated progress and debug logging."""
474+
"""Single LLM call with animated progress, debug logging, and retry support.
475+
476+
Transient errors (Timeout, 5xx, 429) are automatically retried by LiteLLM.
477+
Permanent errors (4xx, truncation, validation) are raised immediately.
478+
"""
410479
messages = _prepare_messages(model, messages)
411480
extra_headers = bundle.extra_headers if bundle is not None else get_extra_headers()
412481
if extra_headers:
@@ -417,6 +486,10 @@ def _llm_call(
417486
if bundle is not None:
418487
kwargs.setdefault("api_key", bundle.api_key)
419488
kwargs.setdefault("base_url", bundle.base_url)
489+
490+
# NEW: Retry configuration for transient errors (fixed: 2 retries, base 2)
491+
kwargs.setdefault("retries", 2)
492+
420493
logger.debug("LLM request [%s]:\n%s", step_name, _fmt_messages(messages))
421494
if kwargs:
422495
logger.debug("LLM kwargs [%s]: %s", step_name, kwargs)
@@ -425,7 +498,27 @@ def _llm_call(
425498
spinner.start()
426499
t0 = time.time()
427500

428-
response = litellm.completion(model=model, messages=messages, **kwargs)
501+
try:
502+
response = litellm.completion(model=model, messages=messages, **kwargs)
503+
except Exception as exc:
504+
# NEW: Better error logging with retry context
505+
if _should_retry_exception(exc):
506+
logger.warning(
507+
"LLM [%s] failed with transient error (retries applied by LiteLLM): %s",
508+
step_name,
509+
exc,
510+
exc_info=False, # Don't spam stack traces for known transient errors
511+
)
512+
else:
513+
logger.warning(
514+
"LLM [%s] failed with permanent error (no retry): %s",
515+
step_name,
516+
exc,
517+
exc_info=True, # Full trace for unexpected errors
518+
)
519+
spinner.stop("[FAILED]")
520+
raise
521+
429522
content = response.choices[0].message.content or ""
430523
truncated = _warn_if_truncated(response, step_name, kwargs.get("max_tokens"))
431524

@@ -449,7 +542,11 @@ async def _llm_call_async(
449542
bundle=None,
450543
**kwargs,
451544
) -> str:
452-
"""Async LLM call with timing output and debug logging."""
545+
"""Async LLM call with timing output, debug logging, and retry support.
546+
547+
Transient errors (Timeout, 5xx, 429) are automatically retried by LiteLLM.
548+
Permanent errors (4xx, truncation, validation) are raised immediately.
549+
"""
453550
messages = _prepare_messages(model, messages)
454551
extra_headers = bundle.extra_headers if bundle is not None else get_extra_headers()
455552
if extra_headers:
@@ -460,13 +557,36 @@ async def _llm_call_async(
460557
if bundle is not None:
461558
kwargs.setdefault("api_key", bundle.api_key)
462559
kwargs.setdefault("base_url", bundle.base_url)
560+
561+
# NEW: Retry configuration for transient errors (fixed: 2 retries, base 2)
562+
kwargs.setdefault("retries", 2)
563+
463564
logger.debug("LLM request [%s]:\n%s", step_name, _fmt_messages(messages))
464565
if kwargs:
465566
logger.debug("LLM kwargs [%s]: %s", step_name, kwargs)
466567

467568
t0 = time.time()
468569

469-
response = await litellm.acompletion(model=model, messages=messages, **kwargs)
570+
try:
571+
response = await litellm.acompletion(model=model, messages=messages, **kwargs)
572+
except Exception as exc:
573+
# NEW: Better error logging with retry context
574+
if _should_retry_exception(exc):
575+
logger.warning(
576+
"LLM [%s] failed with transient error (retries applied by LiteLLM): %s",
577+
step_name,
578+
exc,
579+
exc_info=False,
580+
)
581+
else:
582+
logger.warning(
583+
"LLM [%s] failed with permanent error (no retry): %s",
584+
step_name,
585+
exc,
586+
exc_info=True,
587+
)
588+
raise
589+
470590
content = response.choices[0].message.content or ""
471591
truncated = _warn_if_truncated(response, step_name, kwargs.get("max_tokens"))
472592

‎tests/test_compiler_retry.py‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
"""Tests for LLM retry logic in compiler.py."""
2+
3+
from openkb.agent.compiler import TruncatedResponseError, _should_retry_exception
4+
5+
6+
# Custom exception classes for testing (so we can control the type name)
7+
class TimeoutError(Exception):
8+
"""Simulates litellm.Timeout."""
9+
10+
pass
11+
12+
13+
class APIError(Exception):
14+
"""Simulates litellm.APIError (5xx)."""
15+
16+
pass
17+
18+
19+
class InvalidAPIError(APIError):
20+
"""Simulates InvalidAPIError (4xx)."""
21+
22+
pass
23+
24+
25+
class BadRequestError(Exception):
26+
"""Simulates BadRequestError."""
27+
28+
pass
29+
30+
31+
class RateLimitError(Exception):
32+
"""Simulates litellm.RateLimitError."""
33+
34+
pass
35+
36+
37+
class AuthenticationError(Exception):
38+
"""Simulates AuthenticationError."""
39+
40+
pass
41+
42+
43+
class PermissionError(Exception):
44+
"""Simulates PermissionError."""
45+
46+
pass
47+
48+
49+
class ServiceUnavailableError(Exception):
50+
"""Simulates ServiceUnavailableError."""
51+
52+
pass
53+
54+
55+
class TestShouldRetryException:
56+
"""Test the exception filtering logic for retry decisions."""
57+
58+
def test_retryable_timeout(self):
59+
"""Timeout should be retryable."""
60+
exc = TimeoutError("Gateway Timeout")
61+
assert _should_retry_exception(exc) is True
62+
63+
def test_retryable_api_error_5xx(self):
64+
"""5xx API errors should be retryable."""
65+
exc = APIError("503 Service Unavailable")
66+
assert _should_retry_exception(exc) is True
67+
68+
def test_not_retryable_invalid_api_error(self):
69+
"""InvalidAPIError (4xx) should NOT be retryable."""
70+
exc = InvalidAPIError("400 Bad Request")
71+
assert _should_retry_exception(exc) is False
72+
73+
def test_retryable_rate_limit(self):
74+
"""Rate limit errors should be retryable."""
75+
exc = RateLimitError("429 Too Many Requests")
76+
assert _should_retry_exception(exc) is True
77+
78+
def test_retryable_connection_error(self):
79+
"""Connection errors should be retryable."""
80+
exc = ConnectionError("Connection refused")
81+
assert _should_retry_exception(exc) is True
82+
83+
def test_retryable_service_unavailable(self):
84+
"""Service unavailable errors should be retryable."""
85+
exc = ServiceUnavailableError("Service down")
86+
assert _should_retry_exception(exc) is True
87+
88+
def test_not_retryable_truncation(self):
89+
"""Truncated output should NOT be retryable."""
90+
exc = TruncatedResponseError("hit length limit")
91+
assert _should_retry_exception(exc) is False
92+
93+
def test_not_retryable_value_error(self):
94+
"""ValueError should NOT be retryable."""
95+
exc = ValueError("empty content")
96+
assert _should_retry_exception(exc) is False
97+
98+
def test_not_retryable_type_error(self):
99+
"""TypeError should NOT be retryable."""
100+
exc = TypeError("malformed")
101+
assert _should_retry_exception(exc) is False
102+
103+
def test_not_retryable_auth_error(self):
104+
"""Authentication errors should NOT be retryable."""
105+
exc = AuthenticationError("invalid API key")
106+
assert _should_retry_exception(exc) is False
107+
108+
def test_not_retryable_permission_error(self):
109+
"""Permission errors should NOT be retryable."""
110+
exc = PermissionError("forbidden")
111+
assert _should_retry_exception(exc) is False
112+
113+
def test_not_retryable_bad_request(self):
114+
"""BadRequest errors should NOT be retryable."""
115+
exc = BadRequestError("invalid params")
116+
assert _should_retry_exception(exc) is False
117+
118+
def test_not_retryable_unknown(self):
119+
"""Unknown errors should NOT be retried (conservative)."""
120+
121+
class WeirdCustomError(Exception):
122+
pass
123+
124+
exc = WeirdCustomError("something weird")
125+
assert _should_retry_exception(exc) is False
126+
127+
def test_not_retryable_generic_exception(self):
128+
"""Generic Exception without special name should NOT be retried."""
129+
exc = Exception("generic error")
130+
assert _should_retry_exception(exc) is False

0 commit comments

Comments
 (0)