Skip to content

Commit d5bc863

Browse files
author
Sebastian Braun
committed
fix(agent): retry concepts-plan with summary as source before giving up
1 parent 338d1bb commit d5bc863

2 files changed

Lines changed: 112 additions & 25 deletions

File tree

‎openkb/agent/compiler.py‎

Lines changed: 41 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1892,21 +1892,17 @@ def _write_v1_summary_stripped() -> None:
18921892
)
18931893
_write_summary(wiki_dir, doc_name, cleaned, description=doc_brief)
18941894

1895+
concepts_plan_user_msg = {
1896+
"role": "user",
1897+
"content": _CONCEPTS_PLAN_USER.format(
1898+
concept_briefs=concept_briefs,
1899+
entity_briefs=entity_briefs,
1900+
).replace("__ENTITY_TYPES__", types_str),
1901+
}
18951902
try:
18961903
plan_raw = _llm_call(
18971904
model,
1898-
[
1899-
system_msg,
1900-
doc_msg,
1901-
summary_msg,
1902-
{
1903-
"role": "user",
1904-
"content": _CONCEPTS_PLAN_USER.format(
1905-
concept_briefs=concept_briefs,
1906-
entity_briefs=entity_briefs,
1907-
).replace("__ENTITY_TYPES__", types_str),
1908-
},
1909-
],
1905+
[system_msg, doc_msg, summary_msg, concepts_plan_user_msg],
19101906
"concepts-plan",
19111907
response_format=_JSON_RESPONSE_FORMAT,
19121908
bundle=bundle,
@@ -1916,14 +1912,39 @@ def _write_v1_summary_stripped() -> None:
19161912
# _read_concept_briefs/_read_entity_briefs) and is added on top of the
19171913
# already-cached document — a doc that was fine for the summary call
19181914
# can still blow the window here once the index gets large enough.
1919-
# The summary itself already exists (unlike the doc-too-large case
1920-
# above), so it's kept — same fallback as an unparseable/empty plan,
1921-
# just skipping concept/entity generation for this doc.
1922-
logger.warning("Skipping concept/entity extraction for %s: %s", doc_name, exc)
1923-
if rewrite_summary:
1924-
_write_v1_summary_stripped()
1925-
_update_index(wiki_dir, doc_name, [], doc_brief=doc_brief, doc_type=doc_type)
1926-
return
1915+
# Retry once without the full document: the plan prompt already asks
1916+
# the model to work "based on the summary above" (see
1917+
# _CONCEPTS_PLAN_USER), so dropping doc_msg usually shrinks the
1918+
# prompt back under the window without losing the plan's intent.
1919+
logger.warning(
1920+
"concepts plan exceeded context window for %s: %s; retrying with summary as source",
1921+
doc_name,
1922+
exc,
1923+
)
1924+
sys.stdout.write(
1925+
f" [WARN] concepts plan exceeded context window for {doc_name} — "
1926+
"retrying with the summary as source instead of the full document.\n"
1927+
)
1928+
sys.stdout.flush()
1929+
try:
1930+
plan_raw = _llm_call(
1931+
model,
1932+
[system_msg, summary_msg, concepts_plan_user_msg],
1933+
"concepts-plan",
1934+
response_format=_JSON_RESPONSE_FORMAT,
1935+
bundle=bundle,
1936+
)
1937+
except _NON_RETRYABLE_LLM_ERRORS as retry_exc:
1938+
# Still too large even with just the summary (e.g. the existing
1939+
# concept/entity index alone is huge) — the summary itself
1940+
# already exists (unlike the doc-too-large case above), so it's
1941+
# kept — same fallback as an unparseable/empty plan, just
1942+
# skipping concept/entity generation for this doc.
1943+
logger.warning("Skipping concept/entity extraction for %s: %s", doc_name, retry_exc)
1944+
if rewrite_summary:
1945+
_write_v1_summary_stripped()
1946+
_update_index(wiki_dir, doc_name, [], doc_brief=doc_brief, doc_type=doc_type)
1947+
return
19271948

19281949
try:
19291950
parsed = _parse_json(plan_raw)

‎tests/test_compiler.py‎

Lines changed: 71 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1508,9 +1508,11 @@ async def test_concepts_plan_context_window_exceeded_keeps_real_summary(self, tm
15081508
"""The summary call already succeeded with real content by the time
15091509
concepts-plan runs (unlike TestOversizedDocumentSkip, where the doc
15101510
itself is too large) — this must NOT be replaced with a generic
1511-
"too large" stub. Same fallback as an unparseable/empty plan: keep
1512-
the real v1 summary (ghost-stripped), index it, skip concept/entity
1513-
generation for this doc."""
1511+
"too large" stub. The concepts-plan call is retried once with the
1512+
summary as source (doc_msg dropped) before giving up; if that retry
1513+
ALSO hits the context window, same fallback as an unparseable/empty
1514+
plan: keep the real v1 summary (ghost-stripped), index it, skip
1515+
concept/entity generation for this doc."""
15141516
wiki, source_path = self._setup_kb(tmp_path)
15151517

15161518
v1_summary_content = "# Summary\n\nDiscusses [[concepts/nonexistent]] here."
@@ -1535,8 +1537,15 @@ def side_effect(*args, **kwargs):
15351537
mock_litellm.completion = MagicMock(side_effect=side_effect)
15361538
# Must not raise out of compile_short_doc.
15371539
await compile_short_doc("doc", source_path, tmp_path, "claude-sonnet-4-5")
1538-
# summary + concepts-plan, no wasted retries on the doomed request.
1539-
assert mock_litellm.completion.call_count == 2
1540+
# summary + concepts-plan (full doc) + concepts-plan retry (summary
1541+
# only) — both plan attempts are doomed here, no further retries.
1542+
assert mock_litellm.completion.call_count == 3
1543+
1544+
# The retry attempt must not still include the full document message
1545+
# (_SUMMARY_USER's "Full text:" marker only ever appears in doc_msg).
1546+
retry_messages = mock_litellm.completion.call_args_list[2].kwargs["messages"]
1547+
retry_text = json.dumps(retry_messages)
1548+
assert "Full text:" not in retry_text
15401549

15411550
summary_path = wiki / "summaries" / "doc.md"
15421551
assert summary_path.exists()
@@ -1550,6 +1559,63 @@ def side_effect(*args, **kwargs):
15501559
assert "[[summaries/doc]]" in index_text
15511560
assert not list((wiki / "concepts").glob("*.md"))
15521561

1562+
@pytest.mark.asyncio
1563+
async def test_concepts_plan_context_window_retry_succeeds_with_summary_only(self, tmp_path):
1564+
"""When the retry (summary-only) succeeds, the plan is processed
1565+
normally — concepts are generated from the summary-derived plan
1566+
instead of the doc being treated as incomplete."""
1567+
wiki, source_path = self._setup_kb(tmp_path)
1568+
1569+
v1_summary_content = "# Summary\n\nDiscusses transformers."
1570+
summary_response = json.dumps(
1571+
{"description": "A real summary", "content": v1_summary_content}
1572+
)
1573+
error = litellm.ContextWindowExceededError(
1574+
message="prompt is too long: 220670 tokens > 200000 maximum",
1575+
model="claude-sonnet-4-5",
1576+
llm_provider="anthropic",
1577+
)
1578+
plan_response = json.dumps(
1579+
{
1580+
"create": [{"name": "transformer", "title": "Transformer"}],
1581+
"update": [],
1582+
"related": [],
1583+
}
1584+
)
1585+
concept_response = json.dumps({"description": "C", "content": "# T\n\nBody."})
1586+
call_count = {"n": 0}
1587+
1588+
def side_effect(*args, **kwargs):
1589+
idx = call_count["n"]
1590+
call_count["n"] += 1
1591+
if idx == 0:
1592+
return [_mock_response(summary_response)]
1593+
if idx == 1:
1594+
raise error
1595+
return [_mock_response(plan_response)]
1596+
1597+
with (
1598+
patch("openkb.agent.compiler.litellm") as mock_litellm,
1599+
):
1600+
mock_litellm.completion = MagicMock(side_effect=side_effect)
1601+
mock_litellm.acompletion = AsyncMock(side_effect=_mock_acompletion([concept_response]))
1602+
await compile_short_doc("doc", source_path, tmp_path, "claude-sonnet-4-5")
1603+
# summary + concepts-plan (full doc, fails) + concepts-plan retry
1604+
# (summary only, succeeds) + summary-rewrite (rewrite_summary=True).
1605+
assert mock_litellm.completion.call_count == 4
1606+
1607+
# The successful retry's messages must not include the full document
1608+
# message (_SUMMARY_USER's "Full text:" marker only ever appears in doc_msg).
1609+
retry_messages = mock_litellm.completion.call_args_list[2].kwargs["messages"]
1610+
retry_text = json.dumps(retry_messages)
1611+
assert "Full text:" not in retry_text
1612+
1613+
concept_path = wiki / "concepts" / "transformer.md"
1614+
assert concept_path.exists()
1615+
1616+
index_text = (wiki / "index.md").read_text()
1617+
assert "[[concepts/transformer]]" in index_text
1618+
15531619

15541620
class TestOversizedDocumentSkip:
15551621
"""A document whose content can't fit an LLM call is treated like an

0 commit comments

Comments
 (0)