From 67f44b9bffb47522fdf94659c5ab3d4262d2ae51 Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Tue, 1 Sep 2026 17:34:19 +0800 Subject: [PATCH] refactor: remove unused character limit from navigation configuration This commit removes the `filter_submap_char_limit` parameter from the navigation configuration and related functions, streamlining the retrieval process. Additionally, updates to the `render_submap_observation` function reflect this change by eliminating the character limit logic, ensuring that all matched nodes are displayed without truncation. Tests have been adjusted accordingly to validate the new behavior. --- .../services/retrieval/nav/nav_node_filter.py | 17 +- .../shared/services/retrieval/nav/nav_plan.py | 6 +- .../retrieval/nav/nav_scope_filter.py | 157 +++++++++++------- .../services/retrieval/nav/nav_types.py | 1 - .../shared/services/retrieval/nav_config.py | 1 - .../shared/services/retrieval/trace/mapnav.py | 7 +- .../shared/tests/test_nav_node_filter.py | 5 +- .../shared/tests/test_nav_scope_filter.py | 33 +++- .../shared/tests/test_nav_trace_map.py | 4 +- 9 files changed, 140 insertions(+), 91 deletions(-) diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_node_filter.py b/packages/shared-python/shared/services/retrieval/nav/nav_node_filter.py index 19dc1d8a..9de89760 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_node_filter.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_node_filter.py @@ -126,12 +126,10 @@ def render_submap_observation( ts: Any, result: FilterResult, *, - char_limit: int, doc_ids: Sequence[str] | None = None, ) -> str: - """Hit-count line plus a budgeted preview of matched nodes.""" + """Hit-count line plus every matched node (path + summary).""" del doc_ids - limit = max(0, int(char_limit)) header = f"hits={result.cardinality}" if result.truncated: header = f"{header} truncated=true" @@ -142,8 +140,6 @@ def render_submap_observation( summaries = _load_summaries(ts) lines = [header] - used = len(header) + 1 - shown = 0 for sid in result.matched_section_ids: owner = _owner_document(ts, sid) title = _path_text(ts, sid, owner) or sid @@ -151,16 +147,7 @@ def render_submap_observation( summary = str(summaries.get(sid) or "").strip() if summary: block.append(f" summary: {summary}") - chunk = "\n".join(block) - extra = len(chunk) + (1 if lines else 0) - if limit and used + extra > limit: - lines.append( - f"preview truncated after {shown} nodes; tighten the predicate" - ) - break - lines.append(chunk) - used += extra - shown += 1 + lines.append("\n".join(block)) return "\n".join(lines) diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_plan.py b/packages/shared-python/shared/services/retrieval/nav/nav_plan.py index 2e8a2dff..8e643a6e 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_plan.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_plan.py @@ -653,11 +653,11 @@ def _planner_system_prompt(*, max_subgoals: int) -> str: "10. reason must be English, under 40 words. Document titles stay original " "language.\n" "11. Set use_node_filter=true when the subgoal enumerates or compares " - "named facets you can write as path/summary predicates (filenames, " - "tickers, section titles). Keep it false for vague semantic needs; " + "named facets you can write as path predicates (filenames, section " + "titles). Keep it false for vague semantic needs; " "retrieval_query remains the fuzzy-leg fallback. Optional node_filter " "may seed predicates: " - '[{\"field\":\"path|summary\",\"terms\":[\"...\"],\"match\":\"substring|regex\"}].\n\n' + '[{\"field\":\"path\",\"terms\":[\"...\"],\"match\":\"substring|regex\"}].\n\n' "Return ONLY one JSON object:\n" "{\n" ' "reason": "...",\n' diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_scope_filter.py b/packages/shared-python/shared/services/retrieval/nav/nav_scope_filter.py index 571c2276..c0be92e9 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_scope_filter.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_scope_filter.py @@ -7,6 +7,7 @@ from __future__ import annotations +import json from dataclasses import dataclass, field from typing import Any, Dict, List, Literal, Optional, Sequence @@ -86,9 +87,8 @@ def run_scope_filter( max_rounds = max(1, int(getattr(config, "filter_max_rounds", 3) or 3)) min_hits = max(0, int(getattr(config, "filter_min_hits", 1) or 0)) max_hits = max(min_hits, int(getattr(config, "filter_max_hits", 40) or 0)) - char_limit = max(0, int(getattr(config, "filter_submap_char_limit", 2000) or 0)) wanted = [str(did).strip() for did in doc_ids if str(did).strip()] - map_text = str(map_observation or "").strip() or _compact_map(ts, wanted, char_limit) + map_text = str(map_observation or "").strip() or _compact_map(ts, wanted) current = seed_filter last_result: Optional[FilterResult] = None last_obs = "" @@ -99,11 +99,13 @@ def run_scope_filter( action = _scope_filter_policy_call( config, query=query, - map_observation=map_text, + full_map=map_text, + last_filter=None, last_result=None, last_observation="", round_idx=0, max_rounds=max_rounds, + include_full_map=True, ) current = action.get("filter") last_decision = action.get("decision") @@ -129,9 +131,7 @@ def run_scope_filter( assert current is not None result = apply_node_filter(ts, wanted, current) last_result = result - last_obs = render_submap_observation( - ts, result, char_limit=char_limit, doc_ids=wanted - ) + last_obs = render_submap_observation(ts, result, doc_ids=wanted) in_band = min_hits <= result.cardinality <= max_hits if steps_out is not None: from ._compat import AgentStep @@ -151,8 +151,9 @@ def run_scope_filter( "truncated": result.truncated, "failed_predicates": list(result.failed_predicates), "matched_section_ids": list(result.matched_section_ids), + "action": "", "decision": "", - "reason": last_reason, + "reason": "", } ), ) @@ -160,32 +161,58 @@ def run_scope_filter( is_last = round_idx >= max_rounds if is_last: + if steps_out: + steps_out[-1].detail["action"] = "max_rounds" return _settle( result, in_band=in_band, agent_decision=last_decision, min_hits=min_hits, rounds=round_idx, - reason=last_reason or ("max_rounds" if in_band else "max_rounds_out_of_band"), + reason=("max_rounds" if in_band else "max_rounds_out_of_band"), steps_out=steps_out, ) action = _scope_filter_policy_call( config, query=query, - map_observation=map_text, + full_map=map_text, + last_filter=current, last_result=result, last_observation=last_obs, round_idx=round_idx, max_rounds=max_rounds, + include_full_map=False, ) + if action.get("kind") == "widen": + # Agent judged the sub-map too narrow: one re-look with the full + # map, same round. A second widen settles on whatever it returns. + action = _scope_filter_policy_call( + config, + query=query, + full_map=map_text, + last_filter=current, + last_result=result, + last_observation=last_obs, + round_idx=round_idx, + max_rounds=max_rounds, + include_full_map=True, + ) + if action.get("kind") == "widen": + if action.get("filter") is not None: + action["kind"] = "filter" + else: + action["kind"] = "fallback" + action["reason"] = action.get("reason") or "widen_without_filter" last_decision = action.get("decision") last_reason = str(action.get("reason") or "") kind = str(action.get("kind") or "") if steps_out: - steps_out[-1].detail["decision"] = last_decision or kind + steps_out[-1].detail["action"] = kind steps_out[-1].detail["reason"] = last_reason if kind == "fallback": + if steps_out: + steps_out[-1].detail["decision"] = "fallback" return ScopeFilterOutcome( decision="fallback", settled_section_ids=list(result.matched_section_ids), @@ -195,20 +222,15 @@ def run_scope_filter( reason=last_reason or "policy_fallback", ) if kind == "done": - if in_band: - return _settle( - result, - in_band=True, - agent_decision=last_decision, - min_hits=min_hits, - rounds=round_idx, - reason=last_reason or "done", - steps_out=steps_out, - ) - nxt = action.get("filter") - if nxt is not None: - current = nxt - continue + return _settle( + result, + in_band=in_band, + agent_decision=last_decision, + min_hits=min_hits, + rounds=round_idx, + reason=last_reason or "done", + steps_out=steps_out, + ) nxt = action.get("filter") if nxt is not None: current = nxt @@ -267,11 +289,13 @@ def _scope_filter_policy_call( config: NavConfig, *, query: str, - map_observation: str, + full_map: str, + last_filter: Optional[NodeFilter], last_result: Optional[FilterResult], last_observation: str, round_idx: int, max_rounds: int, + include_full_map: bool, ) -> Dict[str, Any]: from .nav_llm import nav_chat, resolve_nav_model from .nav_policy import _extract_json_obj @@ -285,21 +309,21 @@ def _scope_filter_policy_call( model_env="NAV_LLM_MODEL", fallback_envs=("NAV_LLM_MODEL",), ) - card = last_result.cardinality if last_result is not None else None - user = ( - f"User query: {query}\n" - f"Round: {round_idx}/{max_rounds}\n" - f"Last cardinality: {card}\n" - f"=== Map ===\n{map_observation}\n=== End Map ===\n" - ) + user = f"Query: {query}\nRound: {round_idx}/{max_rounds}\n" + if last_filter is not None: + user += f"Last filter: {json.dumps(_filter_payload(last_filter), ensure_ascii=False)}\n" + if last_result is not None: + user += f"Last hits: {last_result.cardinality}\n" if last_observation: - user += f"\n=== Last filter observation ===\n{last_observation}\n" + user += f"=== Sub-map (hits of last filter) ===\n{last_observation}\n" + if include_full_map: + user += f"=== Full map ===\n{full_map}\n=== End Full Map ===\n" try: cached = nav_chat( purpose=_SCOPE_FILTER_PURPOSE, model=model, messages=[ - {"role": "system", "content": _scope_filter_system_prompt()}, + {"role": "system", "content": _scope_filter_system_prompt(seed=last_filter is None)}, {"role": "user", "content": user}, ], temperature=float(config.llm_temperature), @@ -314,7 +338,7 @@ def _scope_filter_policy_call( text = str(cached.get("content") or "").strip() obj = _extract_json_obj(text) or {} kind = str(obj.get("action") or obj.get("kind") or "filter").strip().lower() - if kind not in {"filter", "done", "fallback"}: + if kind not in {"filter", "done", "widen", "fallback"}: kind = "filter" decision_raw = str(obj.get("decision") or "").strip().lower() decision: Optional[ScopeDecision] = ( @@ -330,18 +354,39 @@ def _scope_filter_policy_call( } -def _scope_filter_system_prompt() -> str: - return ( - "You write a WHERE node filter over document filenames, section paths, " - "and section summaries. Return json.\n" - "Schema: {\"action\":\"filter|done|fallback\",\"predicates\":" - "[{\"field\":\"path|summary\",\"terms\":[\"...\"],\"match\":\"substring|regex\"}]," +def _scope_filter_system_prompt(*, seed: bool) -> str: + field_schema = ( + '"path"' if seed else '"path|summary"' + ) + field_rule = ( + "Match on section paths only (filenames and title chains)." + if seed + else "Fields AND together; terms inside one field OR together." + ) + base = ( + "You write a WHERE filter over document sections. Return json.\n" + "Schema: {\"action\":\"filter|done|widen|fallback\",\"predicates\":" + f"[{{\"field\":{field_schema},\"terms\":[\"...\"],\"match\":\"substring|regex\"}}]," "\"decision\":\"collect_all|scoped_harvest|fallback\",\"reason\":\"...\"}\n" - "Fields AND together; terms inside one field OR together. " - "Use world-knowledge aliases (e.g. 苹果|AAPL|apple). " - "action=filter revises the predicate; action=done keeps the last apply " - "when the hit count is reasonable; action=fallback drops to keyword harvest. " - "decision is used only when settling." + f"{field_rule}\n" + "Write terms as natural-language words from the query or the map " + "(entities, topics, aliases); do not rely on section numbering alone.\n" + ) + if seed: + return base + ( + "Write the first filter for the query against the full map. " + "action=filter returns it; action=fallback only when no path " + "predicate can isolate the target sections." + ) + return base + ( + "You see the sub-map hit by the last filter; judge by its content:\n" + "- sub-map covers the query -> action=done\n" + "- sub-map has off-topic nodes -> action=filter with a narrower " + "revision of the last filter\n" + "- sub-map looks too narrow or misses parts of the query -> " + "action=widen to see the full map once, then revise\n" + "Revise the last filter, never restart from the query. " + "action=fallback only when no predicate can isolate the target sections." ) @@ -352,22 +397,10 @@ def _filter_payload(nf: NodeFilter) -> List[Dict[str, Any]]: ] -def _compact_map(ts: Any, doc_ids: Sequence[str], char_limit: int) -> str: +def _compact_map(ts: Any, doc_ids: Sequence[str]) -> str: path_fn = getattr(ts, "path_titles", None) structure_fn = getattr(ts, "get_structure", None) lines: List[str] = [] - used = 0 - limit = max(0, int(char_limit)) - - def add_line(text: str) -> bool: - nonlocal used - extra = len(text) + 1 - if limit and used + extra > limit: - lines.append("map truncated") - return False - lines.append(text) - used += extra - return True def path_of(sid: str, doc_id: str) -> str: if not callable(path_fn): @@ -391,8 +424,7 @@ def summary_of(sid: str) -> str: root_fn = getattr(ts, "sections_for_doc", None) for doc_id in doc_ids: - if not add_line(path_of(doc_id, doc_id)): - return "\n".join(lines) + lines.append(path_of(doc_id, doc_id)) stack = [str(s) for s in (root_fn(doc_id) if callable(root_fn) else []) if str(s)] seen: set[str] = set() while stack: @@ -403,8 +435,7 @@ def summary_of(sid: str) -> str: path = path_of(sid, doc_id) summary = summary_of(sid) line = path if not summary else f"{path} | {summary}" - if not add_line(line): - return "\n".join(lines) + lines.append(line) kids = [str(c) for c in (child_fn(sid) if callable(child_fn) else []) if str(c)] stack[0:0] = kids return "\n".join(lines) diff --git a/packages/shared-python/shared/services/retrieval/nav/nav_types.py b/packages/shared-python/shared/services/retrieval/nav/nav_types.py index f0e23475..d164d1aa 100644 --- a/packages/shared-python/shared/services/retrieval/nav/nav_types.py +++ b/packages/shared-python/shared/services/retrieval/nav/nav_types.py @@ -103,7 +103,6 @@ class NavConfig: filter_max_rounds: int = 3 filter_min_hits: int = 1 filter_max_hits: int = 40 - filter_submap_char_limit: int = 2000 @property def is_checklist(self) -> bool: diff --git a/packages/shared-python/shared/services/retrieval/nav_config.py b/packages/shared-python/shared/services/retrieval/nav_config.py index b533a37f..be1d3dd8 100644 --- a/packages/shared-python/shared/services/retrieval/nav_config.py +++ b/packages/shared-python/shared/services/retrieval/nav_config.py @@ -65,7 +65,6 @@ "filter_max_rounds": 3, "filter_min_hits": 1, "filter_max_hits": 40, - "filter_submap_char_limit": 2000, } diff --git a/packages/shared-python/shared/services/retrieval/trace/mapnav.py b/packages/shared-python/shared/services/retrieval/trace/mapnav.py index 1df4b581..52822347 100644 --- a/packages/shared-python/shared/services/retrieval/trace/mapnav.py +++ b/packages/shared-python/shared/services/retrieval/trace/mapnav.py @@ -200,6 +200,7 @@ def _map_one( ) if action == "node_filter": + step_action = str(detail.get("action") or detail.get("decision") or "filter") return DecisionTraceStep( step_index=step_index, agent="navigator", @@ -216,13 +217,11 @@ def _map_one( "round": detail.get("round"), }, decision={ - "action": detail.get("decision") or "filter", + "action": step_action, "reason": detail.get("reason") or "", }, result={ - "status": "fallback" - if str(detail.get("decision") or "") == "fallback" - else "ok", + "status": "fallback" if step_action == "fallback" else "ok", "cardinality": detail.get("cardinality"), "decision": detail.get("decision") or "", "reason": detail.get("reason") or "", diff --git a/packages/shared-python/shared/tests/test_nav_node_filter.py b/packages/shared-python/shared/tests/test_nav_node_filter.py index a66074ff..7e059d6c 100644 --- a/packages/shared-python/shared/tests/test_nav_node_filter.py +++ b/packages/shared-python/shared/tests/test_nav_node_filter.py @@ -163,6 +163,7 @@ def test_regex_or_terms_and_preview_budget() -> None: assert set(result.matched_section_ids) == {"sec_q3", "sec_crop"} assert result.cardinality == 2 - preview = render_submap_observation(ts, result, char_limit=40) + preview = render_submap_observation(ts, result) assert preview.startswith("hits=2") - assert "tighten the predicate" in preview + assert "sec_q3" not in preview # paths shown, not ids + assert "Q3 Results" in preview diff --git a/packages/shared-python/shared/tests/test_nav_scope_filter.py b/packages/shared-python/shared/tests/test_nav_scope_filter.py index eb0d9a27..b762e085 100644 --- a/packages/shared-python/shared/tests/test_nav_scope_filter.py +++ b/packages/shared-python/shared/tests/test_nav_scope_filter.py @@ -84,7 +84,6 @@ def _cfg(**kwargs: Any) -> NavConfig: "filter_max_rounds": 3, "filter_min_hits": 1, "filter_max_hits": 40, - "filter_submap_char_limit": 2000, "llm_model": "test-model", "llm_max_tokens": 256, } @@ -106,6 +105,38 @@ def fake_nav_chat(**kwargs: Any) -> dict[str, Any]: ) +def test_widen_relooks_full_map(monkeypatch: Any) -> None: + seen_users: List[str] = [] + queue: List[dict[str, Any]] = [ + {"action": "widen", "reason": "sub-map too narrow"}, + {"action": "done", "decision": "collect_all", "reason": "ok"}, + ] + + def fake_nav_chat(**kwargs: Any) -> dict[str, Any]: + messages = kwargs.get("messages") or [] + seen_users.append(str(messages[-1].get("content") or "")) + obj = queue.pop(0) if queue else {"action": "fallback", "reason": "empty"} + return {"content": json.dumps(obj)} + + monkeypatch.setattr( + "shared.services.retrieval.nav.nav_llm.nav_chat", + fake_nav_chat, + ) + out = run_scope_filter( + _ts(), + _cfg(), + query="apple q3 profit", + doc_ids=["doc_apple", "doc_other"], + seed_filter=node_filter([field_predicate("path", ["Q3"])]), + ) + assert out.decision == "collect_all" + assert len(seen_users) == 2 + assert "Full map" not in seen_users[0] + assert "Sub-map" in seen_users[0] + assert "Full map" in seen_users[1] + assert "Last filter" in seen_users[1] + + def test_zero_hits_widen_then_done(monkeypatch: Any) -> None: _install_script( monkeypatch, diff --git a/packages/shared-python/shared/tests/test_nav_trace_map.py b/packages/shared-python/shared/tests/test_nav_trace_map.py index eaf062ec..e25f6cfe 100644 --- a/packages/shared-python/shared/tests/test_nav_trace_map.py +++ b/packages/shared-python/shared/tests/test_nav_trace_map.py @@ -138,6 +138,7 @@ def test_node_filter_steps_map_and_count_tokens() -> None: ], "fields": ["path"], "cardinality": 2, + "action": "done", "decision": "collect_all", "reason": "small_cardinality", "matched_section_ids": ["sec_q3"], @@ -153,7 +154,8 @@ def test_node_filter_steps_map_and_count_tokens() -> None: assert steps[0].phase == "node_filter" assert steps[0].observation["cardinality"] == 2 assert steps[0].observation["fields"] == ["path"] - assert steps[0].decision["action"] == "collect_all" + assert steps[0].decision["action"] == "done" + assert steps[0].result["decision"] == "collect_all" assert steps[0].budget["tokens_used_delta"] == 80 assert steps[0].budget["token_limit"] == 100000 assert steps[-1].result["layer_llm_steps"]["harvest"] >= 1