feat: TOC outline keep/null-page react + nav node filter

apps/worker/app/services/document_agent/structure/null_page_react.py

@@ -36,10 +36,11 @@ def react_budget() -> int:
 Ordered query strategy (follow this order; skip a step only if already tried
 or not applicable to the given title / parent path). Pattern-level only —
 do not invent document-specific titles:
-1. Derive the search line from the given title by removing leading number /
-   letter / punctuation prefixes and trailing metadata qualifiers (document
-   identifiers/codes, revision labels, and similar). Keep the semantic title
-   body. Prefer that body over a metadata-only query when both are present.
+1. Derive the semantic title body from the given title: strip leading
+   number / letter / punctuation prefixes and trailing metadata qualifiers
+   (document identifiers/codes, revision labels, and similar). The given title
+   may be a merged heading (semantic body plus a code); still reduce it to the
+   semantic body first. Grep that body before any metadata-only query.
 2. When the parent path indicates appendices/annexes (or the title is a
    lettered appendix-style entry): grep the structural form
    "Appendix <letter>" using the letter taken from the title. Prefer this

apps/worker/app/services/document_agent/structure/toc_anchoring.py

@@ -195,15 +195,16 @@ def _try_outline_anchoring_route(
         if (judge_result.payload or {}).get("choice") != OUTLINE_CHOICE:
             return False
 
-    toc_with_level: list[dict[str, Any]] = []
-    for entry in kept:
-        row: dict[str, Any] = {
+    # Judge digest used full ``kept`` (incl. null pages). Anchoring / null-page
+    # ReAct only consume resolvable outline destinations (``paged_kept``).
+    toc_with_level: list[dict[str, Any]] = [
+        {
             "heading": entry["heading"],
             "level": entry["level"],
+            "physical_page": int(entry["page"]),
         }
-        if entry.get("page") is not None:
-            row["physical_page"] = int(entry["page"])
-        toc_with_level.append(row)
+        for entry in paged_kept
+    ]
 
     hierarchy = {
         "source": "pdf_outline",

apps/worker/app/services/document_agent/tools/find_toc_anchor_pages.py

@@ -83,12 +83,30 @@ def _merge_keyword_split_lines(
     return merged
 
 
+def _match_toc_keyword_in_line(normalized_line: str) -> str | None:
+    """Return the longest TOC keyword contained in a normalized line."""
+    if not normalized_line:
+        return None
+    return next(
+        (
+            keyword
+            for keyword in sorted(TOC_KEYWORDS, key=len, reverse=True)
+            if keyword in normalized_line
+        ),
+        None,
+    )
+
+
 def _find_toc_text_matches(lines: list[str]) -> list[dict[str, Any]]:
-    """Match TOC keywords only as whole lines after keyword-split repair."""
+    """Match TOC keywords as line-level containment after keyword-split repair.
+
+    Cross-line handling stays keyword-internal only (e.g. 目+录). Hit rule is
+    ``keyword in normalized_line`` (longest match wins), not whole-line equality.
+    """
     matches: list[dict[str, Any]] = []
     for raw_line, start_idx, end_idx in _merge_keyword_split_lines(lines):
-        keyword = normalize_match_text(raw_line)
-        if keyword not in TOC_KEYWORDS:
+        keyword = _match_toc_keyword_in_line(normalize_match_text(raw_line))
+        if keyword is None:
             continue
         matches.append(
             {
@@ -205,8 +223,9 @@ def _filter_recurring_elements(
 @register_tool(
     name="find.toc_anchor_pages",
     description=(
-        "Scan full PDF page text for whole-line TOC keywords, filter recurring "
-        "navigation elements, then render candidate PNGs for VLM confirmation."
+        "Scan full PDF page text for line-level TOC keywords (containment after "
+        "keyword-split repair), filter recurring navigation elements, then "
+        "render candidate PNGs for VLM confirmation."
     ),
     preconditions=(has_page_labels, has_page_full_text),
 )

apps/worker/app/services/document_agent/tools/probe_outline.py

@@ -1,4 +1,4 @@
-"""probe.outline: read PDF bookmarks via get_toc and build a pruned tree."""
+"""probe.outline: read PDF bookmarks via get_toc and build a nested tree."""
 
 from __future__ import annotations
 
@@ -18,8 +18,12 @@ def _normalize_page(raw: Any) -> int | None:
     return page if page > 0 else None
 
 
-def _flat_toc_to_forest(entries: list[list[Any]]) -> list[dict[str, Any]]:
-    """Convert flat ``[level, title, page]`` rows into a nested forest."""
+def build_outline_forest(entries: list[list[Any]]) -> list[dict[str, Any]]:
+    """Convert flat ``[level, title, page]`` rows into a nested forest.
+
+    No-page destinations (``page <= 0``) become ``page=None`` and are retained;
+    printed TOC likewise keeps entries whose printed pages are out of range.
+    """
     roots: list[dict[str, Any]] = []
     stack: list[dict[str, Any]] = []
     for row in entries:
@@ -45,30 +49,6 @@ def _flat_toc_to_forest(entries: list[list[Any]]) -> list[dict[str, Any]]:
     return roots
 
 
-def prune_outline_forest(nodes: list[dict[str, Any]]) -> list[dict[str, Any]]:
-    """Keep no-page parents when descendants have pages; drop no-page leaves/subtrees."""
-    kept: list[dict[str, Any]] = []
-    for node in nodes:
-        children = prune_outline_forest(list(node.get("children") or []))
-        page = node.get("page")
-        if page is None and not children:
-            # No-page leaf, or entire no-page subtree after child prune.
-            continue
-        kept.append(
-            {
-                "title": node["title"],
-                "level": node["level"],
-                "page": page,
-                "children": children,
-            }
-        )
-    return kept
-
-
-def build_outline_forest(entries: list[list[Any]]) -> list[dict[str, Any]]:
-    return prune_outline_forest(_flat_toc_to_forest(entries))
-
-
 def _count_nodes(nodes: list[dict[str, Any]]) -> int:
     total = 0
     for node in nodes:
@@ -79,8 +59,8 @@ def _count_nodes(nodes: list[dict[str, Any]]) -> int:
 @register_tool(
     name="probe.outline",
     description=(
-        "Read PDF bookmark outline via get_toc(simple=True) and return a pruned tree. "
-        "No-page parents are kept when children have pages; no-page leaves/subtrees are dropped."
+        "Read PDF bookmark outline via get_toc(simple=True) and return a nested "
+        "tree. No-page destinations are kept as page=null."
     ),
     parameters={
         "type": "object",

apps/worker/scripts/page_memory/_debug_pm_shared.py

@@ -604,6 +604,9 @@ def run_stage1_toc(
         persist_anatomy_map(coordinator.ctx, {})
         profile_path = out_dir / DOC_PROFILE_FILENAME
         write_debug_json(profile_path, anatomy.to_dict())
+        # Stage-2 resumes outline separately: anatomy map does not carry it.
+        outline_roots = list(coordinator.blackboard.pdf_outline_roots or [])
+        persist_pdf_outline_roots(out_dir, outline_roots)
         # Canonical profile is at package root; drop nested duplicate.
         try:
             (out_dir / "_doc_agent" / "anatomy_map.json").unlink()
@@ -620,6 +623,7 @@ def run_stage1_toc(
             payload={
                 "toc_pages": list(getattr(anatomy.toc_result, "toc_pages", []) or []),
                 "region_count": len(list(anatomy.toc_hierarchies or [])),
+                "outline_root_count": len(outline_roots),
                 "skip_toc_anchoring": True,
             },
         )
@@ -752,6 +756,7 @@ def pipeline_state_path(out_dir: Path) -> Path:
 
 STAGE0_STATE_NAME = "stage0_state.json"
 PAGE_TEXT_CACHE_NAME = "page_full_text_cache.json"
+PDF_OUTLINE_ROOTS_NAME = "pdf_outline_roots.json"
 
 
 def stage0_state_path(out_dir: Path) -> Path:
@@ -762,6 +767,80 @@ def page_text_cache_path(out_dir: Path) -> Path:
     return out_dir / "_doc_agent" / PAGE_TEXT_CACHE_NAME
 
 
+def pdf_outline_roots_path(out_dir: Path) -> Path:
+    """Stage-1 outline forest consumed by Stage-2 ``run_toc_anchoring``."""
+    return out_dir / "_doc_agent" / PDF_OUTLINE_ROOTS_NAME
+
+
+def persist_pdf_outline_roots(out_dir: Path, roots: list[Any] | None) -> Path:
+    """Write Stage-1 ``probe.outline`` forest for Stage-2 resume."""
+    path = pdf_outline_roots_path(out_dir)
+    write_debug_json(path, list(roots or []))
+    return path
+
+
+def load_pdf_outline_roots(out_dir: Path) -> list[dict[str, Any]] | None:
+    """Load Stage-1 outline forest, or ``None`` when the artifact is missing."""
+    path = pdf_outline_roots_path(out_dir)
+    if not path.exists():
+        return None
+    data = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(data, list):
+        raise ValueError(f"pdf_outline_roots must be a list: {path}")
+    return [row for row in data if isinstance(row, dict)]
+
+
+def restore_pdf_outline_roots_for_anchoring(coordinator, out_dir: Path) -> None:
+    """Put Stage-1 outline on the blackboard (re-probe + persist if missing).
+
+    Production keeps ``probe.outline`` on the same blackboard through
+    ``run_toc_anchoring``. Staged debug must restore that forest explicitly.
+    """
+    outline_roots = load_pdf_outline_roots(out_dir)
+    if outline_roots is not None:
+        coordinator.blackboard.pdf_outline_roots = outline_roots
+        logger.info(
+            "  restored pdf_outline_roots from Stage-1 ({} roots)",
+            len(outline_roots),
+        )
+        return
+
+    from app.services.document_agent.registry import REGISTRY
+
+    logger.warning("  Stage-1 pdf_outline_roots missing; re-running probe.outline")
+    REGISTRY.dispatch("probe.outline", coordinator.ctx, {})
+    persist_pdf_outline_roots(
+        out_dir, list(coordinator.blackboard.pdf_outline_roots or [])
+    )
+
+
+def load_stage1_into_coordinator_for_anchoring(
+    coordinator,
+    out_dir: Path,
+    anatomy,
+) -> None:
+    """Resume Stage-0 + Stage-1 TOC state for production ``run_toc_anchoring``.
+
+    Loads page text/features from Stage-0, TOC extract outputs from Stage-1
+    anatomy, restores outline roots, and clears any prior skeleton_* so
+    anchoring writes a fresh result.
+    """
+    from app.services.document_agent.validators import single_shard_plan
+
+    load_stage0_into_coordinator(coordinator, out_dir)
+    bb = coordinator.blackboard
+    page_count = int(getattr(anatomy, "page_count", None) or bb.page_count or 0)
+    bb.toc_result = anatomy.toc_result
+    bb.toc_hierarchies = list(getattr(anatomy, "toc_hierarchies", None) or [])
+    bb.shard_plan = getattr(anatomy, "shard_plan", None) or single_shard_plan(
+        page_count
+    )
+    bb.skeleton_anchor = None
+    bb.skeleton_nodes = None
+    bb.pending_skeleton_anchors = []
+    restore_pdf_outline_roots_for_anchoring(coordinator, out_dir)
+
+
 def load_pipeline_state(
     state_path: Path,
 ) -> dict[str, Any]:

apps/worker/scripts/page_memory/debug_pm_null_page_react.py

@@ -24,7 +24,7 @@
     _build_debug_coordinator,
     base_argparser,
     load_anatomy_cache,
-    load_stage0_into_coordinator,
+    load_stage1_into_coordinator_for_anchoring,
     page_text_cache_path,
     require_file,
     resolve_anatomy_cache_path,
@@ -46,7 +46,6 @@ def main() -> int:
     args = parser.parse_args()
 
     from app.services.document_agent.structure.toc_anchoring import run_toc_anchoring
-    from app.services.document_agent.validators import single_shard_plan
     from shared.core.config import settings
 
     pdf_path, filename, out_dir = resolve_paths(args)
@@ -56,8 +55,6 @@ def main() -> int:
     require_file(anatomy_cache, hint="Run Stage 1 first")
 
     anatomy = load_anatomy_cache(anatomy_cache, pdf_path, filename)
-    page_count = int(anatomy.page_count or 0)
-    hierarchies = list(getattr(anatomy, "toc_hierarchies", None) or [])
 
     logger.info("█" * 70)
     logger.info("  Production null-page locate dump — {}", filename)
@@ -74,14 +71,8 @@ def main() -> int:
             model=None if args.no_vlm else args.model,
             settings_extra={"skip_toc_anchoring": False},
         )
-        load_stage0_into_coordinator(coordinator, out_dir)
+        load_stage1_into_coordinator_for_anchoring(coordinator, out_dir, anatomy)
         bb = coordinator.blackboard
-        bb.toc_result = anatomy.toc_result
-        bb.toc_hierarchies = hierarchies
-        bb.shard_plan = anatomy.shard_plan or single_shard_plan(page_count)
-        bb.skeleton_anchor = None
-        bb.skeleton_nodes = None
-        bb.pending_skeleton_anchors = []
 
         run_toc_anchoring(coordinator.ctx)
 

apps/worker/scripts/page_memory/debug_pm_stage1_hierarchy.py

@@ -4,8 +4,8 @@
 
 Resumes Stage-0 blackboard (``stage0_state.json`` + ``page_full_text_cache.json``,
 including asset-probe ``has_asset`` flags) and runs the production TOC segment:
-  find.toc_anchor_pages → extract.toc_with_boundaries
-  → persist doc_profile.json
+  find.toc_anchor_pages → probe.outline → extract.toc_with_boundaries
+  → persist doc_profile.json + ``_doc_agent/pdf_outline_roots.json``
 
 Requires Stage 0 first:
   uv run python scripts/page_memory/debug_pm_stage0_bootstrap.py --file ...

apps/worker/scripts/page_memory/debug_pm_stage2_calibration.py

@@ -4,7 +4,7 @@
 
 Same PROFILE anchoring path as production PAGE/TEXT:
 
-  select primary/pending → calibrate (Agent Phase-1 + Phase-2) →
+  restore Stage-1 outline (or re-probe) → calibrate / outline route →
   classify contained/parallel → graft contained → write skeleton_*
 
 Also resolves coarse skeletons (C4 resolve-only) into pipeline state so
@@ -36,7 +36,7 @@
     _serialize_skeletons,
     base_argparser,
     load_anatomy_cache,
-    load_stage0_into_coordinator,
+    load_stage1_into_coordinator_for_anchoring,
     page_text_cache_path,
     pipeline_state_path,
     record_stage,
@@ -125,14 +125,8 @@ def main() -> int:
                 "skip_toc_anchoring": False,
             },
         )
-        load_stage0_into_coordinator(coordinator, out_dir)
+        load_stage1_into_coordinator_for_anchoring(coordinator, out_dir, anatomy)
         bb = coordinator.blackboard
-        bb.toc_result = anatomy.toc_result
-        bb.toc_hierarchies = hierarchies
-        bb.shard_plan = anatomy.shard_plan or single_shard_plan(page_count)
-        bb.skeleton_anchor = None
-        bb.skeleton_nodes = None
-        bb.pending_skeleton_anchors = []
 
         run_toc_anchoring(coordinator.ctx)
 

apps/worker/tests/contract/test_doc_profile_anatomy_contract.py

@@ -84,26 +84,32 @@ def _seed_preprobed_pages(
     coordinator.blackboard.global_signals["assets_probed"] = True
 
 
-def test_toc_anchor_text_scan_whole_line_keyword_and_split_repair() -> None:
+def test_toc_anchor_text_scan_line_contains_keyword_and_split_repair() -> None:
     late_lines = [f"body line {idx}" for idx in range(60)] + ["目录"]
     split_lines = ["Table of", "Con", "tents"]
-    false_positive_lines = [
+    # Containment (not whole-line equality): prefixed titles and body mentions hit.
+    contains_lines = [
+        "General table of contents",
         "Commentary provides guidance on minimum cement contents in different situations.",
         "The basic contents of a typical contract document are shown below:",
     ]
 
     late_matches = toc_anchor_tool._find_toc_text_matches(late_lines)  # noqa: SLF001
     split_matches = toc_anchor_tool._find_toc_text_matches(split_lines)  # noqa: SLF001
-    false_matches = toc_anchor_tool._find_toc_text_matches(  # noqa: SLF001
-        false_positive_lines
+    contains_matches = toc_anchor_tool._find_toc_text_matches(  # noqa: SLF001
+        contains_lines
     )
 
     assert late_matches[0]["line_index"] == 60
     assert late_matches[0]["match_kind"] == "keyword:目录"
     assert split_matches[0]["match_kind"] == "keyword:table of contents"
     assert split_matches[0]["line_index"] == 0
     assert split_matches[0]["line_end_index"] == 2
-    assert false_matches == []
+    assert [m["match_kind"] for m in contains_matches] == [
+        "keyword:table of contents",
+        "keyword:contents",
+        "keyword:contents",
+    ]
 
 
 def test_toc_extraction_raises_on_pipeline_failure(tmp_path: Path) -> None: