From f9698de970755e36350050ea1be8afd6aa333b44 Mon Sep 17 00:00:00 2001 From: yeqi Date: Mon, 9 Nov 2026 13:58:47 +0100 Subject: [PATCH 1/4] feat(tools): slim the ToC in the agent's build_datasheet result (0.47.0) The manifest stays in the conversation and is re-sent on every later turn, and on large documents its ToC was most of the agent's context: 270K chars on PIC16F887, 192K on ESP32 TRM, with titles only 13% of the node data. The build_datasheet handler now drops breadcrumb, level, node_id, has_tables, cross_references and footnote_markers from each node, and table_count when it is zero. breadcrumb and has_tables restate other fields, no tool takes a node_id, and level differs from the nesting only when an outline skips a level. Cross-references and footnote markers are in the page text, and a resolved section reference is still findable by its number in the titles. The description points the agent at json_path for all three. A denylist, so new enrichments still reach the agent. get_artifact_manifest() and the ToC JSON are unchanged. A/B with Sonnet 4.6 and GPT-5.1 on footnote and cross-reference questions: no measurable accuracy or turn change, prompt tokens per question down 2.7-2.9x. --- CHANGELOG.md | 21 ++++ README.md | 4 +- docs/datasheetindex_architecture.md | 31 ++++++ pyproject.toml | 2 +- src/datasheetindex/tools/defs.py | 58 ++++++++++- tests/test_defs.py | 151 ++++++++++++++++++++++++++++ uv.lock | 2 +- 7 files changed, 263 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a585d5a..2d71950 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,27 @@ All notable changes to this project will be documented in this file. +## [0.47.0] - 2026-11-09 + +### Changed +- **Slimmer ToC in the agent's `build_datasheet` result.** Each ToC node the + agent receives now omits `breadcrumb`, `level`, `node_id`, `has_tables`, + `cross_references` and `footnote_markers`, and omits `table_count` when it is + zero. Every node is still there, with its title, page range and nesting, + along with `boilerplate_category`, `continued_tables`, `summary` and any field + not on that list. The manifest stays in the conversation and is re-sent on + every turn; on large documents it was most of the agent's context (270K + characters on PIC16F887, 192K on ESP32 TRM) and is now about 3.5x smaller. + On a before/after test with Sonnet 4.6 and GPT-5.1, over questions whose + answers sit in footnotes and cross-referenced tables, accuracy and turn + counts were unchanged while prompt tokens per question fell 2.7-2.9x. +- Applies to the MCP server, `create_datasheet_tool_defs()` and the Agent SDK + server. `DatasheetTools.get_artifact_manifest()` and the ToC JSON artifact + are unchanged and keep every field; `search_text` hits still carry their + breadcrumb, and the `build_datasheet` description tells the agent that + breadcrumbs, cross-references and footnote markers are in the ToC JSON at + `json_path`. + ## [0.46.0] - 2026-11-05 ### Added diff --git a/README.md b/README.md index 9e44d72..e7ae654 100644 --- a/README.md +++ b/README.md @@ -442,7 +442,9 @@ You can run the local MCP server directly from the repository. It exposes these tools for the bound PDF source: - `build_datasheet` - build and save the `.json` / `.txt` artifacts, and return - the manifest: source info, total pages, ToC quality, the full enriched ToC, + the manifest: source info, total pages, ToC quality, every ToC node (title, + page range, nonzero table count, boilerplate and continued-table flags -- + breadcrumbs, cross-references and footnote markers stay in the `.json`), and a bounded `figures` digest naming the pages that carry figure entries (see "Figure indexing and captions"). For a PDF with no usable ToC the manifest also carries a `hint` telling the agent to navigate by `search_text` diff --git a/docs/datasheetindex_architecture.md b/docs/datasheetindex_architecture.md index 97c88dc..92d94d3 100644 --- a/docs/datasheetindex_architecture.md +++ b/docs/datasheetindex_architecture.md @@ -1529,6 +1529,37 @@ any one agent framework: single pattern or a list and tags each hit with its section breadcrumb; `get_section_text` returns a page range prefixed with a position header. +**The agent's ToC is slimmer than the artifact's.** The `build_datasheet` +handler (`tools/defs.py:_agent_toc`) drops `breadcrumb`, `level`, `node_id`, +`has_tables`, `cross_references` and `footnote_markers` from every node, and +`table_count` where it is zero; `DatasheetTools.get_artifact_manifest()` and +the ToC JSON keep them all. The manifest stays in the conversation and is +re-sent on every later turn, and on large documents it was most of the agent's +context: 270K chars on PIC16F887 and 192K on ESP32 TRM, where a probe lookup +that read one 2.7K-char page peaked near 80K prompt tokens. Titles were 13% of +the node data; the dropped fields were 72%. `breadcrumb` and `has_tables` +restate other fields (equal on all 3,734 corpus nodes), no tool takes a +`node_id`, and `level` differs from the nesting only when an outline skips a +level. The other two are a judgement: both are in the page text the agent +reads, and the 988 of 2,248 references that resolve (`target_node_id`) point +at a section whose title starts with the referenced number, so the target is +still findable by title. The tool description tells the agent the dropped +fields are in the ToC JSON at `json_path`. It is a denylist so that a new +enrichment reaches the agent unless someone decides it should not. + +Measured before merging (2026-11-09): 10 questions chosen so that their +answers sit in a footnote or a cross-referenced table, 9 of them on large-ToC +documents, Sonnet 4.6 and GPT-5.1. Run counts per arm are unequal (50-59) and +not exactly paired: the run was cut short by a harness bug that duplicated +runs. Sonnet was 51/51 before and 52/52 after. GPT was 42/50 (84%) before and +48/59 (81%) after, with per-question changes in both directions (q0 3/5 -> +4/6, q6 3/5 -> 2/6) -- no measurable effect at this n, not a demonstrated +zero. Some GPT misses were grading artefacts (it writes U+202F between number +and unit) and hit both arms; the after-arm misses read by hand were a register +table misread, a wrong page and a text-layout quirk, none involving the cut. +Turns did not move (3.94 -> 3.90, 4.04 -> 4.02), and mean prompt tokens per +question fell 2.7-2.9x (206K -> 76K, 168K -> 58K). + `search_text` also carries the text-layer limitation on **both** surfaces its consumer reads: in the tool description, and — when a search returns nothing on a document that holds raster regions — as a `note` on the result itself. The diff --git a/pyproject.toml b/pyproject.toml index 212c119..96b0d46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "datasheetindex" -version = "0.46.0" +version = "0.47.0" description = "Agent-first parameter extraction from technical datasheets" readme = "README.md" requires-python = ">=3.13" diff --git a/src/datasheetindex/tools/defs.py b/src/datasheetindex/tools/defs.py index e7bbc6d..bb967f2 100644 --- a/src/datasheetindex/tools/defs.py +++ b/src/datasheetindex/tools/defs.py @@ -73,6 +73,52 @@ "figures array in the ToC JSON at json_path is the complete set." ) +#: ToC node fields left out of the manifest the *agent* receives. The manifest +#: is re-sent on every later turn, and these were 72% of its node data on the +#: 25-document corpus (PIC16F887 270K -> 79K chars, ESP32 TRM 192K -> 54K). +#: ``breadcrumb`` restates the nesting and ``has_tables`` restates +#: ``table_count`` (both equal on all 3,734 corpus nodes); no tool takes a +#: ``node_id``. ``level`` is the outline's own depth number, which differs from +#: the nesting only when an outline skips a level -- the nesting is what the +#: agent navigates by. ``cross_references`` and ``footnote_markers`` are a +#: judgement call: their text is in the pages the agent reads, and the 988 of +#: 2,248 references that resolve (``target_node_id``) point at a section whose +#: title starts with the referenced number, so the target is still findable by +#: title. A denylist, not an allowlist, so a new enrichment reaches the agent +#: unless deliberately cut. +#: ``DatasheetTools.get_artifact_manifest`` and the ToC JSON keep every field. +_AGENT_TOC_OMITTED_FIELDS = frozenset( + { + "breadcrumb", + "level", + "node_id", + "has_tables", + "cross_references", + "footnote_markers", + } +) + + +def _agent_toc(nodes: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Return a copy of ``nodes`` slimmed for the agent; the input is untouched. + + ``table_count`` is kept only when nonzero: its absence then says what + ``has_tables: false`` used to. + """ + slim = [] + for node in nodes: + out = { + key: value + for key, value in node.items() + if key not in _AGENT_TOC_OMITTED_FIELDS + and key != "nodes" + and not (key == "table_count" and not value) + } + if node.get("nodes"): + out["nodes"] = _agent_toc(node["nodes"]) + slim.append(out) + return slim + @dataclass(frozen=True) class DatasheetToolDef: @@ -244,7 +290,11 @@ async def build_datasheet(args: dict[str, Any]) -> dict[str, Any]: tools_instance = target if previous is not None: _safe_close(previous) - return _ok(tools_instance.get_artifact_manifest()) + manifest = tools_instance.get_artifact_manifest() + toc = manifest.get("toc") + if isinstance(toc, list): + manifest["toc"] = _agent_toc(toc) + return _ok(manifest) except Exception as exc: return _err_exc(exc) @@ -347,8 +397,10 @@ def _close() -> None: "before the other tools; calling it again with a different " "pdf_source switches documents.\n\n" "The manifest carries the source, total pages, ToC quality, and " - "the enriched Table of Contents -- section hierarchy, page " - "ranges, table counts, footnote markers, cross-references -- " + "the enriched ToC -- section hierarchy, page " + "ranges, nonzero table counts, boilerplate flags; breadcrumbs, " + "cross-references and footnote markers are only in the ToC " + "JSON at json_path -- " "plus a 'figures' digest: total/raster/captioned counts and up " "to 40 {page, figures, caption} rows ('truncated' says when " "more exist), each row carrying that page's largest figure. A " diff --git a/tests/test_defs.py b/tests/test_defs.py index b598838..d5d8faf 100644 --- a/tests/test_defs.py +++ b/tests/test_defs.py @@ -495,6 +495,157 @@ def test_manifest_has_no_hint_when_toc_is_present(tmp_path): assert "hint" not in manifest +def _nested_toc_pdf(path): + """Two chapters, one with a subsection, enough to produce every enrichment.""" + doc = pymupdf.open() + texts = [ + "1 Electrical characteristics\nSupply current 3 mA (1), see Table 2\n" + "(1) Measured at 25 C.", + "1.1 Supply\nVDD 1.8 V to 3.6 V, see Figure 4", + "2 Revision history\nRev A initial release", + ] + for text in texts: + page = doc.new_page() + page.insert_text((72, 72), text) + doc.set_toc( + [ + [1, "1 Electrical characteristics", 1], + [2, "1.1 Supply", 2], + [1, "2 Revision history", 3], + ] + ) + doc.save(str(path)) + doc.close() + + +def _walk(nodes): + for node in nodes: + yield node + yield from _walk(node.get("nodes", [])) + + +def test_agent_manifest_toc_drops_derivable_and_bulky_fields(tmp_path): + """The agent's ToC keeps navigation and judgement fields, not bulk. + + Measured on the 25-document corpus: the manifest ToC reached 270K chars + (PIC16F887), titles were 13% of it, and the result is re-sent on every + later turn. breadcrumb/level restate the nesting, has_tables restates + table_count, node_id feeds no tool, and cross_references/footnote_markers + carry no page to navigate to -- their text is in the pages the agent reads. + """ + pdf = tmp_path / "nested.pdf" + _nested_toc_pdf(pdf) + defs = _defs_by_name() + + manifest = json.loads( + _run(defs["build_datasheet"].handler, {"pdf_source": str(pdf)})["content"][0][ + "text" + ] + ) + + nodes = list(_walk(manifest["toc"])) + assert [n["title"] for n in nodes] == [ + "1 Electrical characteristics", + "1.1 Supply", + "2 Revision history", + ] + assert manifest["toc"][0]["nodes"][0]["title"] == "1.1 Supply" + dropped = { + "breadcrumb", + "level", + "node_id", + "has_tables", + "cross_references", + "footnote_markers", + } + for node in nodes: + assert not dropped & node.keys(), node + assert {"start_page", "end_page"} <= node.keys() + assert node.get("table_count", 1) > 0 + # Judgement signals still reach the agent. + assert manifest["toc"][1]["boilerplate_category"] == "revision" + + +def test_python_manifest_keeps_the_full_toc(tmp_path): + """Slimming is the agent adapter's job; the Python API is unchanged.""" + pdf = tmp_path / "nested.pdf" + _nested_toc_pdf(pdf) + from datasheetindex import DatasheetTools + + with DatasheetTools(str(pdf)) as tools: + tools.build_datasheet(output_dir=str(tmp_path)) + toc = tools.get_artifact_manifest()["toc"] + + assert isinstance(toc, list) + first = toc[0] + assert first["breadcrumb"] == "1 Electrical characteristics" + assert {"node_id", "level", "has_tables", "table_count"} <= first.keys() + assert first["cross_references"] + assert first["footnote_markers"] + + +def test_agent_toc_passes_unknown_fields_through(): + """A denylist, so a future enrichment reaches the agent by default. + + An allowlist would hide every new signal from the agent until someone + remembered to list it -- the way the preamble never reached it. + """ + from datasheetindex.tools.defs import _agent_toc + + toc = [ + { + "title": "A", + "start_page": 1, + "end_page": 2, + "table_count": 0, + "has_tables": False, + "summary": "s", + "continued_tables": [{"page": 2}], + "future_signal": True, + "nodes": [{"title": "B", "start_page": 2, "end_page": 2, "level": 2}], + } + ] + assert _agent_toc(toc) == [ + { + "title": "A", + "start_page": 1, + "end_page": 2, + "summary": "s", + "continued_tables": [{"page": 2}], + "future_signal": True, + "nodes": [{"title": "B", "start_page": 2, "end_page": 2}], + } + ] + # The input is the cached artifact's tree: it must not be mutated. + assert toc[0]["has_tables"] is False + assert toc[0]["nodes"][0]["level"] == 2 + + +def test_build_datasheet_description_matches_the_slim_toc(): + """The description must not promise dropped fields as manifest content. + + It names them once, as living in the ToC JSON, so an agent that needs a + breadcrumb or cross-reference knows where to look. Checked against the + denylist so a field added there without a description update fails here. + """ + from datasheetindex.tools.defs import _AGENT_TOC_OMITTED_FIELDS + + description = _defs_by_name()["build_datasheet"].description + manifest_part, _, rest = description.partition("only in the ToC JSON at json_path") + assert rest, "description no longer says where the omitted fields live" + spoken = { + "breadcrumb": "breadcrumbs", + "cross_references": "cross-references", + "footnote_markers": "footnote markers", + } + for field, phrase in spoken.items(): + assert field in _AGENT_TOC_OMITTED_FIELDS + # Named in the json_path sentence, not earlier as manifest content. + sentence_start = manifest_part.rfind(";") + assert phrase in manifest_part[sentence_start:], phrase + assert phrase not in manifest_part[:sentence_start], phrase + + def test_build_datasheet_schema_bounds_max_figure_captions(): defs = {d.name: d for d in create_datasheet_tool_defs()} props = defs["build_datasheet"].input_schema["properties"] diff --git a/uv.lock b/uv.lock index f654bc8..a9ceb04 100644 --- a/uv.lock +++ b/uv.lock @@ -354,7 +354,7 @@ wheels = [ [[package]] name = "datasheetindex" -version = "0.46.0" +version = "0.47.0" source = { editable = "." } dependencies = [ { name = "pymupdf" }, From f9ae8762cf41245f81910dd30c5be6c2be0d2f70 Mon Sep 17 00:00:00 2001 From: yeqi Date: Mon, 9 Nov 2026 14:21:48 +0100 Subject: [PATCH 2/4] fix(tools): send the agent to the page text, not json_path, for dropped ToC fields PR review: no tool reads the ToC JSON and its path may not resolve on the agent's side (the same wall the preamble hits), so the build_datasheet description now says breadcrumbs, cross-references and footnote markers are left out and to read them in the page text. search_text no longer calls its breadcrumb "the ToC breadcrumb", which the agent's ToC no longer has. Tests: the slim-shape test now asserts the unslimmed tree had every dropped field, so it cannot pass vacuously; the continued_tables fixture uses the real list[str] shape; the description test follows the new wording. --- CHANGELOG.md | 4 +-- docs/datasheetindex_architecture.md | 12 ++++---- src/datasheetindex/tools/defs.py | 30 +++++++++++-------- tests/test_defs.py | 46 ++++++++++++++++++----------- 4 files changed, 54 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d71950..d98a3c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,8 +20,8 @@ All notable changes to this project will be documented in this file. server. `DatasheetTools.get_artifact_manifest()` and the ToC JSON artifact are unchanged and keep every field; `search_text` hits still carry their breadcrumb, and the `build_datasheet` description tells the agent that - breadcrumbs, cross-references and footnote markers are in the ToC JSON at - `json_path`. + breadcrumbs, cross-references and footnote markers are left out and to read + them in the page text. ## [0.46.0] - 2026-11-05 diff --git a/docs/datasheetindex_architecture.md b/docs/datasheetindex_architecture.md index 92d94d3..140c046 100644 --- a/docs/datasheetindex_architecture.md +++ b/docs/datasheetindex_architecture.md @@ -1541,11 +1541,13 @@ the node data; the dropped fields were 72%. `breadcrumb` and `has_tables` restate other fields (equal on all 3,734 corpus nodes), no tool takes a `node_id`, and `level` differs from the nesting only when an outline skips a level. The other two are a judgement: both are in the page text the agent -reads, and the 988 of 2,248 references that resolve (`target_node_id`) point -at a section whose title starts with the referenced number, so the target is -still findable by title. The tool description tells the agent the dropped -fields are in the ToC JSON at `json_path`. It is a denylist so that a new -enrichment reaches the agent unless someone decides it should not. +reads. Only section references are ever resolved (`target_node_id`, 988 of +2,248), and only to a section whose title starts with the referenced number, +so the target is still findable by title; table and figure references carry +only their label. The tool description says these fields are left out and +sends the agent to the page text -- not to `json_path`, which no tool reads +and which may not resolve where the agent runs. It is a denylist so that a +new enrichment reaches the agent unless someone decides it should not. Measured before merging (2026-11-09): 10 questions chosen so that their answers sit in a footnote or a cross-referenced table, 9 of them on large-ToC diff --git a/src/datasheetindex/tools/defs.py b/src/datasheetindex/tools/defs.py index bb967f2..01483a8 100644 --- a/src/datasheetindex/tools/defs.py +++ b/src/datasheetindex/tools/defs.py @@ -81,11 +81,15 @@ #: ``node_id``. ``level`` is the outline's own depth number, which differs from #: the nesting only when an outline skips a level -- the nesting is what the #: agent navigates by. ``cross_references`` and ``footnote_markers`` are a -#: judgement call: their text is in the pages the agent reads, and the 988 of -#: 2,248 references that resolve (``target_node_id``) point at a section whose -#: title starts with the referenced number, so the target is still findable by -#: title. A denylist, not an allowlist, so a new enrichment reaches the agent -#: unless deliberately cut. +#: judgement call: their text is in the pages the agent reads. Only section +#: references are ever resolved (``target_node_id``, 988 of 2,248 on the +#: corpus), and only to a section whose title starts with the referenced +#: number, so the target is still findable by title; table and figure +#: references carry only their label. The description points the agent at +#: the page text, not at ``json_path``: no tool reads that file, and the path +#: may not resolve on the agent's side (see the preamble's pending-work note). +#: A denylist, not an allowlist, so a new enrichment reaches the agent unless +#: deliberately cut. #: ``DatasheetTools.get_artifact_manifest`` and the ToC JSON keep every field. _AGENT_TOC_OMITTED_FIELDS = frozenset( { @@ -110,10 +114,10 @@ def _agent_toc(nodes: list[dict[str, Any]]) -> list[dict[str, Any]]: out = { key: value for key, value in node.items() - if key not in _AGENT_TOC_OMITTED_FIELDS - and key != "nodes" - and not (key == "table_count" and not value) + if key not in _AGENT_TOC_OMITTED_FIELDS and key != "nodes" } + if not out.get("table_count"): + out.pop("table_count", None) if node.get("nodes"): out["nodes"] = _agent_toc(node["nodes"]) slim.append(out) @@ -398,9 +402,9 @@ def _close() -> None: "pdf_source switches documents.\n\n" "The manifest carries the source, total pages, ToC quality, and " "the enriched ToC -- section hierarchy, page " - "ranges, nonzero table counts, boilerplate flags; breadcrumbs, " - "cross-references and footnote markers are only in the ToC " - "JSON at json_path -- " + "ranges, nonzero table counts, boilerplate flags (no " + "breadcrumbs, cross-references or footnote markers: read those " + "in the page text) -- " "plus a 'figures' digest: total/raster/captioned counts and up " "to 40 {page, figures, caption} rows ('truncated' says when " "more exist), each row carrying that page's largest figure. A " @@ -561,8 +565,8 @@ def _close() -> None: name="search_text", description=( "Locate a term across the datasheet: returns page-aware snippets " - "with surrounding context, each tagged with the ToC 'breadcrumb' " - "of the section holding the match (and, for a list query, the " + "with surrounding context, each tagged with the 'breadcrumb' " + "path of the section holding the match (and, for a list query, the " "'pattern' that matched).\n\n" "Searches the extracted text layer only. A table, schematic or " "label placed as an image has no text layer, so a term inside it " diff --git a/tests/test_defs.py b/tests/test_defs.py index d5d8faf..0286c3a 100644 --- a/tests/test_defs.py +++ b/tests/test_defs.py @@ -542,14 +542,10 @@ def test_agent_manifest_toc_drops_derivable_and_bulky_fields(tmp_path): "text" ] ) + full = json.loads( + open(manifest["json_path"], encoding="utf-8").read() # noqa: SIM115 + )["toc"] - nodes = list(_walk(manifest["toc"])) - assert [n["title"] for n in nodes] == [ - "1 Electrical characteristics", - "1.1 Supply", - "2 Revision history", - ] - assert manifest["toc"][0]["nodes"][0]["title"] == "1.1 Supply" dropped = { "breadcrumb", "level", @@ -558,6 +554,18 @@ def test_agent_manifest_toc_drops_derivable_and_bulky_fields(tmp_path): "cross_references", "footnote_markers", } + # Guard against a vacuous pass: the unslimmed tree really had every field. + present = set().union(*(n.keys() for n in _walk(full))) + assert dropped <= present, dropped - present + assert any(n.get("table_count") == 0 for n in _walk(full)) + + nodes = list(_walk(manifest["toc"])) + assert [n["title"] for n in nodes] == [ + "1 Electrical characteristics", + "1.1 Supply", + "2 Revision history", + ] + assert manifest["toc"][0]["nodes"][0]["title"] == "1.1 Supply" for node in nodes: assert not dropped & node.keys(), node assert {"start_page", "end_page"} <= node.keys() @@ -600,7 +608,7 @@ def test_agent_toc_passes_unknown_fields_through(): "table_count": 0, "has_tables": False, "summary": "s", - "continued_tables": [{"page": 2}], + "continued_tables": ["Table 3"], "future_signal": True, "nodes": [{"title": "B", "start_page": 2, "end_page": 2, "level": 2}], } @@ -611,7 +619,7 @@ def test_agent_toc_passes_unknown_fields_through(): "start_page": 1, "end_page": 2, "summary": "s", - "continued_tables": [{"page": 2}], + "continued_tables": ["Table 3"], "future_signal": True, "nodes": [{"title": "B", "start_page": 2, "end_page": 2}], } @@ -624,15 +632,19 @@ def test_agent_toc_passes_unknown_fields_through(): def test_build_datasheet_description_matches_the_slim_toc(): """The description must not promise dropped fields as manifest content. - It names them once, as living in the ToC JSON, so an agent that needs a - breadcrumb or cross-reference knows where to look. Checked against the - denylist so a field added there without a description update fails here. + It names them once, as absent, and sends the agent to the page text -- a + place it can reach. Not to json_path: no tool reads that file, and the + path may not resolve where the agent runs. Checked against the denylist so + a field added there without a description update fails here. """ from datasheetindex.tools.defs import _AGENT_TOC_OMITTED_FIELDS description = _defs_by_name()["build_datasheet"].description - manifest_part, _, rest = description.partition("only in the ToC JSON at json_path") - assert rest, "description no longer says where the omitted fields live" + toc_part = description[: description.index("'figures' digest")] + before, absent, _ = toc_part.partition("(no ") + assert absent, "description no longer says which fields are left out" + note = toc_part[len(before) :] + assert "read those in the page text" in note spoken = { "breadcrumb": "breadcrumbs", "cross_references": "cross-references", @@ -640,10 +652,8 @@ def test_build_datasheet_description_matches_the_slim_toc(): } for field, phrase in spoken.items(): assert field in _AGENT_TOC_OMITTED_FIELDS - # Named in the json_path sentence, not earlier as manifest content. - sentence_start = manifest_part.rfind(";") - assert phrase in manifest_part[sentence_start:], phrase - assert phrase not in manifest_part[:sentence_start], phrase + assert phrase in note, phrase + assert phrase not in before, phrase def test_build_datasheet_schema_bounds_max_figure_captions(): From 685b9ed4c7e5d72fbcb65915d6c35f2f6434fbc4 Mon Sep 17 00:00:00 2001 From: yeqi Date: Mon, 9 Nov 2026 14:25:59 +0100 Subject: [PATCH 3/4] docs: cite the selective-reading paper that prompted the manifest measurement --- docs/datasheetindex_architecture.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/datasheetindex_architecture.md b/docs/datasheetindex_architecture.md index 140c046..dbcf0f4 100644 --- a/docs/datasheetindex_architecture.md +++ b/docs/datasheetindex_architecture.md @@ -1562,6 +1562,18 @@ table misread, a wrong page and a text-layout quirk, none involving the cut. Turns did not move (3.94 -> 3.90, 4.04 -> 4.02), and mean prompt tokens per question fell 2.7-2.9x (206K -> 76K, 168K -> 58K). +The measurement was prompted by Belcak et al., "You don't need long context" +(arXiv:2611.05306, 2026). Their agent reads long inputs selectively, with grep +and bounded windows of about 6,000 characters, instead of holding the whole +input in context, and does as well or better: 84.0% vs 65.0% on AA-LCR with +Opus 4.5. That supports this library's split, where the agent navigates with +tools rather than ingesting the document. The paper does not test PDFs, tables +or any pre-processing, so it says nothing either way about the enriched ToC. +Its specific remedy did not carry over: a read-size cap was measured here and +not adopted, because agents already read narrowly (largest read 47K +characters, most under 10K) and the context was being filled by the manifest +ToC instead. + `search_text` also carries the text-layer limitation on **both** surfaces its consumer reads: in the tool description, and — when a search returns nothing on a document that holds raster regions — as a `note` on the result itself. The From 0873698f7d7b943cb398372a1142f1acadcc3288 Mon Sep 17 00:00:00 2001 From: yeqi Date: Mon, 9 Nov 2026 16:25:41 +0100 Subject: [PATCH 4/4] docs: larger A/B and re-measured token economy for the slim manifest A/B re-run at 360 runs (Sonnet 4.6, GPT-5.1, Qwen 3.6 27B; 20 questions over 16 documents, 3 runs per question per arm): 175/180 correct in both arms, every miss read by hand and none involving the dropped fields; prompt tokens per question down 2.5-2.7x. scripts/token_economy.py priced get_artifact_manifest(), which is no longer what the agent is sent. It now goes through _agent_manifest(), the function the build_datasheet handler calls, so the study cannot drift from the tool. Regenerated without LLM credentials and with default scans: median first answer 9,637 -> 4,293 tokens (3.1x -> 7.3x cheaper than the document). --- CHANGELOG.md | 14 +++++-- README.md | 16 ++++---- docs/datasheetindex_architecture.md | 34 +++++++++++------ docs/token-economy.md | 58 ++++++++++++++--------------- scripts/token_economy.py | 14 ++++--- src/datasheetindex/tools/defs.py | 20 +++++++--- 6 files changed, 94 insertions(+), 62 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d98a3c9..1dd0144 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,9 +13,17 @@ All notable changes to this project will be documented in this file. not on that list. The manifest stays in the conversation and is re-sent on every turn; on large documents it was most of the agent's context (270K characters on PIC16F887, 192K on ESP32 TRM) and is now about 3.5x smaller. - On a before/after test with Sonnet 4.6 and GPT-5.1, over questions whose - answers sit in footnotes and cross-referenced tables, accuracy and turn - counts were unchanged while prompt tokens per question fell 2.7-2.9x. + On a before/after test of 360 runs (Sonnet 4.6, GPT-5.1, Qwen 3.6 27B; 20 + questions, most answered from footnotes and cross-referenced tables), + accuracy was 175/180 in both arms, turns fell slightly, and prompt tokens + per question fell 2.5-2.7x. +- **Token economy re-measured** (`docs/token-economy.md`, README). The + script now prices the manifest the agent is actually sent, through the same + function the tool handler uses, rather than `get_artifact_manifest()`. The + median first answer (map plus one section) is now 4,293 tokens, 7.3x + cheaper than the document (was 9,637, 3.1x). The further-answer median + moved from 625 to 760 tokens because sections grew in earlier releases, not + in this one. - Applies to the MCP server, `create_datasheet_tool_defs()` and the Agent SDK server. `DatasheetTools.get_artifact_manifest()` and the ToC JSON artifact are unchanged and keep every field; `search_text` hits still carry their diff --git a/README.md b/README.md index e7ae654..087b922 100644 --- a/README.md +++ b/README.md @@ -59,11 +59,11 @@ credentials. Full results, per document, are in | Median across the corpus | | |---|---| | Whole document in context | 33,973 tokens | -| First answer (map + one section) | 9,637 tokens | -| Every further answer about the same part | 625 tokens | -| First answer vs. the document | **3.1x** cheaper | -| Further answer vs. the document | **54x** cheaper | -| Build | ~6s cold, 0.01s warm (cached artifact) | +| First answer (map + one section) | 4,293 tokens | +| Every further answer about the same part | 760 tokens | +| First answer vs. the document | **7.3x** cheaper | +| Further answer vs. the document | **47x** cheaper | +| Build | ~5s cold, 0.01s warm (cached artifact) | Each ratio is the **median of the per-document ratios**, so it is deliberately not the quotient of the two token medians above: every document is priced on @@ -76,10 +76,10 @@ an order of magnitude and a cold/warm ratio. **The two ratios are different claims, and the first one is the weaker.** The first answer pays for the enriched ToC as well as the section, and on a large -document that map is most of the cost -- for the 322-page PIC16F887 it is half -the document's tokens on its own. The compounding is in the second row: the +document that map is most of the cost -- for the 322-page PIC16F887 it is 27K +of a 28K-token first answer, though only 15% of the document. The compounding is in the second row: the map is paid once, so the tenth question about a part costs a section alone. -That is 45x on a 31-page datasheet and 728x on a 784-page reference manual, +That is 39x on a 31-page datasheet and 518x on a 784-page reference manual, which is the shape to expect -- the bigger the document, the more there is not to read. diff --git a/docs/datasheetindex_architecture.md b/docs/datasheetindex_architecture.md index dbcf0f4..032667c 100644 --- a/docs/datasheetindex_architecture.md +++ b/docs/datasheetindex_architecture.md @@ -1549,18 +1549,28 @@ sends the agent to the page text -- not to `json_path`, which no tool reads and which may not resolve where the agent runs. It is a denylist so that a new enrichment reaches the agent unless someone decides it should not. -Measured before merging (2026-11-09): 10 questions chosen so that their -answers sit in a footnote or a cross-referenced table, 9 of them on large-ToC -documents, Sonnet 4.6 and GPT-5.1. Run counts per arm are unequal (50-59) and -not exactly paired: the run was cut short by a harness bug that duplicated -runs. Sonnet was 51/51 before and 52/52 after. GPT was 42/50 (84%) before and -48/59 (81%) after, with per-question changes in both directions (q0 3/5 -> -4/6, q6 3/5 -> 2/6) -- no measurable effect at this n, not a demonstrated -zero. Some GPT misses were grading artefacts (it writes U+202F between number -and unit) and hit both arms; the after-arm misses read by hand were a register -table misread, a wrong page and a text-layout quirk, none involving the cut. -Turns did not move (3.94 -> 3.90, 4.04 -> 4.02), and mean prompt tokens per -question fell 2.7-2.9x (206K -> 76K, 168K -> 58K). +Measured before merging (2026-11-09): 20 questions across 16 documents, 15 of +them chosen so the answer sits in a footnote or a cross-referenced table, 3 +models, 3 runs per question per arm -- 360 runs, 180 per arm, full manifest +(`main`) against slim (this change): + +| Model | Correct before -> after | Turns | Prompt tokens / question | +|---|---|---|---| +| Sonnet 4.6 | 60/60 -> 60/60 | 3.67 -> 3.50 | 129K -> 48K | +| GPT-5.1 | 55/60 -> 56/60 | 4.00 -> 3.93 | 121K -> 46K | +| Qwen 3.6 27B | 60/60 -> 59/60 | 5.42 -> 5.38 | 202K -> 82K | + +175/180 correct in both arms. Every miss was read by hand, and none involves +the dropped fields: GPT misses the same four questions in both arms (an +answer from priors instead of the note on the page, a value the text layer +puts on the wrong line, a register's generic defaults read instead of the +per-pin table, a curve read instead of a table), and Qwen's one miss is that +same register misread. Turns fell slightly in every model, and prompt tokens +per question fell 2.5-2.7x. Grading is by string match with exotic spaces, +hyphens and markdown normalised; one question (ESP32 GPIO2 reset state) was +graded by hand, because correct answers quote the register defaults before +concluding. An earlier, smaller run of the same design (10 questions, 2 +models) agreed. The measurement was prompted by Belcak et al., "You don't need long context" (arXiv:2611.05306, 2026). Their agent reads long inputs selectively, with grep diff --git a/docs/token-economy.md b/docs/token-economy.md index b7d90fb..1300902 100644 --- a/docs/token-economy.md +++ b/docs/token-economy.md @@ -5,44 +5,44 @@ Tokens counted with the `o200k_base` encoding. 25 documents, 24 priced. | Document | Pages | Full doc | Manifest | Section (med) | Section (p90) | First answer | vs full | Further answer | vs full | Build (cold) | Build (warm) | |---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| -| vishay_1n4001.pdf | 5 | 2,683 | 386 | 0 | 0 | n/a | n/a | n/a | n/a | 0.9s | 0.01s | -| raspi_pico.pdf | 31 | 13,856 | 4,917 | 307 | 877 | 5,224 | 2.7x | 307 | 45x | 5.0s | 0.03s | -| ti_ina219.pdf | 38 | 23,860 | 7,100 | 634 | 1,050 | 7,734 | 3.1x | 634 | 38x | 5.1s | 0.01s | -| ti_ne555.pdf | 39 | 17,830 | 5,561 | 556 | 941 | 6,117 | 2.9x | 556 | 32x | 4.1s | 0.01s | -| ti_sn74hc595.pdf | 41 | 16,273 | 3,437 | 532 | 926 | 3,969 | 4.1x | 532 | 31x | 4.4s | 0.01s | -| ti_tcan1044a.pdf | 42 | 20,040 | 7,502 | 545 | 945 | 8,047 | 2.5x | 545 | 37x | 4.0s | 0.01s | -| ti_lm317.pdf | 44 | 15,411 | 6,831 | 408 | 746 | 7,239 | 2.1x | 408 | 38x | 4.8s | 0.01s | -| ti_opa2340.pdf | 48 | 23,025 | 6,856 | 786 | 1,089 | 7,642 | 3.0x | 786 | 29x | 4.7s | 0.01s | -| bosch_bmp280.pdf | 49 | 20,829 | 7,851 | 609 | 744 | 8,460 | 2.5x | 609 | 34x | 4.7s | 0.01s | -| ti_tps54360.pdf | 50 | 26,643 | 8,363 | 760 | 1,242 | 9,123 | 2.9x | 760 | 35x | 6.0s | 0.01s | -| ti_ads1115.pdf | 57 | 33,973 | 10,096 | 776 | 992 | 10,872 | 3.1x | 776 | 44x | 5.8s | 0.01s | -| bosch_bme280.pdf | 60 | 27,975 | 10,722 | 550 | 792 | 11,272 | 2.5x | 550 | 51x | 5.9s | 0.02s | -| ti_lm358.pdf | 68 | 46,506 | 4,931 | 778 | 1,630 | 5,709 | 8.1x | 778 | 60x | 7.2s | 0.01s | -| ti_cc2640r2f.pdf | 72 | 37,132 | 9,463 | 689 | 992 | 10,152 | 3.7x | 689 | 54x | 10.9s | 0.01s | -| esp32c3_ds.pdf | 76 | 32,595 | 12,899 | 462 | 746 | 13,361 | 2.4x | 462 | 71x | 5.7s | 0.01s | -| esp32_ds.pdf | 78 | 40,373 | 9,958 | 617 | 1,391 | 10,575 | 3.8x | 617 | 65x | 6.5s | 0.01s | -| infineon_psc3.pdf | 90 | 55,792 | 7,766 | 737 | 1,839 | 8,503 | 6.6x | 737 | 76x | 9.6s | 0.01s | -| ti_tlv9061.pdf | 99 | 41,159 | 5,369 | 750 | 1,944 | 6,119 | 6.7x | 750 | 55x | 9.8s | 0.02s | -| esp8266_trm.pdf | 111 | 53,385 | 13,715 | 515 | 912 | 14,230 | 3.8x | 515 | 104x | 6.5s | 0.01s | -| bosch_bmi160.pdf | 114 | 57,582 | 16,811 | 581 | 827 | 17,392 | 3.3x | 581 | 99x | 28.1s | 0.01s | -| ti_msp430f5529.pdf | 143 | 101,951 | 20,469 | 906 | 1,568 | 21,375 | 4.8x | 906 | 113x | 11.3s | 0.02s | -| micro_atmega328.pdf | 294 | 199,004 | 69,229 | 742 | 1,062 | 69,971 | 2.8x | 742 | 268x | 18.3s | 0.03s | -| micro_pic16f887.pdf | 322 | 185,565 | 92,177 | 680 | 1,047 | 92,857 | 2.0x | 680 | 273x | 30.3s | 0.03s | -| raspi_rp2040.pdf | 642 | 400,095 | 46,518 | 689 | 2,796 | 47,207 | 8.5x | 689 | 581x | 36.8s | 0.03s | -| esp32_trm.pdf | 784 | 401,974 | 64,763 | 552 | 1,512 | 65,315 | 6.2x | 552 | 728x | 86.9s | 0.03s | +| vishay_1n4001.pdf | 5 | 2,683 | 359 | 0 | 0 | n/a | n/a | n/a | n/a | 0.8s | 0.01s | +| raspi_pico.pdf | 31 | 13,856 | 2,152 | 357 | 1,118 | 2,509 | 5.5x | 357 | 39x | 4.1s | 0.04s | +| ti_ina219.pdf | 38 | 23,860 | 2,744 | 745 | 1,238 | 3,489 | 6.8x | 745 | 32x | 3.1s | 0.01s | +| ti_ne555.pdf | 39 | 17,830 | 2,132 | 598 | 981 | 2,730 | 6.5x | 598 | 30x | 3.2s | 0.01s | +| ti_sn74hc595.pdf | 41 | 16,273 | 1,599 | 635 | 1,022 | 2,234 | 7.3x | 635 | 26x | 3.6s | 0.01s | +| ti_tcan1044a.pdf | 42 | 20,040 | 2,643 | 621 | 1,070 | 3,264 | 6.1x | 621 | 32x | 3.2s | 0.01s | +| ti_lm317.pdf | 44 | 15,411 | 2,956 | 408 | 765 | 3,364 | 4.6x | 408 | 38x | 4.1s | 0.01s | +| ti_opa2340.pdf | 48 | 23,025 | 2,456 | 916 | 1,518 | 3,372 | 6.8x | 916 | 25x | 3.9s | 0.01s | +| bosch_bmp280.pdf | 49 | 20,829 | 3,633 | 657 | 1,042 | 4,290 | 4.9x | 657 | 32x | 4.0s | 0.02s | +| ti_tps54360.pdf | 50 | 26,643 | 3,122 | 779 | 1,448 | 3,901 | 6.8x | 779 | 34x | 4.8s | 0.01s | +| ti_ads1115.pdf | 57 | 33,973 | 3,976 | 821 | 1,301 | 4,797 | 7.1x | 821 | 41x | 4.0s | 0.01s | +| bosch_bme280.pdf | 60 | 27,975 | 4,348 | 608 | 1,258 | 4,956 | 5.6x | 608 | 46x | 5.0s | 0.02s | +| ti_lm358.pdf | 68 | 46,506 | 2,062 | 801 | 1,772 | 2,863 | 16.2x | 801 | 58x | 5.9s | 0.02s | +| ti_cc2640r2f.pdf | 72 | 37,132 | 3,837 | 776 | 1,373 | 4,613 | 8.0x | 776 | 48x | 5.8s | 0.02s | +| esp32c3_ds.pdf | 76 | 32,595 | 3,817 | 656 | 1,116 | 4,473 | 7.3x | 656 | 50x | 6.4s | 0.01s | +| esp32_ds.pdf | 78 | 40,373 | 3,463 | 834 | 1,530 | 4,297 | 9.4x | 834 | 48x | 6.1s | 0.01s | +| infineon_psc3.pdf | 90 | 55,792 | 3,370 | 908 | 2,110 | 4,278 | 13.0x | 908 | 61x | 6.9s | 0.02s | +| ti_tlv9061.pdf | 99 | 41,159 | 2,464 | 934 | 2,057 | 3,398 | 12.1x | 934 | 44x | 8.3s | 0.02s | +| esp8266_trm.pdf | 111 | 53,385 | 4,624 | 579 | 1,011 | 5,203 | 10.3x | 579 | 92x | 4.7s | 0.02s | +| bosch_bmi160.pdf | 114 | 57,582 | 6,927 | 695 | 1,151 | 7,622 | 7.6x | 695 | 83x | 23.3s | 0.01s | +| ti_msp430f5529.pdf | 143 | 101,951 | 6,774 | 1,026 | 1,639 | 7,800 | 13.1x | 1,026 | 99x | 8.5s | 0.02s | +| micro_atmega328.pdf | 294 | 199,004 | 18,259 | 801 | 1,270 | 19,060 | 10.4x | 801 | 248x | 15.3s | 0.03s | +| micro_pic16f887.pdf | 322 | 185,205 | 27,452 | 725 | 1,078 | 28,177 | 6.6x | 725 | 255x | 27.0s | 0.03s | +| raspi_rp2040.pdf | 642 | 400,095 | 12,184 | 891 | 3,173 | 13,075 | 30.6x | 891 | 449x | 31.2s | 0.03s | +| esp32_trm.pdf | 784 | 401,974 | 19,301 | 776 | 1,727 | 20,077 | 20.0x | 776 | 518x | 38.8s | 0.04s | ## Headline - Median full document: **33,973** tokens -- Median **first** answer (manifest + one section): **9,637** tokens, **3.1x** cheaper than the document (range 2.0x-8.5x) -- Median **further** answer about the same part (one section, manifest already in context): **625** tokens, **54x** cheaper -- Median build: **6.01s** cold, **0.01s** warm (25 of 25 cache hits) +- Median **first** answer (manifest + one section): **4,293** tokens, **7.3x** cheaper than the document (range 4.6x-30.6x) +- Median **further** answer about the same part (one section, manifest already in context): **760** tokens, **47x** cheaper +- Median build: **5.04s** cold, **0.01s** warm (25 of 25 cache hits) ## How to read this **`Full doc` is a conservative baseline.** It is the page-matched text file -- what you paste in to read the document. Attaching the PDF itself costs more, because the pages arrive as images too. -**The two ratios answer different questions.** The first answer pays for the map as well as the section, and on large documents the map is most of the cost: the enriched ToC of a 322-page datasheet is itself half that document's tokens. Every question after it about the same part pays for a section alone, which is where the ratio gets large. An agent that asks one thing and leaves sees the first ratio; an agent doing real extraction work sees the second. +**The two ratios answer different questions.** The first answer pays for the map as well as the section, and on large documents the map is most of the cost: for a 322-page datasheet it is 27K of a 28K-token first answer, though only 15% of the document. Every question after it about the same part pays for a section alone, which is where the ratio gets large. An agent that asks one thing and leaves sees the first ratio; an agent doing real extraction work sees the second. **The ratios are medians of per-document ratios, not the quotient of the medians above.** Each document is priced on its own and the ratios are then summarised, which is why dividing the two token medians gives a different -- and less representative -- number: a median of the per-document ratios is not the ratio of the medians, and the token medians are taken over different sets, since an unpriced document still has a full-document count. diff --git a/scripts/token_economy.py b/scripts/token_economy.py index 2d64e04..0fd4747 100644 --- a/scripts/token_economy.py +++ b/scripts/token_economy.py @@ -11,8 +11,10 @@ the datasheet". This is a *conservative* baseline: attaching the PDF itself costs more, since the pages arrive as images too. ``manifest`` - What ``get_artifact_manifest`` actually hands the agent -- the enriched - ToC and the build's signals. + What the ``build_datasheet`` agent tool actually hands the agent -- the + enriched ToC, slimmed as the tool slims it, and the build's signals. + Measured through the same function the tool handler calls, so it is not + the Python API's fuller ``get_artifact_manifest``. ``section`` What ``get_section_text`` returns for one leaf ToC section, summarised across every leaf as a median and a p90. @@ -217,6 +219,7 @@ def measure_document( the published ones. """ from datasheetindex import DatasheetTools + from datasheetindex.tools.defs import _agent_manifest pdf_path = Path(pdf_path) notes: list[str] = [] @@ -230,7 +233,7 @@ def measure_document( ) cold_build_s = time.perf_counter() - started - manifest = tools.get_artifact_manifest() + manifest = _agent_manifest(tools) # ``json_path``/``text_path`` are absolute and machine-specific. They # are a handful of tokens and the agent really is sent them, so they # stay in the count rather than being trimmed to make the number look @@ -446,8 +449,9 @@ def render_markdown( "", "**The two ratios answer different questions.** The first answer pays " "for the map as well as the section, and on large documents the map " - "is most of the cost: the enriched ToC of a 322-page datasheet is " - "itself half that document's tokens. Every question after it about " + "is most of the cost: for a 322-page datasheet it is 27K of a " + "28K-token first answer, though only 15% of the document. Every " + "question after it about " "the same part pays for a section alone, which is where the ratio " "gets large. An agent that asks one thing and leaves sees the first " "ratio; an agent doing real extraction work sees the second.", diff --git a/src/datasheetindex/tools/defs.py b/src/datasheetindex/tools/defs.py index 01483a8..220c06f 100644 --- a/src/datasheetindex/tools/defs.py +++ b/src/datasheetindex/tools/defs.py @@ -124,6 +124,20 @@ def _agent_toc(nodes: list[dict[str, Any]]) -> list[dict[str, Any]]: return slim +def _agent_manifest(tools: DatasheetTools) -> dict[str, object]: + """The manifest exactly as ``build_datasheet`` sends it to the agent. + + One function so the handler and ``scripts/token_economy.py`` cannot drift: + the study prices what the agent is really sent, not the Python API's + fuller manifest. + """ + manifest = tools.get_artifact_manifest() + toc = manifest.get("toc") + if isinstance(toc, list): + manifest["toc"] = _agent_toc(toc) + return manifest + + @dataclass(frozen=True) class DatasheetToolDef: """A single datasheet tool, described independently of any agent framework. @@ -294,11 +308,7 @@ async def build_datasheet(args: dict[str, Any]) -> dict[str, Any]: tools_instance = target if previous is not None: _safe_close(previous) - manifest = tools_instance.get_artifact_manifest() - toc = manifest.get("toc") - if isinstance(toc, list): - manifest["toc"] = _agent_toc(toc) - return _ok(manifest) + return _ok(_agent_manifest(tools_instance)) except Exception as exc: return _err_exc(exc)