diff --git a/.env.example b/.env.example index a2eb421..2d82fc5 100644 --- a/.env.example +++ b/.env.example @@ -43,6 +43,16 @@ LITELLM_BASE_URL=https://your-litellm-gateway.example.com # before any caption text is written. # DATASHEETINDEX_VISION_MODEL= +# Optional. Folders the MCP server searches with its find_documents tool, +# separated like PATH (";" on Windows, ":" elsewhere). Unset or blank means the +# tool is not offered. Read by the MCP server only. +# DATASHEETINDEX_LIBRARY_ROOTS= + +# --------------------------------------------------------------------------- +# Advanced: gateway compatibility. Rarely needed -- every setting below has a +# working default; set one only when your gateway or model needs it. +# --------------------------------------------------------------------------- + # Optional generation settings. Text (including structured ToCs) and vision # reasoning are INDEPENDENT; unset/blank means omit the field, not "none". # Supported spellings: none, minimal, low, medium, high, xhigh, max. Only set diff --git a/CHANGELOG.md b/CHANGELOG.md index d456135..a585d5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,36 @@ All notable changes to this project will be documented in this file. +## [0.46.0] - 2026-11-05 + +### Added +- **`find_documents` MCP tool.** Lists PDFs under the folders named in the new + `DATASHEETINDEX_LIBRARY_ROOTS` setting (separated like `PATH`), matching every + query term against the file name and folder path, so an agent whose shell + cannot see a network share can still find the PDF to pass to + `build_datasheet`. Results carry the full server-side path, last-modified date + and size, and a status per root. +- **On-disk library cache.** One file per root, valid for one day, refreshed in + a background thread at server startup and on `refresh=true`. An unreachable + root or unreadable folder is reported in the response, never raised; a PDF + that cannot be read or whose name is not valid UTF-8 is left out and counted + in `skipped_files`. +- `DATASHEETINDEX_LIBRARY_ROOTS` (and `DATASHEETINDEX_OUTPUT_DIR`, which places + the cache) can be set in `.env` on a source checkout (the server loads it at + startup); an installed server takes them from the MCP client config's `env` + block. +- Nothing changes when `DATASHEETINDEX_LIBRARY_ROOTS` is unset or blank: the + tool is not registered and the server serves its five tools as before. The + tool is MCP-server-only and is not in `create_datasheet_tool_defs()`. + +### Changed +- **Shorter configuration docs.** The README and `.env.example` now lead with + the five settings most deployments need (`LITELLM_BASE_URL`, + `LITELLM_MASTER_KEY`, `DATASHEETINDEX_MODEL`, `DATASHEETINDEX_VISION_MODEL`, + `LITELLM_TLS_VERIFY`). The gateway-tuning variables (reasoning effort, output + budgets, token-limit spelling, `store`, timeout and retries) moved to an + "Advanced: gateway compatibility" section. No variable was removed or renamed. + ## [0.45.0] - 2026-11-03 ### Fixed diff --git a/README.md b/README.md index b7660a3..9e44d72 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Agent-first parameter extraction from technical datasheets. - [What it does](#what-it-does) · [Philosophy](#philosophy) · [Supported products](#supported-products) - [Token economy](#token-economy) · [Benchmark](#benchmark) · [Links](#links) - **Getting started** — [Setup](#setup) · [Development](#development) · [Input sources](#input-sources) -- **Using it from an agent** — [Hand the MCP server to an agent](#hand-the-mcp-server-to-an-agent) · [Realize the tools without the Claude Agent SDK](#realize-the-tools-without-the-claude-agent-sdk) · [Run a local MCP server](#run-a-local-mcp-server) +- **Using it from an agent** — [Hand the MCP server to an agent](#hand-the-mcp-server-to-an-agent) · [Realize the tools without the Claude Agent SDK](#realize-the-tools-without-the-claude-agent-sdk) · [Run a local MCP server](#run-a-local-mcp-server) · [Find documents in a library](#find-documents-in-a-library) - **What the artifacts tell an agent** — [Datasheets without a ToC](#datasheets-without-a-toc) · [Knowing where the ToC came from](#knowing-where-the-toc-came-from) · [Asking for a better ToC](#asking-for-a-better-toc) · [Datasheets that cover a product family](#datasheets-that-cover-a-product-family) · [Figure indexing and captions](#figure-indexing-and-captions) - **Reference** — [Python API](#python-api) · [CLI](#cli) · [Project structure](#project-structure) · [License](#license) @@ -177,15 +177,19 @@ uv sync --extra mcp uv pip install claude-agent-sdk ``` -For LLM-backed ToC fallback, summaries and figure captions, configure -`LITELLM_BASE_URL` and `LITELLM_MASTER_KEY` (see `.env.example`). Two optional -variables name the models, since which ones a gateway serves is a property of -your deployment rather than of this library: +For LLM-backed ToC fallback, summaries and figure captions, these are the only +settings most deployments need (see `.env.example`): -| variable | names | default | +| variable | sets | default | |---|---|---| -| `DATASHEETINDEX_MODEL` | summaries and the ToC fallback | `qwen3.8-27b` | -| `DATASHEETINDEX_VISION_MODEL` | figure captions only | follows the text model | +| `LITELLM_BASE_URL` | the gateway URL | required | +| `LITELLM_MASTER_KEY` | your gateway key | required | +| `DATASHEETINDEX_MODEL` | model for summaries and the ToC fallback | `qwen3.8-27b` | +| `DATASHEETINDEX_VISION_MODEL` | model for figure captions only | follows the text model | +| `LITELLM_TLS_VERIFY` | gateway certificate verification | on | + +The model names are settings because which models a gateway serves is a +property of your deployment rather than of this library. `build_datasheet`'s `model` argument overrides `DATASHEETINDEX_MODEL` for one call; the vision model is deployment-level only, and deliberately not a tool @@ -195,10 +199,22 @@ Deployments that do not serve `qwen3.8-27b` must explicitly name an available model with `DATASHEETINDEX_MODEL`. Unless separately overridden, captions use that same model and require image-input support. -### Chat Completions configuration +TLS verification is **on** by default. `LITELLM_TLS_VERIFY=false` (also `0`, +`no`, `off`) turns it off, for a gateway whose certificate the local trust store +cannot accept -- an internal CA missing from the image, or a self-signed cert on +localhost. Adding that CA to the trust store is the better fix: `LITELLM_MASTER_KEY` +travels on this channel. A certificate that fails verification raises +`LlmTlsVerificationError` (exported from the package root) naming the gateway +and both remedies, rather than degrading to an index with no ToC entries. Figure +captioning is the exception: it still degrades, because the artifact is complete +without captions. -All library LLM calls use Chat Completions, including images and strict JSON -schemas; there is no automatic fallback to a different API or different model. +### Advanced: gateway compatibility + +Rarely needed: these settings exist for gateways or models that need different +request parameters, and every one has a working default. All library LLM calls +use Chat Completions, including images and strict JSON schemas; there is no +automatic fallback to a different API or different model. | variable | controls | default | |---|---|---| @@ -273,16 +289,6 @@ threads, so use a thread-safe collector in applications. Callback exceptions are ignored with a payload-free warning. Optional metadata fields remain `None` when the gateway does not supply them. -TLS verification is **on** by default. `LITELLM_TLS_VERIFY=false` (also `0`, -`no`, `off`) turns it off, for a gateway whose certificate the local trust store -cannot accept -- an internal CA missing from the image, or a self-signed cert on -localhost. Adding that CA to the trust store is the better fix: `LITELLM_MASTER_KEY` -travels on this channel. A certificate that fails verification raises -`LlmTlsVerificationError` (exported from the package root) naming the gateway -and both remedies, rather than degrading to an index with no ToC entries. Figure -captioning is the exception: it still degrades, because the artifact is complete -without captions. - `claude-agent-sdk` is only required for the SDK-flavored handoff (`create_datasheet_tools_server`). The tool logic itself is framework-neutral -- non-SDK hosts get it via `create_datasheet_tool_defs()` with no SDK import (see @@ -513,6 +519,51 @@ server object inside another Python runtime, use `create_datasheet_tools_server()` instead; it exposes the same tool surface, with the document loaded at runtime via the `build_datasheet` tool. +### Find documents in a library + +`build_datasheet` needs a PDF path the server can open. When the datasheets sit +on a network share, the agent's own shell often cannot see it (a Linux dev +container, say), so it cannot tell which PDF to load -- but the server can. +Set `DATASHEETINDEX_LIBRARY_ROOTS` and the server adds a sixth tool, +`find_documents`, which lists PDFs under those folders. Unset or blank, the +tool is not registered and the server serves exactly its five datasheet tools. + +```json +{ + "servers": { + "datasheetindex": { + "command": "uvx", + "args": ["--from", "datasheetindex[mcp]", "datasheetindex-mcp-server"], + "env": { + "DATASHEETINDEX_LIBRARY_ROOTS": "\\\\server\\share\\Datasheets;D:\\Manuals" + } + } + } +} +``` + +- Several roots are separated like `PATH`: `;` on Windows, `:` on Linux and macOS. + A server running under WSL (or any Linux) therefore takes `/mnt/z/...`-style + roots: a `Z:\...` root would split at its colon. Quotes around a root and + blank entries are tolerated. On a source checkout the variable may also be set + in `.env`; an installed server takes it from the MCP client config's `env` + block. +- A mapped drive (`Z:\...`) works only if the server process itself sees the + mapping; a UNC path does not depend on that. +- Arguments: `query` (whitespace-separated terms; a document matches when its + path below a root contains every term, case-insensitively), `limit` (default + 50, maximum 500) and `refresh` (rescan now). +- Results are path matches, not content matches: only file and folder names are + searched, and the root itself never matches. Each result has `path`, + `modified` (date) and `size_kb`; pass `path` to `build_datasheet` unchanged. + The response also carries `total_matches`, `truncated` and a status per root. +- The list is cached on disk for one day (fixed), so a restarted server does not + rescan. The server refreshes a missing or stale list in the background at + startup, and `refresh=true` forces a rescan. An unreachable root or an + unreadable folder is reported in the response, never raised. A PDF that + cannot be read, or whose name is not valid UTF-8, is left out and counted in + its root's `skipped_files`. + ### Datasheets without a ToC The enriched ToC comes from the PDF's own bookmarks, and plenty of real @@ -796,6 +847,7 @@ src/datasheetindex/ defs.py # create_datasheet_tool_defs (framework-neutral defs) registry.py # Claude Agent SDK adapter (re-exports DatasheetTools) mcp_server.py # Local stdio/HTTP MCP server entry point + library.py # find_documents: PDF listing for MCP library roots llm/ client.py # LLM client factory (free-text + structured_json + vision) toc_fallback.py # LLM-based ToC generation fallback diff --git a/docs/datasheetindex_architecture.md b/docs/datasheetindex_architecture.md index 3c8537c..97c88dc 100644 --- a/docs/datasheetindex_architecture.md +++ b/docs/datasheetindex_architecture.md @@ -1726,6 +1726,56 @@ So the SDK server, the local MCP server, and non-SDK hosts all present identical tool names, descriptions, and JSON schemas -- a change to a tool def propagates to every surface from one place. +### Library discovery: `find_documents` + +`mcp_server.py` registers a sixth tool, `find_documents`, only when +`DATASHEETINDEX_LIBRARY_ROOTS` names at least one folder (roots separated by +`os.pathsep`). It lists PDFs by file name and folder path, never by content. The +reason it exists: an agent can only call `build_datasheet` on a path it can name, +and when the agent's shell and the server see different filesystems (a dev +container against a Windows share) only the server can do the finding. + +- **MCP-only.** It is added in `mcp_server.py`, not in + `create_datasheet_tool_defs()`. Embedders of the shared defs did not opt into a + filesystem lister, and the five-tool surface stays true for them. Unset or + blank, the tool is not registered at all (not a tool that answers "not + configured"), so nothing changes for anyone who does not use it. +- **Full paths.** Each result carries the full server-side path, in the server's + own format (`\\server\share\...`), which the agent passes to + `build_datasheet` unchanged. `build_datasheet`'s contract is untouched, and + there is no root-plus-relative join for an agent on a different OS to get wrong. +- **Matching.** Every whitespace-separated term must appear, case-insensitively, + in the path relative to its root (folders and file name, either separator). + The root is excluded, or a root named `Datasheets` would match `data`. Results + are sorted by relative path, capped by `limit`, and the response says + `total_matches` and `truncated`. Files under overlapping roots are listed once. +- **Cache.** One JSON file per root at `/library`, where the output + dir is the artifacts' (`DATASHEETINDEX_OUTPUT_DIR`, else the temp default), + written atomically, valid for one day (not configurable). It is loaded and, if + missing or stale, rescanned in a background thread at server startup, so the + slow scan of a share happens before the agent asks. A list whose `listed_at` + lies in the future (a host with its clock ahead sharing the folder) counts as + stale. The server loads `.env` before reading either setting, since both are + read once, at startup. `load_dotenv()` finds `.env` by walking up from the + installed package, not the CWD, so this works on a source checkout only; an + installed server takes both from the MCP client config's `env` block. +- **Failure rules.** One bad root or folder never becomes a tool error. An + unreadable subfolder is skipped and counted (`skipped_folders`). A PDF that + cannot be stat'ed, or whose path is not valid UTF-8 (a non-UTF-8 name on POSIX, + an unpaired surrogate on Windows), is skipped and counted (`skipped_files`): + no cache file or JSON-RPC message can carry such a path, and returning one + killed the stdio transport. A root whose rescan fails keeps its previous list, + if any: its `status` stays `ok` (or `refreshing`) and an `error` field says + what failed; it is `unavailable` only when there is no list at all. A stale + list is served at once while a rescan runs (`refreshing`). Only a root with no + list makes a call wait, for at most 30 seconds, after which it reports + `scanning` with the elapsed time. A corrupt, damaged (non-finite numbers, deep + nesting) or old-format cache is treated as missing. Symlinks and junctions are not followed. Scans are de-duplicated + within one process only; several server processes may each scan a stale share. +- **Threads, not processes.** A process pool inside the MCP stdio server + deadlocks on Windows (see the scan-worker notes), and the scan is I/O bound, so + scans run in daemon threads that never hold the state lock while scanning. + ### Page-cut truncation signal A section's ToC page range does not always contain the whole of its table. In @@ -2113,6 +2163,8 @@ datasheetindex/ │ │ # bounded by max_figure_captions │ └── untrusted.py # Framing for document text sent to an LLM │ # (PDF text is untrusted input, not instructions) +├── mcp_server.py # Local MCP server (stdio / streamable-http / sse) +├── library.py # find_documents: library scan, disk cache, matching ├── index.py # Main DatasheetIndex class └── models.py # Data models ``` diff --git a/pyproject.toml b/pyproject.toml index 9c47e57..212c119 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "datasheetindex" -version = "0.45.0" +version = "0.46.0" description = "Agent-first parameter extraction from technical datasheets" readme = "README.md" requires-python = ">=3.13" diff --git a/src/datasheetindex/library.py b/src/datasheetindex/library.py new file mode 100644 index 0000000..ed43850 --- /dev/null +++ b/src/datasheetindex/library.py @@ -0,0 +1,519 @@ +"""Library discovery for the MCP server: list PDFs under configured folders.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import logging +import math +import os +import tempfile +import threading +import time +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any + +from datasheetindex.index import resolve_default_output_dir +from datasheetindex.tools.defs import DatasheetToolDef + +LIBRARY_ROOTS_ENV_VAR = "DATASHEETINDEX_LIBRARY_ROOTS" +CACHE_SCHEMA_VERSION = 2 +CACHE_TTL_SECONDS = 86400.0 +SCAN_WAIT_SECONDS = 30.0 +DEFAULT_LIMIT = 50 +MAX_LIMIT = 500 +SCAN_THREAD_NAME = "datasheetindex-library-scan" + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class DocumentEntry: + relpath: str + path: str + mtime: float + size: int + + +@dataclass(frozen=True) +class ScanResult: + root: str + listed_at: float + entries: tuple[DocumentEntry, ...] + skipped_folders: int + skipped_files: int = 0 + + +def parse_library_roots(value: str | None) -> list[str]: + """Split a roots setting into absolute, de-duplicated folder paths.""" + roots: list[str] = [] + seen: set[str] = set() + for part in (value or "").split(os.pathsep): + part = part.strip() + if len(part) >= 2 and part[0] == part[-1] == '"': + part = part[1:-1].strip() + if not part: + continue + root = os.path.abspath(part) + try: + root.encode("utf-8") + except UnicodeEncodeError: + # Not valid UTF-8 (POSIX bytes): no cache name, tool description or + # JSON-RPC message can carry it, and one such root must not stop + # the server from starting. + logger.warning("ignoring library root that is not valid UTF-8: %r", root) + continue + key = os.path.normcase(root) + if key not in seen: + seen.add(key) + roots.append(root) + return roots + + +def scan_root(root: str, *, clock: Callable[[], float] = time.time) -> ScanResult: + """List every PDF under ``root``, never following symlinks or junctions. + + Raises ``OSError`` only when ``root`` itself cannot be listed; an unreadable + subfolder is skipped and counted, and so is a PDF that cannot be read or + whose path is not valid UTF-8 (a lone surrogate from a non-UTF-8 name on + POSIX or an unpaired one on Windows), which no cache file or JSON-RPC + message can carry. + """ + entries: list[DocumentEntry] = [] + skipped = 0 + skipped_files = 0 + stack = [root] + while stack: + folder = stack.pop() + try: + with os.scandir(folder) as it: + children = list(it) + except OSError: + if folder == root: + raise + skipped += 1 + continue + for child in children: + try: + if child.is_symlink() or child.is_junction(): + continue + if child.is_dir(follow_symlinks=False): + stack.append(child.path) + continue + if not ( + child.is_file(follow_symlinks=False) + and child.name.lower().endswith(".pdf") + ): + continue + except OSError: + # Its type is unknown, and a folder is what loses the most. + skipped += 1 + continue + try: + child.path.encode("utf-8") + st = child.stat(follow_symlinks=False) + except (OSError, UnicodeEncodeError): + skipped_files += 1 + continue + entries.append( + DocumentEntry( + relpath=os.path.relpath(child.path, root), + path=child.path, + mtime=st.st_mtime, + size=st.st_size, + ) + ) + return ScanResult( + root=root, + listed_at=clock(), + entries=tuple(entries), + skipped_folders=skipped, + skipped_files=skipped_files, + ) + + +def _fold(text: str) -> str: + return text.replace("\\", "/").casefold() + + +def match_entries(entries: Iterable[DocumentEntry], query: str) -> list[DocumentEntry]: + """Entries whose relpath contains every query term, sorted by relpath.""" + terms = [_fold(t) for t in query.split()] + hits = [e for e in entries if all(t in _fold(e.relpath) for t in terms)] + return sorted(hits, key=lambda e: e.relpath.casefold()) + + +def default_cache_dir() -> Path: + return Path(resolve_default_output_dir()) / "library" + + +def cache_file(cache_dir: Path, root: str) -> Path: + digest = hashlib.sha256(os.path.normcase(root).encode("utf-8")).hexdigest() + return cache_dir / f"{digest[:16]}.json" + + +def save_cache(path: Path, result: ScanResult) -> None: + """Write ``result`` atomically so a concurrent reader never sees a partial file.""" + payload = { + "schema": CACHE_SCHEMA_VERSION, + "root": result.root, + "listed_at": result.listed_at, + "skipped_folders": result.skipped_folders, + "skipped_files": result.skipped_files, + "entries": [[e.relpath, e.path, e.mtime, e.size] for e in result.entries], + } + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + "w", encoding="utf-8", dir=path.parent, suffix=".tmp", delete=False + ) as tmp: + tmp_name = tmp.name + try: + json.dump(payload, tmp, ensure_ascii=False) + except BaseException: + tmp.close() + os.unlink(tmp_name) + raise + try: + # On Windows this fails (access denied) while another server process + # has the file open for reading. The caller logs it and still serves + # the fresh list from memory; only the next start's warm load misses. + os.replace(tmp_name, path) + except BaseException: + os.unlink(tmp_name) + raise + + +def load_cache(path: Path, root: str) -> ScanResult | None: + """Read a cache written by ``save_cache``; ``None`` if absent or unusable. + + Any failure means unusable: a damaged file can raise far more than the + obvious errors (``OverflowError`` for an ``Infinity`` size, + ``RecursionError`` for deep nesting), and the caller must rescan, not fail. + """ + try: + data = json.loads(path.read_text(encoding="utf-8")) + if data["schema"] != CACHE_SCHEMA_VERSION: + return None + if os.path.normcase(data["root"]) != os.path.normcase(root): + return None + listed_at = float(data["listed_at"]) + if not math.isfinite(listed_at): # NaN would never go stale + return None + entries = tuple( + DocumentEntry(relpath=str(r), path=str(p), mtime=float(m), size=int(z)) + for r, p, m, z in data["entries"] + ) + for e in entries: # a lone surrogate would kill the stdio writer + e.relpath.encode("utf-8") + e.path.encode("utf-8") + return ScanResult( + root=root, + listed_at=listed_at, + entries=entries, + skipped_folders=int(data["skipped_folders"]), + skipped_files=int(data["skipped_files"]), + ) + except Exception: + return None + + +def _local_time(ts: float) -> datetime | None: + # A share can report an mtime the platform cannot represent; that file's + # date is unknown, which must not fail the whole listing. + try: + return datetime.fromtimestamp(ts).astimezone() + except (OverflowError, OSError, ValueError): + return None + + +def _document(entry: DocumentEntry) -> dict[str, Any]: + modified = _local_time(entry.mtime) + return { + "path": entry.path, + "modified": modified.strftime("%Y-%m-%d") if modified else None, + "size_kb": math.ceil(entry.size / 1024), + } + + +@dataclass +class _RootState: + """One root's listing; every field is read and written under the library lock.""" + + root: str + cache_path: Path + loaded: bool = False + result: ScanResult | None = None + error: str | None = None + done: threading.Event | None = None + thread: threading.Thread | None = None + started_at: float = 0.0 + + @property + def scanning(self) -> bool: + return self.done is not None and not self.done.is_set() + + +class DocumentLibrary: + """PDFs under configured roots, listed by background scans and cached on disk. + + A root has at most one scan thread at a time. ``find`` serves a fresh list at + once and a stale one at once while it is rescanned; it waits, for at most + ``wait_seconds``, only for a root with no list or when ``refresh`` is asked + for. A root that cannot be scanned is reported in its status entry and never + raised. The lock guards state only: nothing scans or waits while holding it. + """ + + def __init__( + self, + roots: list[str], + cache_dir: Path, + *, + scanner: Callable[[str], ScanResult] = scan_root, + clock: Callable[[], float] = time.time, + ttl: float = CACHE_TTL_SECONDS, + wait_seconds: float = SCAN_WAIT_SECONDS, + ) -> None: + self._roots = list(roots) + self._states = [_RootState(r, cache_file(cache_dir, r)) for r in self._roots] + self._scanner = scanner + self._clock = clock + self._ttl = ttl + self._wait_seconds = wait_seconds + self._lock = threading.Lock() + + @property + def roots(self) -> list[str]: + return list(self._roots) + + def warm(self) -> None: + """Load cached lists and rescan, in the background, any missing or stale.""" + for state in self._states: + self._load(state) + with self._lock: + if state.result is None or self._is_stale(state.result): + self._start_scan_locked(state) + + def find( + self, query: str = "", limit: int = DEFAULT_LIMIT, refresh: bool = False + ) -> dict[str, Any]: + """Documents whose path under a root contains every term of ``query``.""" + limit = max(1, min(int(limit), MAX_LIMIT)) + pending: list[threading.Event] = [] + for state in self._states: + self._load(state) + with self._lock: + result = state.result + if refresh or result is None: + pending.append(self._start_scan_locked(state)) + elif self._is_stale(result): + self._start_scan_locked(state) + # One deadline for all roots, so the call waits wait_seconds at most. + deadline = time.monotonic() + self._wait_seconds + for done in pending: + done.wait(max(0.0, deadline - time.monotonic())) + with self._lock: + snapshot = [ + (s.root, s.result, s.error, s.scanning, s.started_at) + for s in self._states + ] + return self._response(snapshot, query, limit) + + def wait_for_scans(self, timeout: float | None = None) -> bool: + """Join the scan threads running now; ``False`` if ``timeout`` ran out. + + The server never needs this (scan threads are daemons). It lets tests + and embedding code finish without leaving a scan behind. + """ + with self._lock: + threads = [s.thread for s in self._states if s.thread is not None] + deadline = None if timeout is None else time.monotonic() + timeout + for thread in threads: + if deadline is None: + thread.join() + else: + thread.join(max(0.0, deadline - time.monotonic())) + return not any(t.is_alive() for t in threads) + + def _is_stale(self, result: ScanResult) -> bool: + age = self._clock() - result.listed_at + # A list from the future was written by a host whose clock runs ahead + # of ours (a shared cache folder); its age is unknown, so rescan. + return age < 0 or age > self._ttl + + def _load(self, state: _RootState) -> None: + """Adopt the root's on-disk cache once; read outside the lock.""" + with self._lock: + if state.loaded: + return + cached = load_cache(state.cache_path, state.root) + with self._lock: + if not state.loaded: + state.loaded = True + # A scan that finished meanwhile is newer than the file. + if state.result is None: + state.result = cached + + def _start_scan_locked(self, state: _RootState) -> threading.Event: + """Start a scan of ``state.root`` unless one runs; return its done event.""" + if state.done is not None and not state.done.is_set(): + return state.done + done = threading.Event() + thread = threading.Thread( + target=self._scan, args=(state, done), name=SCAN_THREAD_NAME, daemon=True + ) + state.done, state.thread, state.started_at = done, thread, self._clock() + try: + thread.start() + except RuntimeError as exc: # interpreter cannot start another thread + state.error = f"{type(exc).__name__}: {exc}" + state.thread = None + done.set() + return done + + def _scan(self, state: _RootState, done: threading.Event) -> None: + result: ScanResult | None = None + error = "scan ended without a result" + started = time.monotonic() + try: + result = self._scanner(state.root) + try: + save_cache(state.cache_path, result) + except Exception: + logger.warning( + "could not save the library cache for %s", + state.root, + exc_info=True, + ) + except BaseException as exc: + # Nothing above this thread would see it: record it for the status. + error = f"{type(exc).__name__}: {exc}" + finally: + # Publishing and setting `done` together means a waiter that wakes + # always sees the outcome, and the next call can start a new scan. + with self._lock: + if result is None: + state.error = error + else: + state.result, state.error = result, None + done.set() + elapsed = time.monotonic() - started + if result is None: + logger.warning("library scan of %s failed: %s", state.root, error) + else: + logger.info( + "library scan of %s listed %d PDFs in %.1fs", + state.root, + len(result.entries), + elapsed, + ) + + def _response( + self, + snapshot: list[tuple[str, ScanResult | None, str | None, bool, float]], + query: str, + limit: int, + ) -> dict[str, Any]: + now = self._clock() + roots: list[dict[str, Any]] = [] + matches: list[DocumentEntry] = [] + seen: set[str] = set() + for root, result, error, scanning, started_at in snapshot: + if scanning: + status = "scanning" if result is None else "refreshing" + else: + status = "unavailable" if result is None else "ok" + listed_at = _local_time(result.listed_at) if result else None + entry: dict[str, Any] = { + "root": root, + "status": status, + "listed_at": ( + listed_at.isoformat(timespec="seconds") if listed_at else None + ), + "documents": len(result.entries) if result else 0, + "skipped_folders": result.skipped_folders if result else 0, + "skipped_files": result.skipped_files if result else 0, + } + if error is not None: + entry["error"] = error + if status == "scanning": + entry["scan_elapsed_seconds"] = int(max(0.0, now - started_at)) + roots.append(entry) + if result is None: + continue + # Overlapping roots list a file twice; the first root keeps it. + for doc in match_entries(result.entries, query): + key = os.path.normcase(doc.path) + if key not in seen: + seen.add(key) + matches.append(doc) + return { + "roots": roots, + "total_matches": len(matches), + "truncated": len(matches) > limit, + "documents": [_document(e) for e in matches[:limit]], + } + + +#: Appended to the server ``instructions`` when a library is configured. +LIBRARY_INSTRUCTIONS = ( + "A document library is configured: when you need a datasheet or manual " + "and do not have its path, call find_documents with a part number first " + "and pass a result's path to build_datasheet." +) + +_FIND_DOCUMENTS_DESCRIPTION = ( + "Find PDF documents in the configured library by file name and folder path " + "(not by content). Library roots: {roots}. query: whitespace-separated " + "terms, case-insensitive; a document matches when its path below a root " + 'contains every term (e.g. "tle987" or "tle987 um"). Each result\'s path is ' + "ready to pass to build_datasheet as pdf_source unchanged; modified and " + "size_kb help tell versions apart. If truncated is true, narrow the query. " + "The list is cached for a day: set refresh=true only when a document you " + 'expect is missing. A root with status "scanning" is still being listed ' + "for the first time: call again shortly." +) + + +def create_find_documents_def(library: DocumentLibrary) -> DatasheetToolDef: + """The ``find_documents`` tool over ``library``. + + Lives here, not in ``tools.defs``, so embedders of the neutral defs never + get a tool that needs configured roots. + """ + + async def handler(args: dict[str, Any]) -> dict[str, Any]: + result = await asyncio.to_thread( + library.find, + args.get("query", ""), + args.get("limit", DEFAULT_LIMIT), + args.get("refresh", False), + ) + return { + "content": [ + {"type": "text", "text": json.dumps(result, ensure_ascii=False)} + ], + "is_error": False, + } + + return DatasheetToolDef( + name="find_documents", + description=_FIND_DOCUMENTS_DESCRIPTION.format(roots="; ".join(library.roots)), + input_schema={ + "type": "object", + "properties": { + "query": {"type": "string", "default": ""}, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": MAX_LIMIT, + "default": DEFAULT_LIMIT, + }, + "refresh": {"type": "boolean", "default": False}, + }, + }, + handler=handler, + ) diff --git a/src/datasheetindex/mcp_server.py b/src/datasheetindex/mcp_server.py index e92880b..341c7b7 100644 --- a/src/datasheetindex/mcp_server.py +++ b/src/datasheetindex/mcp_server.py @@ -5,7 +5,9 @@ same five tools -- with the same names, descriptions, and JSON schemas -- that the Claude Agent SDK surface (``create_datasheet_tools_server``) and non-SDK hosts get. There is a single source of truth for the tool definitions; this module only -wires them onto MCP transports (stdio / streamable-http / sse). +wires them onto MCP transports (stdio / streamable-http / sse). The one addition +is opt-in and MCP-only: a sixth tool, ``find_documents``, served only when +``DATASHEETINDEX_LIBRARY_ROOTS`` names a folder (see ``datasheetindex.library``). """ from __future__ import annotations @@ -14,12 +16,24 @@ import contextlib import importlib import inspect +import os import sys +from collections.abc import Sequence from typing import Any from datasheetindex._version import package_version +from datasheetindex.library import ( + LIBRARY_INSTRUCTIONS, + LIBRARY_ROOTS_ENV_VAR, + DocumentLibrary, + create_find_documents_def, + default_cache_dir, + parse_library_roots, +) +from datasheetindex.llm.client import ensure_dotenv_loaded from datasheetindex.tools.defs import ( DATASHEET_TOOL_INSTRUCTIONS, + DatasheetToolDef, DatasheetToolSession, create_datasheet_tool_session, ) @@ -165,7 +179,12 @@ def _passthrough_exceptions() -> tuple[type[BaseException], ...]: def _build_mcp_server( - session: DatasheetToolSession, server_cls: Any, types_module: Any + session: DatasheetToolSession, + server_cls: Any, + types_module: Any, + *, + extra_defs: Sequence[DatasheetToolDef] = (), + instructions: str = SERVER_INSTRUCTIONS, ): """Register the neutral tool session's defs onto a low-level MCP ``Server``. @@ -183,9 +202,13 @@ def _build_mcp_server( ``try``/``except`` enforces "always return a result" structurally rather than leaving it to be maintained by counting guards -- which is how (c) was missed the first time. Do not "simplify" any of it away. + + ``extra_defs`` (the opt-in ``find_documents``) join ``session.defs`` in one + list, so listing, validation and dispatch cover them through the same path. """ - by_name = {d.name: d for d in session.defs} - validators = _compile_validators(session.defs) + all_defs = [*session.defs, *extra_defs] + by_name = {d.name: d for d in all_defs} + validators = _compile_validators(all_defs) passthrough = _passthrough_exceptions() # Positive probe rather than letting the constructor fail. The extra has no @@ -221,7 +244,7 @@ async def _on_list_tools(_ctx: Any, _params: Any) -> Any: description=d.description, input_schema=d.input_schema, ) - for d in session.defs + for d in all_defs ] ) @@ -269,7 +292,7 @@ async def _on_call_tool(_ctx: Any, params: Any) -> Any: return server_cls( name="datasheetindex", version=package_version(), - instructions=SERVER_INSTRUCTIONS, + instructions=instructions, on_list_tools=_on_list_tools, on_call_tool=_on_call_tool, ) @@ -290,7 +313,10 @@ def __init__( host: str, port: int, streamable_http_path: str, + *, + library: DocumentLibrary | None = None, ) -> None: + self.library = library self.mcp_server = mcp_server self.session = session self.host = host @@ -388,8 +414,32 @@ def create_local_mcp_server( """ server_cls, types_module = _load_mcp_modules() session = create_datasheet_tool_session() - mcp_server = _build_mcp_server(session, server_cls, types_module) - return LocalMcpServer(mcp_server, session, host, port, streamable_http_path) + # Read .env before the library settings below: nothing else loads it until + # the first build_datasheet, and the roots and the cache's output dir are + # read only here, at startup. + ensure_dotenv_loaded() + roots = parse_library_roots(os.environ.get(LIBRARY_ROOTS_ENV_VAR)) + library: DocumentLibrary | None = None + extra_defs: list[DatasheetToolDef] = [] + instructions = SERVER_INSTRUCTIONS + if roots: + # Opt-in: with no roots configured the server serves exactly its five + # tools. warm() only starts background scans, so an unreachable root + # cannot stop the server from starting. + library = DocumentLibrary(roots, default_cache_dir()) + library.warm() + extra_defs.append(create_find_documents_def(library)) + instructions = SERVER_INSTRUCTIONS + " " + LIBRARY_INSTRUCTIONS + mcp_server = _build_mcp_server( + session, + server_cls, + types_module, + extra_defs=extra_defs, + instructions=instructions, + ) + return LocalMcpServer( + mcp_server, session, host, port, streamable_http_path, library=library + ) def run_mcp_server( diff --git a/tests/conftest.py b/tests/conftest.py index 06e80cd..1f3bfce 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -165,6 +165,12 @@ def _hermetic_llm_env(request, monkeypatch): monkeypatch.setattr(dotenv, "load_dotenv", lambda *args, **kwargs: False) +@pytest.fixture(autouse=True) +def _hermetic_library_env(monkeypatch): + """Keep a developer's library roots from adding find_documents to tool lists.""" + monkeypatch.delenv("DATASHEETINDEX_LIBRARY_ROOTS", raising=False) + + @pytest.fixture(autouse=True) def _hermetic_wsl_env(monkeypatch): """Keep the developer's own distro out of every test. diff --git a/tests/test_library.py b/tests/test_library.py new file mode 100644 index 0000000..4682d0a --- /dev/null +++ b/tests/test_library.py @@ -0,0 +1,792 @@ +"""Tests for library discovery: root parsing, scanning, matching and serving.""" + +import json +import logging +import os +import re +import threading +import time +from datetime import datetime + +import pytest + +from datasheetindex import library +from datasheetindex.library import ( + MAX_LIMIT, + DocumentEntry, + DocumentLibrary, + ScanResult, + cache_file, + default_cache_dir, + load_cache, + match_entries, + parse_library_roots, + save_cache, + scan_root, +) + + +def _touch(path, size=1): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"x" * size) + + +def _entry(relpath): + return DocumentEntry( + relpath=relpath, path=os.path.join("root", relpath), mtime=1.0, size=1 + ) + + +def test_parse_roots_none_and_blank_are_empty(): + assert parse_library_roots(None) == [] + assert parse_library_roots(" " + os.pathsep + " ") == [] + + +def test_parse_roots_strips_quotes_whitespace_and_trailing_separator(tmp_path): + raw = f' "{tmp_path}{os.sep}" {os.pathsep}{tmp_path / "b"}' + assert parse_library_roots(raw) == [str(tmp_path), str(tmp_path / "b")] + + +def test_parse_roots_dedupes_and_makes_absolute(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + raw = f"lib{os.pathsep}{tmp_path / 'lib'}" + assert parse_library_roots(raw) == [str(tmp_path / "lib")] + + +def test_parse_roots_drops_root_that_is_not_valid_utf8(caplog): + good = os.path.abspath("good") + bad = "/data/Datenbl\udce4tter" + with caplog.at_level(logging.WARNING, logger=library.logger.name): + roots = parse_library_roots(os.pathsep.join([bad, good])) + assert roots == [good] + assert "UTF-8" in caplog.text + + +def test_scan_lists_pdfs_recursively_with_stat_fields(tmp_path): + name = "Infineon-TLE987x-UM-v01_92-EN.pdf" + _touch(tmp_path / "TLE" / "TLE987x" / name, size=2048) + _touch(tmp_path / "notes.txt") + _touch(tmp_path / "UPPER.PDF") + result = scan_root(str(tmp_path), clock=lambda: 1000.0) + rel = sorted(e.relpath for e in result.entries) + assert rel == [os.path.join("TLE", "TLE987x", name), "UPPER.PDF"] + um = next(e for e in result.entries if e.relpath.endswith("UM-v01_92-EN.pdf")) + assert um.path == str(tmp_path / "TLE" / "TLE987x" / name) + assert um.size == 2048 and um.mtime > 0 + assert result.root == str(tmp_path) + assert result.listed_at == 1000.0 and result.skipped_folders == 0 + + +def test_scan_does_not_follow_directory_symlinks(tmp_path): + _touch(tmp_path / "real" / "a.pdf") + try: + (tmp_path / "loop").symlink_to(tmp_path, target_is_directory=True) + except OSError: + pytest.skip("symlinks not permitted") + result = scan_root(str(tmp_path)) + assert [e.relpath for e in result.entries] == [os.path.join("real", "a.pdf")] + + +def test_scan_skips_and_counts_unreadable_folder(tmp_path, monkeypatch): + _touch(tmp_path / "ok.pdf") + _touch(tmp_path / "locked" / "hidden.pdf") + real_scandir = os.scandir + locked = str(tmp_path / "locked") + + def fake_scandir(path): + if os.fspath(path) == locked: + raise PermissionError(13, "denied", locked) + return real_scandir(path) + + monkeypatch.setattr(library.os, "scandir", fake_scandir) + result = scan_root(str(tmp_path)) + assert result.skipped_folders == 1 + assert [e.relpath for e in result.entries] == ["ok.pdf"] + + +class _UnstattableEntry: + """A scandir entry whose ``stat`` fails, as a file deleted mid-scan does.""" + + def __init__(self, entry): + self._entry = entry + self.name, self.path = entry.name, entry.path + + def __getattr__(self, attr): + return getattr(self._entry, attr) + + def stat(self, *, follow_symlinks=True): + raise PermissionError(13, "denied", self.path) + + +def test_scan_skips_and_counts_pdf_that_cannot_be_read(tmp_path, monkeypatch): + _touch(tmp_path / "ok.pdf") + _touch(tmp_path / "gone.pdf") + real_scandir = os.scandir + + class _Listing: + def __init__(self, path): + self._it = real_scandir(path) + + def __enter__(self): + return [ + _UnstattableEntry(e) if e.name == "gone.pdf" else e + for e in self._it.__enter__() + ] + + def __exit__(self, *exc): + return self._it.__exit__(*exc) + + monkeypatch.setattr(library.os, "scandir", _Listing) + result = scan_root(str(tmp_path)) + assert [e.relpath for e in result.entries] == ["ok.pdf"] + assert result.skipped_files == 1 and result.skipped_folders == 0 + + +def _make_unencodable_pdf(folder): + """Create ``TLE\xff-DS.pdf``, a name that is not valid UTF-8, or skip.""" + name = b"TLE\xff-DS.pdf" + try: + with open(os.path.join(os.fsencode(folder), name), "wb") as f: + f.write(b"x") + except (OSError, ValueError): + pytest.skip("this filesystem cannot hold a non-UTF-8 file name") + if os.fsencode(os.fsdecode(name)) != name: + pytest.skip("file names do not round-trip through surrogateescape") + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX byte file names only") +def test_scan_skips_and_counts_name_not_encodable_as_utf8(tmp_path): + _touch(tmp_path / "ok.pdf") + _make_unencodable_pdf(tmp_path) + result = scan_root(str(tmp_path)) + assert [e.relpath for e in result.entries] == ["ok.pdf"] + assert result.skipped_files == 1 + # The listing it produces must survive the cache and the JSON response. + save_cache(tmp_path / "c.json", result) + assert load_cache(tmp_path / "c.json", result.root) == result + + +def test_scan_counts_child_whose_type_cannot_be_determined(tmp_path, monkeypatch): + _touch(tmp_path / "ok.pdf") + _touch(tmp_path / "odd" / "hidden.pdf") + real_scandir = os.scandir + + class _TypeUnknownEntry: + def __init__(self, entry): + self._entry = entry + self.name, self.path = entry.name, entry.path + + def __getattr__(self, attr): + return getattr(self._entry, attr) + + def is_dir(self, *, follow_symlinks=True): + raise PermissionError(13, "denied", self.path) + + class _Listing: + def __init__(self, path): + self._it = real_scandir(path) + + def __enter__(self): + return [ + _TypeUnknownEntry(e) if e.name == "odd" else e + for e in self._it.__enter__() + ] + + def __exit__(self, *exc): + return self._it.__exit__(*exc) + + monkeypatch.setattr(library.os, "scandir", _Listing) + result = scan_root(str(tmp_path)) + assert result.skipped_folders == 1 + assert [e.relpath for e in result.entries] == ["ok.pdf"] + + +def test_scan_raises_when_root_is_missing(tmp_path): + with pytest.raises(OSError): + scan_root(str(tmp_path / "absent")) + + +def test_match_requires_every_term_case_insensitive(): + um = os.path.join("TLE987x", "Infineon-TLE987x-UM.pdf") + entries = [ + _entry(um), + _entry(os.path.join("TLE987x", "Infineon-TLE987x-DS.pdf")), + _entry(os.path.join("TLE9140", "Infineon-TLE9140-UM.pdf")), + ] + assert [e.relpath for e in match_entries(entries, "tle987 um")] == [um] + + +def test_match_empty_query_returns_all_sorted(): + entries = [_entry("b.pdf"), _entry("A.pdf"), _entry("c.pdf")] + assert [e.relpath for e in match_entries(entries, " ")] == [ + "A.pdf", + "b.pdf", + "c.pdf", + ] + + +def test_match_searches_folders_too(): + entries = [_entry(os.path.join("TLE987x", "4711.pdf")), _entry("other.pdf")] + got = match_entries(entries, "tle987x") + assert [e.relpath for e in got] == [os.path.join("TLE987x", "4711.pdf")] + + +@pytest.mark.parametrize("query", ["TLE987x/um", "tle987x\\UM"]) +@pytest.mark.parametrize("sep", ["/", "\\"]) +def test_match_normalises_separators_and_case(sep, query): + rel = "TLE987x" + sep + "UM-guide.pdf" + assert [e.relpath for e in match_entries([_entry(rel)], query)] == [rel] + + +def _result(root, relpaths=("a.pdf",)): + return ScanResult( + root=root, + listed_at=123.5, + entries=tuple(_entry(r) for r in relpaths), + skipped_folders=2, + ) + + +def test_cache_round_trip(tmp_path): + path = tmp_path / "cache" / "x.json" + result = _result(str(tmp_path / "lib"), ("a.pdf", os.path.join("d", "b.pdf"))) + save_cache(path, result) + assert load_cache(path, result.root) == result + + +def test_cache_round_trip_preserves_non_ascii_and_upper_case_names(tmp_path): + path = tmp_path / "x.json" + result = _result(str(tmp_path / "lib"), ("Datenblatt-Gr\u00f6\u00dfe.PDF",)) + save_cache(path, result) + loaded = load_cache(path, result.root) + assert loaded is not None + assert loaded.entries[0].relpath == "Datenblatt-Gr\u00f6\u00dfe.PDF" + + +def test_load_cache_missing_corrupt_or_old_schema_is_none(tmp_path): + root = str(tmp_path / "lib") + assert load_cache(tmp_path / "absent.json", root) is None + bad = tmp_path / "bad.json" + bad.write_bytes(b"{not json") + assert load_cache(bad, root) is None + old = tmp_path / "old.json" + old.write_text( + json.dumps( + { + "schema": 0, + "root": root, + "listed_at": 1.0, + "skipped_folders": 0, + "entries": [], + } + ), + encoding="utf-8", + ) + assert load_cache(old, root) is None + + +@pytest.mark.parametrize( + "payload", + [ + [], + {"schema": library.CACHE_SCHEMA_VERSION}, + { + "schema": library.CACHE_SCHEMA_VERSION, + "root": "R", + "listed_at": 1.0, + "skipped_folders": 0, + "skipped_files": 0, + }, + { + "schema": 1, + "root": "R", + "listed_at": 1.0, + "skipped_folders": 0, + "entries": ["notalist"], + }, + { + "schema": 1, + "root": "R", + "listed_at": "x", + "skipped_folders": 0, + "entries": [], + }, + { + "schema": 1, + "root": "R", + "listed_at": 1.0, + "skipped_folders": 0, + "entries": [["a", "b", 1.0]], + }, + ], +) +def test_load_cache_wrong_shape_is_none(tmp_path, payload): + path = tmp_path / "x.json" + path.write_text(json.dumps(payload), encoding="utf-8") + assert load_cache(path, "R") is None + + +def _cache_text(root, *, listed_at="1.0", size="1", mtime="1.0"): + """A cache file's text with raw JSON literals spliced into its numbers.""" + return ( + f'{{"schema": {library.CACHE_SCHEMA_VERSION}, "root": {json.dumps(root)}, ' + f'"listed_at": {listed_at}, "skipped_folders": 0, "skipped_files": 0, ' + f'"entries": [["a.pdf", "a.pdf", {mtime}, {size}]]}}' + ) + + +@pytest.mark.parametrize( + "text", + [ + _cache_text("R", size="Infinity"), + _cache_text("R", size="1e400"), + _cache_text("R", listed_at="1e400"), + _cache_text("R", listed_at="Infinity"), + _cache_text("R", listed_at="NaN"), + "[" * 100_000 + "]" * 100_000, + ], + ids=["size-inf", "size-1e400", "listed-1e400", "listed-inf", "listed-nan", "deep"], +) +def test_load_cache_damaged_numbers_or_nesting_is_none(tmp_path, text): + path = tmp_path / "x.json" + path.write_text(text, encoding="utf-8") + assert load_cache(path, "R") is None + + +def test_load_cache_unrepresentable_mtime_is_kept(tmp_path): + # One file's date is unknown; the response renders it as None (_local_time). + path = tmp_path / "x.json" + path.write_text(_cache_text("R", mtime="1e400"), encoding="utf-8") + loaded = load_cache(path, "R") + assert loaded is not None and loaded.entries[0].mtime == float("inf") + + +@pytest.mark.parametrize("field", [0, 1]) +def test_load_cache_with_unencodable_path_is_none(tmp_path, field): + entry = ["a.pdf", "a.pdf", 1.0, 1] + entry[field] = "TLE\udc80.pdf" + text = json.dumps( + { + "schema": library.CACHE_SCHEMA_VERSION, + "root": "R", + "listed_at": 1.0, + "skipped_folders": 0, + "skipped_files": 0, + "entries": [entry], + } + ) + assert "\\udc80" in text # json.dumps escaped the lone surrogate + path = tmp_path / "c.json" + path.write_text(text, encoding="utf-8") + assert load_cache(path, "R") is None + + +def test_load_cache_for_other_root_is_none(tmp_path): + path = tmp_path / "x.json" + save_cache(path, _result(str(tmp_path / "one"))) + assert load_cache(path, str(tmp_path / "two")) is None + + +def test_cache_file_is_stable_and_distinct_per_root(tmp_path): + a = cache_file(tmp_path, str(tmp_path / "a")) + assert a == cache_file(tmp_path, str(tmp_path / "a")) + assert a != cache_file(tmp_path, str(tmp_path / "b")) + assert a.parent == tmp_path + assert re.fullmatch(r"[0-9a-f]{16}\.json", a.name) + + +def test_default_cache_dir_is_under_output_dir(tmp_path, monkeypatch): + monkeypatch.setenv("DATASHEETINDEX_OUTPUT_DIR", str(tmp_path)) + assert default_cache_dir() == tmp_path / "library" + + +def test_save_cache_is_atomic(tmp_path, monkeypatch): + path = tmp_path / "x.json" + root = str(tmp_path / "lib") + old = _result(root, ("old.pdf",)) + save_cache(path, old) + + def boom(src, dst): + raise OSError("replace failed") + + monkeypatch.setattr(os, "replace", boom) + with pytest.raises(OSError): + save_cache(path, _result(root, ("new.pdf",))) + monkeypatch.undo() + assert load_cache(path, root) == old + assert [p.name for p in tmp_path.iterdir()] == ["x.json"] + + +# --- DocumentLibrary ------------------------------------------------------- + +NOW = 1_000_000.0 +DAY = 86400.0 + + +def _listing(root, relpaths, *, listed_at=NOW, size=1): + return ScanResult( + root=root, + listed_at=listed_at, + entries=tuple( + DocumentEntry(relpath=r, path=os.path.join(root, r), mtime=NOW, size=size) + for r in relpaths + ), + skipped_folders=0, + ) + + +def _lists(*relpaths): + """A scanner outcome: list ``relpaths`` under whichever root is scanned.""" + return lambda root: _listing(root, relpaths) + + +class FakeScanner: + """Canned outcomes in call order (the last repeats); optionally gated.""" + + def __init__(self, *outcomes, gated=False): + self.outcomes = list(outcomes) + self.gated = gated + self.gate = threading.Event() + self.calls = [] + + def __call__(self, root): + self.calls.append(root) + if self.gated: + assert self.gate.wait(5), "test never released the scanner gate" + outcome = self.outcomes.pop(0) if len(self.outcomes) > 1 else self.outcomes[0] + if isinstance(outcome, BaseException): + raise outcome + return outcome(root) + + +@pytest.fixture +def cache_dir(tmp_path): + return tmp_path / "cache" + + +@pytest.fixture +def make_library(cache_dir): + made = [] + + def make(roots, scanner, **kwargs): + kwargs.setdefault("clock", lambda: NOW) + kwargs.setdefault("wait_seconds", 5.0) + lib = DocumentLibrary(list(roots), cache_dir, scanner=scanner, **kwargs) + made.append((lib, scanner)) + return lib + + yield make + for lib, scanner in made: + if isinstance(scanner, FakeScanner): + scanner.gate.set() + assert lib.wait_for_scans(timeout=5) + + +def _seed(cache_dir, root, relpaths, *, age): + save_cache( + cache_file(cache_dir, root), _listing(root, relpaths, listed_at=NOW - age) + ) + + +def _names(response): + return [os.path.basename(d["path"]) for d in response["documents"]] + + +def test_find_serves_fresh_cache_without_scanning(tmp_path, cache_dir, make_library): + root = str(tmp_path / "lib") + _seed(cache_dir, root, ["a.pdf", "b.pdf"], age=60) + scanner = FakeScanner(_lists("new.pdf")) + got = make_library([root], scanner).find() + assert scanner.calls == [] + assert _names(got) == ["a.pdf", "b.pdf"] + [status] = got["roots"] + assert status["root"] == root and status["status"] == "ok" + assert status["documents"] == 2 and "error" not in status + + +def test_warm_scans_missing_and_stale_roots_in_background( + tmp_path, cache_dir, make_library +): + missing, stale, fresh = (str(tmp_path / n) for n in ("missing", "stale", "fresh")) + _seed(cache_dir, stale, ["old.pdf"], age=DAY + 1) + _seed(cache_dir, fresh, ["kept.pdf"], age=60) + scanner = FakeScanner(_lists("new.pdf"), gated=True) + lib = make_library([missing, stale, fresh], scanner) + lib.warm() # returns although both scans are blocked on the gate + assert not scanner.gate.is_set() + scanner.gate.set() + assert lib.wait_for_scans(timeout=5) + assert sorted(scanner.calls) == sorted([missing, stale]) + got = lib.find() + assert [r["status"] for r in got["roots"]] == ["ok", "ok", "ok"] + assert _names(got) == ["new.pdf", "new.pdf", "kept.pdf"] + + +def test_stale_list_is_served_at_once_and_refreshed(tmp_path, cache_dir, make_library): + root = str(tmp_path / "lib") + _seed(cache_dir, root, ["old.pdf"], age=DAY + 1) + scanner = FakeScanner(_lists("new.pdf"), gated=True) + lib = make_library([root], scanner) + start = time.monotonic() + got = lib.find() + assert time.monotonic() - start < 1.0 + assert _names(got) == ["old.pdf"] + assert got["roots"][0]["status"] == "refreshing" + assert "scan_elapsed_seconds" not in got["roots"][0] + scanner.gate.set() + assert lib.wait_for_scans(timeout=5) + got = lib.find() + assert _names(got) == ["new.pdf"] + assert got["roots"][0]["status"] == "ok" + assert scanner.calls == [root] + + +def test_refresh_true_waits_for_the_new_list(tmp_path, cache_dir, make_library): + root = str(tmp_path / "lib") + _seed(cache_dir, root, ["old.pdf"], age=60) + scanner = FakeScanner(_lists("new.pdf"), gated=True) + lib = make_library([root], scanner) + timer = threading.Timer(0.05, scanner.gate.set) + timer.start() + try: + got = lib.find(refresh=True) + finally: + timer.join() + assert _names(got) == ["new.pdf"] + assert got["roots"][0]["status"] == "ok" + + +def test_call_during_running_scan_does_not_start_another( + tmp_path, cache_dir, make_library +): + root = str(tmp_path / "lib") + scanner = FakeScanner(_lists("a.pdf"), gated=True) + lib = make_library([root], scanner, wait_seconds=0.05) + lib.warm() + assert lib.find()["roots"][0]["status"] == "scanning" + assert lib.find(refresh=True)["roots"][0]["status"] == "scanning" + scanner.gate.set() + assert lib.wait_for_scans(timeout=5) + assert _names(lib.find()) == ["a.pdf"] + assert scanner.calls == [root] + + +def test_wait_is_bounded_and_reports_scanning(tmp_path, cache_dir, make_library): + root = str(tmp_path / "lib") + now = [NOW] + scanner = FakeScanner(_lists("a.pdf"), gated=True) + lib = make_library([root], scanner, clock=lambda: now[0], wait_seconds=0.2) + lib.warm() + now[0] += 7.0 + start = time.monotonic() + got = lib.find() + elapsed = time.monotonic() - start + assert 0.15 <= elapsed < 1.0 + [status] = got["roots"] + assert status["status"] == "scanning" + assert status["scan_elapsed_seconds"] == 7 + assert status["listed_at"] is None and status["documents"] == 0 + assert got["documents"] == [] and got["total_matches"] == 0 + + +def test_unreachable_root_without_cache_is_unavailable_not_error( + tmp_path, cache_dir, make_library +): + root = str(tmp_path / "lib") + scanner = FakeScanner(OSError("share offline")) + got = make_library([root], scanner).find() + [status] = got["roots"] + assert status["status"] == "unavailable" + assert status["error"].startswith("OSError") + assert "share offline" in status["error"] + assert status["listed_at"] is None + assert got["documents"] == [] + + +def test_any_scan_exception_is_recorded_not_raised(tmp_path, cache_dir, make_library): + root = str(tmp_path / "lib") + scanner = FakeScanner(ValueError("bad entry")) + status = make_library([root], scanner).find()["roots"][0] + assert status["status"] == "unavailable" + assert status["error"] == "ValueError: bad entry" + + +def test_unreachable_root_with_cache_serves_old_list(tmp_path, cache_dir, make_library): + root = str(tmp_path / "lib") + _seed(cache_dir, root, ["old.pdf"], age=60) + scanner = FakeScanner(OSError("share offline")) + lib = make_library([root], scanner) + got = lib.find(refresh=True) + assert _names(got) == ["old.pdf"] + [status] = got["roots"] + assert status["status"] == "ok" + assert status["error"].startswith("OSError") + assert status["listed_at"] is not None + # The fresh list is still served without another attempt; the error stays. + got = lib.find() + assert _names(got) == ["old.pdf"] + assert got["roots"][0]["error"].startswith("OSError") + assert scanner.calls == [root] + + +def test_failed_scan_without_cache_is_retried_on_next_call( + tmp_path, cache_dir, make_library +): + root = str(tmp_path / "lib") + scanner = FakeScanner(OSError("share offline"), _lists("a.pdf")) + lib = make_library([root], scanner) + assert lib.find()["roots"][0]["status"] == "unavailable" + got = lib.find() + assert got["roots"][0]["status"] == "ok" + assert "error" not in got["roots"][0] + assert _names(got) == ["a.pdf"] + assert scanner.calls == [root, root] + + +def test_overlapping_roots_list_each_pdf_once(tmp_path, cache_dir, make_library): + lib_dir = tmp_path / "lib" + _touch(lib_dir / "Datasheet" / "a.pdf") + _touch(lib_dir / "b.pdf") + outer, inner = str(lib_dir), str(lib_dir / "Datasheet") + got = make_library([outer, inner], scan_root).find() + assert [d["path"] for d in got["documents"]] == [ + str(lib_dir / "b.pdf"), + str(lib_dir / "Datasheet" / "a.pdf"), + ] + assert got["total_matches"] == 2 + assert [r["documents"] for r in got["roots"]] == [2, 1] + + +def test_limit_truncates_and_reports_total(tmp_path, cache_dir, make_library): + root = str(tmp_path / "lib") + _seed(cache_dir, root, ["tle-a.pdf", "tle-b.pdf", "tle-c.pdf", "x.pdf"], age=60) + lib = make_library([root], FakeScanner(_lists())) + got = lib.find("tle", limit=2) + assert _names(got) == ["tle-a.pdf", "tle-b.pdf"] + assert got["total_matches"] == 3 and got["truncated"] is True + assert len(lib.find(limit=0)["documents"]) == 1 + assert lib.find(limit=MAX_LIMIT + 1)["truncated"] is False + + +def test_result_fields_are_formatted(tmp_path, cache_dir, make_library): + root = str(tmp_path / "lib") + save_cache(cache_file(cache_dir, root), _listing(root, ["a.pdf"], size=1025)) + lib = make_library([root], FakeScanner(_lists())) + got = lib.find() + json.dumps(got) + [doc] = got["documents"] + assert doc == { + "path": os.path.join(root, "a.pdf"), + "modified": datetime.fromtimestamp(NOW).strftime("%Y-%m-%d"), + "size_kb": 2, + } + assert re.fullmatch(r"\d{4}-\d{2}-\d{2}", doc["modified"]) + listed_at = datetime.fromisoformat(got["roots"][0]["listed_at"]) + assert listed_at.tzinfo is not None + assert listed_at.timestamp() == NOW + assert got["roots"][0]["skipped_folders"] == 0 + assert lib.roots == [root] + + +def test_successful_scan_is_persisted_for_the_next_process( + tmp_path, cache_dir, make_library +): + root = str(tmp_path / "lib") + assert _names(make_library([root], FakeScanner(_lists("a.pdf"))).find()) == [ + "a.pdf" + ] + second = FakeScanner(OSError("must not scan")) + got = make_library([root], second).find() + assert _names(got) == ["a.pdf"] + assert second.calls == [] + + +def test_cache_save_failure_is_logged_not_raised( + tmp_path, cache_dir, make_library, monkeypatch, caplog +): + def fail(path, result): + raise OSError("disk full") + + monkeypatch.setattr(library, "save_cache", fail) + root = str(tmp_path / "lib") + with caplog.at_level(logging.WARNING, logger="datasheetindex.library"): + got = make_library([root], FakeScanner(_lists("a.pdf"))).find() + assert _names(got) == ["a.pdf"] + assert got["roots"][0]["status"] == "ok" + assert "disk full" in caplog.text + + +@pytest.mark.parametrize( + "text", + [ + _cache_text("ROOT", size="Infinity"), + _cache_text("ROOT", listed_at="NaN"), + "[" * 100_000 + "]" * 100_000, + ], + ids=["size-inf", "listed-nan", "deep"], +) +def test_warm_on_a_damaged_cache_rescans_instead_of_raising( + tmp_path, cache_dir, make_library, text +): + root = str(tmp_path / "lib") + path = cache_file(cache_dir, root) + path.parent.mkdir(parents=True) + path.write_text(text.replace('"ROOT"', json.dumps(root)), encoding="utf-8") + scanner = FakeScanner(_lists("new.pdf")) + lib = make_library([root], scanner) + lib.warm() + assert lib.wait_for_scans(timeout=5) + assert scanner.calls == [root] + assert _names(lib.find()) == ["new.pdf"] + + +def test_list_from_the_future_is_stale(tmp_path, cache_dir, make_library): + # Another host sharing the cache folder, with its clock ahead of ours. + root = str(tmp_path / "lib") + _seed(cache_dir, root, ["old.pdf"], age=-3600) + scanner = FakeScanner(_lists("new.pdf")) + lib = make_library([root], scanner) + lib.warm() + assert lib.wait_for_scans(timeout=5) + assert scanner.calls == [root] + assert _names(lib.find()) == ["new.pdf"] + + +def test_skipped_files_are_reported_per_root(tmp_path, cache_dir, make_library): + root = str(tmp_path / "lib") + listing = _listing(root, ["a.pdf"]) + save_cache( + cache_file(cache_dir, root), + ScanResult(listing.root, listing.listed_at, listing.entries, 0, 3), + ) + got = make_library([root], FakeScanner(_lists())).find() + assert got["roots"][0]["skipped_files"] == 3 + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX byte file names only") +def test_unencodable_name_is_left_out_of_the_response( + tmp_path, cache_dir, make_library +): + lib_dir = tmp_path / "lib" + _touch(lib_dir / "TLE-UM.pdf") + _make_unencodable_pdf(lib_dir) + lib = make_library([str(lib_dir)], scan_root) + got = lib.find() + assert _names(got) == ["TLE-UM.pdf"] + assert got["roots"][0]["skipped_files"] == 1 + json.dumps(got, ensure_ascii=False).encode("utf-8") + assert load_cache(cache_file(cache_dir, str(lib_dir)), str(lib_dir)) is not None + + +def test_tool_def_description_names_roots_and_schema_caps_limit(tmp_path): + from datasheetindex.library import create_find_documents_def + + lib = DocumentLibrary(["/a", "/b"], tmp_path) + d = create_find_documents_def(lib) + assert d.name == "find_documents" + assert "Library roots: /a; /b." in d.description + props = d.input_schema["properties"] + assert props["limit"]["maximum"] == MAX_LIMIT + assert props["limit"]["default"] == 50 + assert props["query"]["default"] == "" + assert props["refresh"]["default"] is False diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index fbf92df..bd4af8b 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -11,6 +11,8 @@ import asyncio import importlib import json +import os +import sys import types import pymupdf @@ -667,3 +669,168 @@ def test_local_server_reports_installed_version(): assert server.mcp_server.version == package_version() assert server.mcp_server.version != "1.0.0" + + +def _instructions(server): + return server.mcp_server.instructions + + +def test_no_library_env_serves_exactly_the_five_tools(monkeypatch): + pytest.importorskip("mcp.types") + from datasheetindex.mcp_server import SERVER_INSTRUCTIONS, create_local_mcp_server + from datasheetindex.tools.defs import create_datasheet_tool_defs + + monkeypatch.setenv("DATASHEETINDEX_LIBRARY_ROOTS", " ") + server = create_local_mcp_server() + assert server.library is None + assert {t.name for t in _list_tools(server.mcp_server)} == { + d.name for d in create_datasheet_tool_defs() + } + assert _instructions(server) == SERVER_INSTRUCTIONS + + +def _library_server(tmp_path, monkeypatch, root=None): + from datasheetindex.mcp_server import create_local_mcp_server + + root = root if root is not None else tmp_path / "lib" + monkeypatch.setenv("DATASHEETINDEX_LIBRARY_ROOTS", str(root)) + monkeypatch.setenv("DATASHEETINDEX_OUTPUT_DIR", str(tmp_path / "out")) + return create_local_mcp_server() + + +def _payload(result): + return json.loads(result.content[0].text) + + +def test_library_env_adds_find_documents(tmp_path, monkeypatch): + pytest.importorskip("mcp.types") + from datasheetindex.library import LIBRARY_INSTRUCTIONS + from datasheetindex.mcp_server import SERVER_INSTRUCTIONS + + (tmp_path / "lib").mkdir() + server = _library_server(tmp_path, monkeypatch) + try: + tools = {t.name: t for t in _list_tools(server.mcp_server)} + assert "find_documents" in tools + assert str(tmp_path / "lib") in tools["find_documents"].description + assert _instructions(server) == SERVER_INSTRUCTIONS + " " + ( + LIBRARY_INSTRUCTIONS + ) + assert _instructions(server).endswith(LIBRARY_INSTRUCTIONS) + finally: + server.library.wait_for_scans(10) + + +def test_find_documents_end_to_end_feeds_build_datasheet(tmp_path, monkeypatch): + pytest.importorskip("mcp.types") + _make_pdf_dir = tmp_path / "lib" / "TLE" + _make_pdf_dir.mkdir(parents=True) + _make_pdf(_make_pdf_dir / "Infineon-TLE987x-DS.pdf") + server = _library_server(tmp_path, monkeypatch) + try: + found = _payload( + _call(server.mcp_server, "find_documents", {"query": "tle987"}) + ) + docs = found["documents"] + assert len(docs) == 1 + built = _call( + server.mcp_server, "build_datasheet", {"pdf_source": docs[0]["path"]} + ) + assert not built.is_error + assert _payload(built)["total_pages"] == 1 + finally: + server.library.wait_for_scans(10) + + +def test_find_documents_limit_over_max_is_a_schema_error(tmp_path, monkeypatch): + pytest.importorskip("mcp.types") + (tmp_path / "lib").mkdir() + server = _library_server(tmp_path, monkeypatch) + try: + result = _call(server.mcp_server, "find_documents", {"limit": 501}) + assert result.is_error + assert result.content[0].text.startswith("Input validation error") + finally: + server.library.wait_for_scans(10) + + +def test_unreachable_root_does_not_stop_server_start(tmp_path, monkeypatch): + pytest.importorskip("mcp.types") + server = _library_server(tmp_path, monkeypatch, root=tmp_path / "missing") + try: + result = _call(server.mcp_server, "find_documents", {}) + assert not result.is_error + assert _payload(result)["roots"][0]["status"] == "unavailable" + finally: + server.library.wait_for_scans(10) + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX byte env values only") +def test_root_that_is_not_valid_utf8_does_not_stop_server_start(tmp_path, monkeypatch): + pytest.importorskip("mcp.types") + from datasheetindex.tools.defs import create_datasheet_tool_defs + + server = _library_server(tmp_path, monkeypatch, root="/data/Datenbl\udce4tter") + assert server.library is None + assert {t.name for t in _list_tools(server.mcp_server)} == { + d.name for d in create_datasheet_tool_defs() + } + + +def test_library_roots_from_dotenv_register_find_documents(tmp_path, monkeypatch): + """A ``.env`` that names the roots configures the library like the env does. + + The roots are read when the server is created, before any tool call has + loaded ``.env``; ``_hermetic_llm_env`` keeps a developer's real one out. + """ + pytest.importorskip("mcp.types") + from datasheetindex.mcp_server import create_local_mcp_server + + (tmp_path / "lib").mkdir() + + def load_dotenv(*_args, **_kwargs): + monkeypatch.setenv("DATASHEETINDEX_LIBRARY_ROOTS", str(tmp_path / "lib")) + monkeypatch.setenv("DATASHEETINDEX_OUTPUT_DIR", str(tmp_path / "out")) + return True + + monkeypatch.delenv("DATASHEETINDEX_OUTPUT_DIR", raising=False) + monkeypatch.setitem( + sys.modules, "dotenv", types.SimpleNamespace(load_dotenv=load_dotenv) + ) + server = create_local_mcp_server() + try: + assert server.library is not None + assert "find_documents" in {t.name for t in _list_tools(server.mcp_server)} + assert server.library.wait_for_scans(10) + # The cache follows the .env output dir, beside the artifacts. + assert list((tmp_path / "out" / "library").glob("*.json")) + finally: + server.library.wait_for_scans(10) + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX byte file names only") +def test_find_documents_result_with_unencodable_name_reaches_the_wire( + tmp_path, monkeypatch +): + """The stdio transport serializes the result; a lone surrogate killed it.""" + pytest.importorskip("mcp.types") + lib = tmp_path / "lib" + lib.mkdir() + (lib / "TLE-UM.pdf").write_bytes(b"x") + try: + with open(os.path.join(os.fsencode(lib), b"TLE\xff-DS.pdf"), "wb") as f: + f.write(b"x") + except (OSError, ValueError): + pytest.skip("this filesystem cannot hold a non-UTF-8 file name") + server = _library_server(tmp_path, monkeypatch) + try: + result = _call(server.mcp_server, "find_documents", {}) + assert not result.is_error + result.model_dump_json(by_alias=True, exclude_none=True) + payload = _payload(result) + assert [os.path.basename(d["path"]) for d in payload["documents"]] == [ + "TLE-UM.pdf" + ] + assert payload["roots"][0]["skipped_files"] == 1 + finally: + server.library.wait_for_scans(10) diff --git a/uv.lock b/uv.lock index b635f46..f654bc8 100644 --- a/uv.lock +++ b/uv.lock @@ -354,7 +354,7 @@ wheels = [ [[package]] name = "datasheetindex" -version = "0.45.0" +version = "0.46.0" source = { editable = "." } dependencies = [ { name = "pymupdf" },