From 000f803753cc71eb1ac9ffa30660b9067817ddd9 Mon Sep 17 00:00:00 2001 From: Nelson Mfinda Date: Thu, 7 May 2026 18:47:59 +0100 Subject: [PATCH 1/2] feat(scraper): content-hash provenance and enrichment cache --- ...nt-hash-provenance-and-enrichment-cache.md | 78 ++++++++++++++++ CHANGELOG.md | 17 ++++ src/king_context/scraper/chunk.py | 15 +++- src/king_context/scraper/enrich.py | 34 ++++++- src/king_context/scraper/enrich_cache.py | 71 +++++++++++++++ src/king_context/scraper/export.py | 27 +++++- src/king_context/scraper/fetch.py | 16 ++++ tests/conftest.py | 12 +++ tests/test_scraper/test_chunk.py | 29 ++++++ tests/test_scraper/test_enrich.py | 2 +- tests/test_scraper/test_enrich_cache.py | 88 +++++++++++++++++++ tests/test_scraper/test_export.py | 24 +++++ tests/test_scraper/test_fetch.py | 22 +++++ 13 files changed, 429 insertions(+), 6 deletions(-) create mode 100644 .king-context/adr/0012-content-hash-provenance-and-enrichment-cache.md create mode 100644 src/king_context/scraper/enrich_cache.py create mode 100644 tests/test_scraper/test_enrich_cache.py diff --git a/.king-context/adr/0012-content-hash-provenance-and-enrichment-cache.md b/.king-context/adr/0012-content-hash-provenance-and-enrichment-cache.md new file mode 100644 index 0000000..dd0f028 --- /dev/null +++ b/.king-context/adr/0012-content-hash-provenance-and-enrichment-cache.md @@ -0,0 +1,78 @@ +--- +id: ADR-0012 +title: Content-hash provenance for chunks and a file per hash enrichment cache +status: accepted +date: 2026-05-07 +areas: + - scraper + - corpus + - cache + - performance +supersedes: [] +superseded_by: [] +related: + - ADR-0009 + - ADR-0010 +keywords: + - content-hash + - provenance + - enrichment-cache + - sha256 + - delta-update + - drift-detection + - cache-key + - prompt-version +tags: + - architecture + - scraper + - cache +--- + + +# ADR-0012: Content-hash provenance for chunks and a file-per-hash enrichment cache + +## Context + +`king-scrape` today is a one-shot pipeline. Re-running it against the same docs site silently skips any URL whose `.md` file already exists in `.king-context/_temp//pages/` (`src/king_context/scraper/fetch.py:61`), which makes "the upstream docs changed last week — refresh my corpus" a `rm -rf` operation. Re-`rm` forces a full re-fetch and a full re-enrichment. Enrichment is the expensive step (OpenRouter, $1-3 per FastAPI-sized corpus), and almost none of those calls are necessary: most sections do not change between scrapes. + +Two follow on capabilities drift detection and incremental refresh both depend on knowing whether a given chunk's content is identical to what was previously enriched. The exported corpus (`data/.json`) carries no such information today: no `content_hash`, no `fetched_at`, no `scraper_version`. The enrichment stage's resume logic (`src/king_context/scraper/enrich.py`) keys off batch position (`already_enriched = len(previous_data); chunks = chunks[already_enriched:]`), which silently misaligns enrichment with chunks if the upstream docs add or remove a section between runs. + +ADR-0010 established that scraper providers stay thin and the pipeline owns checkpoint, concurrency, and disk IO. The work below sits in the pipeline layer for exactly that reason — it is independent of which scrape backend (Firecrawl, Crawl4AI) produced the markdown. + +## Decision + +Add content-hash provenance at every layer of the pipeline, and a small file-per-hash cache for enrichment outputs. + +1. **Per-page sidecar.** For each fetched page, write `pages/.meta.json` next to `pages/.md` containing `{ url, slug, content_hash (sha256), fetched_at (UTC ISO), byte_size }`. Sidecars are atomic per-file, so they are race-free under the existing concurrent fetch model with no additional locking. +2. **`Chunk.content_hash`.** Extend `Chunk` (and `EnrichedChunk`) with a `content_hash` field populated by `sha256(content)` via `__post_init__`. The field auto-computes when constructors are called with the existing positional or keyword arguments, so every existing call site keeps working unchanged. +3. **Enriched checkpoint includes the hash.** The per-batch checkpoint files (`enriched/batch_NNNN.json`) gain a `content_hash` per item; the resume reader reads it back, falling back to recomputing from `content` when an old checkpoint lacks the field. +4. **Exported `_meta`.** Each section in `data/.json` gains `_meta.content_hash`, and the top-level dict gains `_meta = { schema_version, scraper_version, scraped_at, source_url, section_count }`. All `_meta` fields are optional — consumers (the MCP server, `kctx`, existing readers) ignore unknown keys, so older corpora keep working with no migration. +5. **Enrichment cache.** A new module `src/king_context/scraper/enrich_cache.py` stores enrichment results at `.king-context/cache/enrichment/.json`. The cache key is `sha256(content + "|" + model_id + "|" + prompt_version)`. Writes are atomic via `tempfile.NamedTemporaryFile` + `os.replace`; reads return `None` on miss, JSON decode error, or any IO error. The cache is wired into `_enrich_one` so a content/model/prompt-stable chunk pays zero LLM cost on subsequent runs. A `PROMPT_VERSION = "1"` constant in `enrich.py` is bumped when `ENRICHMENT_PROMPT` changes — that is the cache's correctness handle. + +## Alternatives Considered + +**`cachehash` PyPI library.** Considered as a backing store. Rejected on two grounds: (a) license, the package is published as "free for non commercial use only", which is incompatible with king-context's MIT license and would taint downstream consumers; (b) shape opaque SQLite key-value with eviction/TTL hooks we do not need, when the project ethos (ADR-0010) is pipeline owned IO with inspectable on disk artifacts. File per hash JSON gives `cat .king-context/cache/enrichment/.json` for free and lets a contributor invalidate one entry by deleting one file. + +**SQLite index alongside files.** Discussed as a "best of both worlds" hybrid. Deferred until disk-stat overhead is shown by measurement to matter; for typical corpora (30-200 chunks), 200 stat calls is microseconds and the simpler layout wins. + +**Single aggregated `pages/_manifest.json`.** Rejected in favour of per-page sidecars. A single manifest would need an in-memory lock or merge-on-write logic to be safe under the concurrent fetch model in `fetch.py`. Per-page files are atomic by construction. An aggregated view is trivially derivable on demand if a future stage needs one. + +**Cache key without model + prompt version.** Rejected as a real footgun: a prompt edit or model swap would silently serve stale enrichments. Putting both into the key means changes invalidate the cache automatically. + +## Consequences + +A fresh scrape now produces richer artifacts (per-page sidecars, content hashes through every chunk, `_meta` in the exported JSON) without changing the public command, the schema consumed by `seed_data`, or the provider Protocols. Existing corpora (`data/*.json` shipped before this change) keep working — the optional `_meta` fields just aren't there. + +The enrichment cache makes a re-run on unchanged content free, which is the foundation a future `--update` flag needs (ADR-0014). It is also a meaningful day-one win for contributors who scrape, tweak the chunker, and re-scrape — chunks whose content survives the re-chunk pay no LLM cost. + +The cache directory grows monotonically under `.king-context/cache/enrichment/`. At ~2 KB per entry and typical corpus sizes, this is bounded enough to ignore for the foreseeable future. If it ever matters, `rm -rf .king-context/cache/enrichment/` is a complete, debuggable invalidation. + +The hash on each chunk and section is the data dependency drift detection (ADR-0013) and incremental refresh (ADR-0014) build on. Without it, both are impossible to implement correctly. With it, both reduce to small focused PRs. + +## Links + +src/king_context/scraper/enrich_cache.py +src/king_context/scraper/chunk.py +src/king_context/scraper/fetch.py +src/king_context/scraper/enrich.py +src/king_context/scraper/export.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e2fe67..87ffe08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Content-hash provenance through every layer of the scraper pipeline + (ADR-0012). `Chunk` and `EnrichedChunk` carry a `content_hash` field + populated by `sha256(content)`. Each fetched page now writes a sidecar + `pages/.meta.json` containing `{ url, slug, content_hash, + fetched_at, byte_size }`. The exported `data/.json` gains an + optional `_meta.content_hash` per section and a top-level `_meta` with + `schema_version`, `scraper_version`, `scraped_at`, `source_url`, and + `section_count`. All `_meta` fields are optional; older corpora and + consumers that don't recognise them continue to work unchanged. +- File-per-hash enrichment cache at + `.king-context/cache/enrichment/.json`. Cache key is + `sha256(content + model_id + prompt_version)`. Writes are atomic via + `tempfile` + `os.replace`. A re-run on unchanged content (or a re-chunk + that produces structurally identical chunks) pays zero LLM cost. + ## [0.4.0] - 2026-05-06 ### Added diff --git a/src/king_context/scraper/chunk.py b/src/king_context/scraper/chunk.py index f12636e..4859e11 100644 --- a/src/king_context/scraper/chunk.py +++ b/src/king_context/scraper/chunk.py @@ -1,13 +1,14 @@ +import hashlib import json import re -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from urllib.parse import urlparse from king_context.scraper.config import ScraperConfig -@dataclass +@dataclass(frozen=True) class Chunk: title: str breadcrumb: str @@ -15,6 +16,15 @@ class Chunk: source_url: str path: str token_count: int + content_hash: str = field(default="") + + def __post_init__(self) -> None: + if not self.content_hash: + object.__setattr__( + self, + "content_hash", + hashlib.sha256(self.content.encode("utf-8")).hexdigest(), + ) def _estimate_tokens(text: str) -> int: @@ -200,6 +210,7 @@ def chunk_pages(pages_dir: Path, output_dir: Path, config: ScraperConfig) -> lis "source_url": c.source_url, "path": c.path, "token_count": c.token_count, + "content_hash": c.content_hash, } for c in page_chunks ] diff --git a/src/king_context/scraper/enrich.py b/src/king_context/scraper/enrich.py index 699c978..f850f23 100644 --- a/src/king_context/scraper/enrich.py +++ b/src/king_context/scraper/enrich.py @@ -1,9 +1,11 @@ import asyncio +import hashlib import json -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Any +from king_context.scraper import enrich_cache from king_context.scraper.chunk import Chunk from king_context.scraper.config import ScraperConfig from king_context.scraper.discover import _update_step @@ -37,7 +39,12 @@ Return only the JSON object, no explanation.""" -@dataclass +# Derived from the prompt itself so any edit to ENRICHMENT_PROMPT automatically +# invalidates cached entries — no human discipline required. +PROMPT_VERSION = hashlib.sha256(ENRICHMENT_PROMPT.encode("utf-8")).hexdigest()[:16] + + +@dataclass(frozen=True) class EnrichedChunk: title: str path: str @@ -47,6 +54,15 @@ class EnrichedChunk: use_cases: list[str] tags: list[str] priority: int + content_hash: str = field(default="") + + def __post_init__(self) -> None: + if not self.content_hash: + object.__setattr__( + self, + "content_hash", + hashlib.sha256(self.content.encode("utf-8")).hexdigest(), + ) def validate_enrichment(enrichment: dict) -> list[str]: @@ -155,6 +171,7 @@ def _to_enriched_chunk(chunk: Chunk, enrichment: dict) -> EnrichedChunk: use_cases=enrichment["use_cases"], tags=enrichment["tags"], priority=enrichment["priority"], + content_hash=chunk.content_hash, ) @@ -215,6 +232,15 @@ async def _enrich_one( schema_fallback: LLMClient | None = None, ) -> EnrichedChunk | None: """Enrich a single chunk with up to 2 retries on validation failure.""" + cache_key = enrich_cache.make_key( + chunk.content, + primary_client.model, + PROMPT_VERSION, + ) + cached = enrich_cache.get(cache_key) + if cached is not None and not validate_enrichment(cached): + return _to_enriched_chunk(chunk, cached) + prompt = ENRICHMENT_PROMPT.format(title=chunk.title, content=chunk.content) for attempt in range(3): # 1 initial attempt + 2 retries @@ -222,6 +248,7 @@ async def _enrich_one( enrichment = await primary_client.complete(prompt) errors = validate_enrichment(enrichment) if not errors: + enrich_cache.put(cache_key, enrichment) return _to_enriched_chunk(chunk, enrichment) except ProviderError as exc: if not exc.transient or attempt == 2: @@ -241,6 +268,7 @@ async def _enrich_one( enrichment = await schema_fallback.complete(prompt) fallback_errors = validate_enrichment(enrichment) if not fallback_errors: + enrich_cache.put(cache_key, enrichment) return _to_enriched_chunk(chunk, enrichment) raise ProviderError( "validation_failed_3x", @@ -296,6 +324,7 @@ async def enrich_chunks( use_cases=item["use_cases"], tags=item["tags"], priority=item["priority"], + content_hash=item.get("content_hash", ""), )) total_chunks = len(chunks) @@ -355,6 +384,7 @@ async def guarded(chunk: Chunk) -> EnrichedChunk | None: "use_cases": e.use_cases, "tags": e.tags, "priority": e.priority, + "content_hash": e.content_hash, } for e in enriched ] diff --git a/src/king_context/scraper/enrich_cache.py b/src/king_context/scraper/enrich_cache.py new file mode 100644 index 0000000..982397c --- /dev/null +++ b/src/king_context/scraper/enrich_cache.py @@ -0,0 +1,71 @@ +"""Content addressed cache for enrichment results. + +Stores LLM enrichment output keyed by sha256 of (chunk content + model id + prompt +version). File-per-hash JSON layout (`/.json`) so each entry is +independently inspectable, debuggable, and trivially invalidated by deleting one +file or the whole directory. + +Atomic writes via ``tempfile.NamedTemporaryFile`` + ``os.replace`` so a crash mid-write +never leaves half-written JSON. Reads return ``None`` on miss, JSON decode error, or +any IO error callers treat every failure as a cache miss and re-enrich. + +The cache key intentionally bakes in the model identifier and prompt version so that +a prompt edit or model swap silently invalidates prior entries. Forgetting that is +the most likely correctness footgun in this layer, so it is enforced by signature. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +from pathlib import Path +from typing import Any + +from king_context import PROJECT_ROOT + + +DEFAULT_CACHE_DIR = PROJECT_ROOT / ".king-context" / "cache" / "enrichment" + + +def make_key(content: str, model: str, prompt_version: str) -> str: + """Return the cache key for a ``(content, model, prompt_version)`` triple.""" + payload = f"{content}|{model}|{prompt_version}".encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def get(key: str, cache_dir: Path | None = None) -> dict[str, Any] | None: + """Return the cached enrichment dict for ``key``, or ``None`` on any failure.""" + cache_dir = cache_dir if cache_dir is not None else DEFAULT_CACHE_DIR + path = cache_dir / f"{key}.json" + try: + return json.loads(path.read_text(encoding="utf-8")) + except (FileNotFoundError, OSError, json.JSONDecodeError): + return None + + +def put(key: str, value: dict[str, Any], cache_dir: Path | None = None) -> None: + """Atomically write ``value`` to the cache under ``key``. Best-effort, never raises.""" + cache_dir = cache_dir if cache_dir is not None else DEFAULT_CACHE_DIR + tmp_name: str | None = None + try: + # Serialize before any IO so a TypeError on a non-JSON value short-circuits + # before we create a tempfile that could leak. + payload = json.dumps(value, ensure_ascii=False) + cache_dir.mkdir(parents=True, exist_ok=True) + fd, tmp_name = tempfile.mkstemp( + dir=cache_dir, prefix=f".{key}.", suffix=".tmp" + ) + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.write(payload) + os.replace(tmp_name, cache_dir / f"{key}.json") + tmp_name = None + except (OSError, TypeError, ValueError): + return + finally: + if tmp_name is not None: + try: + os.unlink(tmp_name) + except OSError: + pass diff --git a/src/king_context/scraper/export.py b/src/king_context/scraper/export.py index f7c2cb7..a76715c 100644 --- a/src/king_context/scraper/export.py +++ b/src/king_context/scraper/export.py @@ -1,15 +1,27 @@ import json +from datetime import datetime, timezone +from importlib.metadata import PackageNotFoundError, version as _pkg_version from pathlib import Path from king_context import seed_data from king_context.scraper.enrich import EnrichedChunk +CORPUS_SCHEMA_VERSION = 1 + + def _sanitize_path(path: str) -> str: """Sanitize section path for filesystem safety — replace / with - to avoid subdirs.""" return path.replace("/", "-").strip("-") +def _scraper_version() -> str: + try: + return _pkg_version("king-context") + except PackageNotFoundError: + return "unknown" + + def export_to_json( enriched_chunks: list[EnrichedChunk], doc_name: str, @@ -19,7 +31,10 @@ def export_to_json( ) -> dict: """Build a King Context documentation dict from enriched chunks. - The returned dict matches the schema expected by seed_data.seed_one(). + The returned dict matches the schema expected by seed_data.seed_one(). The + optional ``_meta`` fields carry provenance (content hashes, scrape timestamp, + scraper version) used by drift detection and incremental refresh; consumers + that don't recognise them ignore them. """ sections = [ { @@ -31,6 +46,9 @@ def export_to_json( "tags": chunk.tags, "priority": chunk.priority, "content": chunk.content, + "_meta": { + "content_hash": chunk.content_hash, + }, } for chunk in enriched_chunks ] @@ -40,6 +58,13 @@ def export_to_json( "version": version, "base_url": base_url, "sections": sections, + "_meta": { + "schema_version": CORPUS_SCHEMA_VERSION, + "scraper_version": _scraper_version(), + "scraped_at": datetime.now(timezone.utc).isoformat(), + "source_url": base_url, + "section_count": len(sections), + }, } diff --git a/src/king_context/scraper/fetch.py b/src/king_context/scraper/fetch.py index d400fc0..0298bca 100644 --- a/src/king_context/scraper/fetch.py +++ b/src/king_context/scraper/fetch.py @@ -1,6 +1,9 @@ import asyncio +import hashlib +import json import re from dataclasses import dataclass +from datetime import datetime, timezone from pathlib import Path from scraper_providers import FetchProvider @@ -32,6 +35,18 @@ def _url_to_slug(url: str) -> str: return slug[:200] +def _write_page_meta(pages_dir: Path, slug: str, url: str, markdown: str) -> None: + encoded = markdown.encode("utf-8") + meta = { + "url": url, + "slug": slug, + "content_hash": hashlib.sha256(encoded).hexdigest(), + "fetched_at": datetime.now(timezone.utc).isoformat(), + "byte_size": len(encoded), + } + (pages_dir / f"{slug}.meta.json").write_text(json.dumps(meta, indent=2)) + + async def _fetch_one( url: str, semaphore: asyncio.Semaphore, @@ -43,6 +58,7 @@ async def _fetch_one( page = await provider.fetch_one(url) markdown = page.markdown slug = _url_to_slug(url) + _write_page_meta(pages_dir, slug, url, markdown) (pages_dir / f"{slug}.md").write_text(markdown) return PageResult(url=url, markdown=markdown, success=True, error=None) except Exception as e: diff --git a/tests/conftest.py b/tests/conftest.py index b540e40..2b81fac 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,9 +2,21 @@ import pytest import king_context.db as db +from king_context.scraper import enrich_cache from types import SimpleNamespace +@pytest.fixture(autouse=True) +def _isolate_enrich_cache(tmp_path, monkeypatch): + """Redirect the enrichment cache to a fresh tmp dir per test. + + Tests routinely reuse identical chunk content; without isolation, the + persistent on-disk cache short-circuits the LLM call paths the tests + are exercising. + """ + monkeypatch.setattr(enrich_cache, "DEFAULT_CACHE_DIR", tmp_path / "_enrich_cache") + + class FakeLLMClient: name = "openrouter" model = "test-model" diff --git a/tests/test_scraper/test_chunk.py b/tests/test_scraper/test_chunk.py index 250112b..4b418f8 100644 --- a/tests/test_scraper/test_chunk.py +++ b/tests/test_scraper/test_chunk.py @@ -167,3 +167,32 @@ def test_chunk_token_count(): assert len(chunks) == 1 expected = int(100 * 1.33) assert chunks[0].token_count == expected + + +def test_chunk_content_hash_is_populated(): + chunk = Chunk( + title="t", + breadcrumb="t", + content="hello world", + source_url="https://example.com", + path="/p", + token_count=2, + ) + assert len(chunk.content_hash) == 64 + assert all(c in "0123456789abcdef" for c in chunk.content_hash) + + +def test_chunk_content_hash_is_deterministic(): + a = Chunk(title="", breadcrumb="", content="same content", + source_url="u", path="p", token_count=1) + b = Chunk(title="", breadcrumb="", content="same content", + source_url="u", path="p", token_count=1) + assert a.content_hash == b.content_hash + + +def test_chunk_content_hash_changes_with_content(): + a = Chunk(title="", breadcrumb="", content="A", + source_url="u", path="p", token_count=1) + b = Chunk(title="", breadcrumb="", content="B", + source_url="u", path="p", token_count=1) + assert a.content_hash != b.content_hash diff --git a/tests/test_scraper/test_enrich.py b/tests/test_scraper/test_enrich.py index 5872f3c..d5209ab 100644 --- a/tests/test_scraper/test_enrich.py +++ b/tests/test_scraper/test_enrich.py @@ -41,7 +41,7 @@ def test_enrich_batch_processing(): openrouter_api_key="test-key", enrichment_batch_size=5, ) - chunks = [make_chunk(f"Section {i}") for i in range(12)] + chunks = [make_chunk(f"Section {i}", content=f"Body {i}") for i in range(12)] call_count = 0 diff --git a/tests/test_scraper/test_enrich_cache.py b/tests/test_scraper/test_enrich_cache.py new file mode 100644 index 0000000..51c8d5c --- /dev/null +++ b/tests/test_scraper/test_enrich_cache.py @@ -0,0 +1,88 @@ +import json +from pathlib import Path + +from king_context.scraper import enrich_cache + + +SAMPLE_ENRICHMENT = { + "keywords": ["kw1", "kw2", "kw3", "kw4", "kw5"], + "use_cases": ["Use when X", "Configure when Y"], + "tags": ["testing"], + "priority": 7, +} + + +def test_make_key_is_deterministic(): + a = enrich_cache.make_key("hello", "model-1", "1") + b = enrich_cache.make_key("hello", "model-1", "1") + assert a == b + assert len(a) == 64 + + +def test_make_key_changes_with_content(): + a = enrich_cache.make_key("hello", "model-1", "1") + b = enrich_cache.make_key("hello world", "model-1", "1") + assert a != b + + +def test_make_key_changes_with_model(): + a = enrich_cache.make_key("same", "model-1", "1") + b = enrich_cache.make_key("same", "model-2", "1") + assert a != b + + +def test_make_key_changes_with_prompt_version(): + a = enrich_cache.make_key("same", "model-1", "1") + b = enrich_cache.make_key("same", "model-1", "2") + assert a != b + + +def test_get_returns_none_on_missing(tmp_path: Path): + assert enrich_cache.get("deadbeef", cache_dir=tmp_path) is None + + +def test_get_returns_none_on_corrupt_json(tmp_path: Path): + (tmp_path / "abc.json").write_text("{not valid json") + assert enrich_cache.get("abc", cache_dir=tmp_path) is None + + +def test_put_then_get_roundtrip(tmp_path: Path): + enrich_cache.put("abc", SAMPLE_ENRICHMENT, cache_dir=tmp_path) + got = enrich_cache.get("abc", cache_dir=tmp_path) + assert got == SAMPLE_ENRICHMENT + + +def test_put_creates_cache_dir(tmp_path: Path): + cache_dir = tmp_path / "deeply" / "nested" / "cache" + enrich_cache.put("abc", SAMPLE_ENRICHMENT, cache_dir=cache_dir) + assert (cache_dir / "abc.json").exists() + + +def test_put_atomic_no_partial_files(tmp_path: Path): + enrich_cache.put("abc", SAMPLE_ENRICHMENT, cache_dir=tmp_path) + assert not list(tmp_path.glob("*.tmp")) + assert not list(tmp_path.glob(".*.tmp")) + final = list(tmp_path.glob("*.json")) + assert len(final) == 1 + assert final[0].name == "abc.json" + + +def test_put_does_not_leak_tmp_on_non_serializable(tmp_path: Path): + bad_value = {"keywords": {1, 2, 3}} + enrich_cache.put("abc", bad_value, cache_dir=tmp_path) + assert not list(tmp_path.glob("*.tmp")) + assert not list(tmp_path.glob(".*.tmp")) + assert not list(tmp_path.glob("*.json")) + + +def test_put_overwrites_existing(tmp_path: Path): + enrich_cache.put("abc", SAMPLE_ENRICHMENT, cache_dir=tmp_path) + new_value = {**SAMPLE_ENRICHMENT, "priority": 1} + enrich_cache.put("abc", new_value, cache_dir=tmp_path) + assert enrich_cache.get("abc", cache_dir=tmp_path) == new_value + + +def test_put_does_not_raise_on_io_error(tmp_path: Path): + target_file = tmp_path / "blocker" + target_file.write_text("not a directory") + enrich_cache.put("abc", SAMPLE_ENRICHMENT, cache_dir=target_file) diff --git a/tests/test_scraper/test_export.py b/tests/test_scraper/test_export.py index 097582a..354bc72 100644 --- a/tests/test_scraper/test_export.py +++ b/tests/test_scraper/test_export.py @@ -102,3 +102,27 @@ def test_save_and_index_creates_parent_dirs(tmp_path): save_and_index(doc_data, output_path, auto_seed=True) assert output_path.exists() + + +def test_export_section_carries_content_hash(): + chunk = make_enriched_chunk() + result = export_to_json([chunk], "example", "Example", "https://example.com") + section = result["sections"][0] + assert "_meta" in section + assert section["_meta"]["content_hash"] == chunk.content_hash + assert len(section["_meta"]["content_hash"]) == 64 + + +def test_export_top_level_meta(): + result = export_to_json( + [make_enriched_chunk()], + "example", + "Example", + "https://docs.example.com", + ) + meta = result["_meta"] + assert meta["schema_version"] == 1 + assert meta["source_url"] == "https://docs.example.com" + assert meta["section_count"] == 1 + assert "scraped_at" in meta + assert "scraper_version" in meta diff --git a/tests/test_scraper/test_fetch.py b/tests/test_scraper/test_fetch.py index 07b768e..89c680a 100644 --- a/tests/test_scraper/test_fetch.py +++ b/tests/test_scraper/test_fetch.py @@ -116,3 +116,25 @@ def test_fetch_updates_manifest(tmp_path): assert "fetch" in manifest assert manifest["fetch"]["completed"] == 1 assert manifest["fetch"]["total"] == 1 + + +def test_fetch_writes_page_meta_sidecar(tmp_path): + config = ScraperConfig(firecrawl_api_key="fc-test") + provider = _FakeFetchProvider(fixed_markdown="# Hello\n\nbody") + + urls = ["https://docs.example.com/api/intro"] + asyncio.run(fetch_pages(urls, tmp_path, config, provider)) + + pages_dir = tmp_path / "pages" + md_files = list(pages_dir.glob("*.md")) + assert len(md_files) == 1 + + sidecars = list(pages_dir.glob("*.meta.json")) + assert len(sidecars) == 1 + + meta = json.loads(sidecars[0].read_text()) + assert meta["url"] == urls[0] + assert meta["slug"] == md_files[0].stem + assert len(meta["content_hash"]) == 64 + assert meta["byte_size"] == len("# Hello\n\nbody".encode("utf-8")) + assert "fetched_at" in meta From 6dca77603bd2dabe7c104e356c3b3911f3b132d9 Mon Sep 17 00:00:00 2001 From: Nelson Mfinda Date: Thu, 7 May 2026 19:38:24 +0100 Subject: [PATCH 2/2] feat(scraper): content-hash provenance and enrichment cache --- src/king_context/scraper/enrich_cache.py | 15 ++++++++++++--- tests/test_scraper/test_enrich_cache.py | 11 +++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/king_context/scraper/enrich_cache.py b/src/king_context/scraper/enrich_cache.py index 982397c..a8b7608 100644 --- a/src/king_context/scraper/enrich_cache.py +++ b/src/king_context/scraper/enrich_cache.py @@ -30,9 +30,18 @@ def make_key(content: str, model: str, prompt_version: str) -> str: - """Return the cache key for a ``(content, model, prompt_version)`` triple.""" - payload = f"{content}|{model}|{prompt_version}".encode("utf-8") - return hashlib.sha256(payload).hexdigest() + """Return the cache key for a ``(content, model, prompt_version)`` triple. + + Each component is hashed to a fixed-length digest before concatenation, then + the concatenation is hashed. A character delimiter would in principle allow + collisions when ``content`` contains the delimiter (markdown tables use ``|``); + fixed-length digests make boundary ambiguity impossible. + """ + digests = b"".join( + hashlib.sha256(part.encode("utf-8")).digest() + for part in (content, model, prompt_version) + ) + return hashlib.sha256(digests).hexdigest() def get(key: str, cache_dir: Path | None = None) -> dict[str, Any] | None: diff --git a/tests/test_scraper/test_enrich_cache.py b/tests/test_scraper/test_enrich_cache.py index 51c8d5c..154604d 100644 --- a/tests/test_scraper/test_enrich_cache.py +++ b/tests/test_scraper/test_enrich_cache.py @@ -37,6 +37,17 @@ def test_make_key_changes_with_prompt_version(): assert a != b +def test_make_key_no_delimiter_collision_with_pipe_in_content(): + # Markdown tables put `|` inside content. Length-prefixed digests must keep + # these distinguishable rather than colliding via a string-join boundary. + a = enrich_cache.make_key("foo|model-1|1", "model-2", "2") + b = enrich_cache.make_key("foo", "model-1", "1|model-2|2") + c = enrich_cache.make_key("foo", "model-1", "1") + assert a != b + assert a != c + assert b != c + + def test_get_returns_none_on_missing(tmp_path: Path): assert enrich_cache.get("deadbeef", cache_dir=tmp_path) is None