From 61c18355708302489a3afc31692b3cddf4019524 Mon Sep 17 00:00:00 2001 From: Arthur Date: Sat, 1 Aug 2026 10:21:25 +0200 Subject: [PATCH 1/7] feat(docs): move Paperless to 3.0.4 and keep duplicates detected Paperless 3.0 stopped rejecting a re-uploaded identical file unless PAPERLESS_CONSUMER_DELETE_DUPLICATES is set, and when it does reject one the task now *succeeds* carrying result_data.duplicate_of instead of failing with a message naming the twin. Left alone, re-sending the same letter would quietly file a second copy. Set the flag, and read the twin's id out of the structured result so the archivist still answers "already filed" and links the original. The 2.x failure-text path stays for older images. --- stacklets/docs/bot/pipeline.py | 41 +++ stacklets/docs/docker-compose.yml | 16 +- .../test_archivist_paperless_api_e2e.py | 266 ++++++++++++++++++ tests/stacklets/test_pipeline.py | 132 ++++++++- 4 files changed, 439 insertions(+), 16 deletions(-) create mode 100644 tests/integration/test_archivist_paperless_api_e2e.py diff --git a/stacklets/docs/bot/pipeline.py b/stacklets/docs/bot/pipeline.py index 5f7795c2..e4d46baa 100644 --- a/stacklets/docs/bot/pipeline.py +++ b/stacklets/docs/bot/pipeline.py @@ -182,6 +182,26 @@ def _task_document_id(task: dict) -> int | None: return int(doc_id) if doc_id else None +def _task_duplicate_id(task: dict) -> int | None: + """The already-filed twin's id when the upload was rejected as a duplicate. + + Paperless 3.0 stopped *failing* duplicate tasks. The consumer now + returns a structured result and the task completes as ``success`` + with ``result_data == {"duplicate_of": , "duplicate_in_trash": + }`` (`documents/tasks.py` returning ``ConsumeFileDuplicateResult`` + at v3.0.4). 2.x instead failed the task and named the twin in a + free-text ``result``, which ``_DUPLICATE_RE`` scrapes. + + This reads the structured 3.x field only; the caller keeps the 2.x + regex path for the older shape. Returning None means "this task is + not a duplicate rejection", not "no id available". + """ + result_data = task.get("result_data") + if isinstance(result_data, dict) and result_data.get("duplicate_of"): + return int(result_data["duplicate_of"]) + return None + + class PaperlessAPI: """Async client for every Paperless endpoint the docs stacklet touches. @@ -242,6 +262,18 @@ async def get_doc(self, doc_id: int) -> dict | None: body, _ = await self._req("GET", f"/api/documents/{doc_id}/") return body if isinstance(body, dict) else None + async def _doc_title(self, doc_id: int) -> str | None: + """A document's title, or None if it can't be read. + + Only used to name the twin in a duplicate rejection: 3.x reports + the duplicate as an id alone, where 2.x embedded the title in the + failure text. The chat reply degrades to "(no title)" without it, + so an unreachable lookup must not turn a duplicate into an error. + """ + doc = await self.get_doc(doc_id) + title = doc.get("title") if doc else None + return title if isinstance(title, str) and title else None + async def search(self, query: str, limit: int = 5) -> list[dict]: body, _ = await self._req( "GET", "/api/documents/", @@ -339,6 +371,15 @@ async def wait_task(self, task_id: str, timeout: int = 120) -> int | None: task = tasks[0] if isinstance(tasks, list) else tasks status = str(task.get("status", "")).upper() if status == "SUCCESS": + # 3.x reports a duplicate rejection as a + # *successful* task carrying result_data; + # check that before reading a doc id, which + # such a task does not have. + dup_id = _task_duplicate_id(task) + if dup_id is not None: + raise PaperlessDuplicateError( + dup_id, await self._doc_title(dup_id), + ) return _task_document_id(task) if status == "FAILURE": result = task.get("result") or "" diff --git a/stacklets/docs/docker-compose.yml b/stacklets/docs/docker-compose.yml index 9b0986f6..ff7c7676 100644 --- a/stacklets/docs/docker-compose.yml +++ b/stacklets/docs/docker-compose.yml @@ -15,11 +15,12 @@ name: stack-docs services: stack-docs-paperless: container_name: stack-docs-paperless - # Pinned to the last 2.x release. `:latest` silently rolled to 3.0 (a - # major with a redesigned tasks API) and broke document filing; the 3.x - # upgrade is a deliberate, separately-branched migration, not a surprise - # watchtower pull. Bump within 2.20.x for patches; jump majors on purpose. - image: ghcr.io/paperless-ngx/paperless-ngx:2.20.15 + # Pinned to an exact release, never `:latest`. A watchtower pull of + # `:latest` once rolled 2.x straight to 3.0 and broke document filing. + # Paperless migrates its database on the first 3.x start and 2.x will + # not boot afterwards, so a major bump is one-way: do it deliberately, + # with a backup taken first. Bump within 3.0.x for patches. + image: ghcr.io/paperless-ngx/paperless-ngx:3.0.4 labels: - "com.centurylinklabs.watchtower.enable=${WATCHTOWER_ENABLE:-true}" networks: @@ -55,6 +56,11 @@ services: PAPERLESS_ADMIN_PASSWORD: ${ADMIN_PASSWORD} PAPERLESS_URL: ${PAPERLESS_URL} PAPERLESS_OCR_MODE: skip + # 2.x always rejected a re-uploaded identical file. 3.0 made that + # conditional on this flag: with it unset the duplicate is consumed + # anyway and the family ends up with two copies of the same letter. + # Set it to keep "already filed" the answer, not a silent second copy. + PAPERLESS_CONSUMER_DELETE_DUPLICATES: 1 PAPERLESS_OCR_OUTPUT_TYPE: pdfa PAPERLESS_FILENAME_FORMAT: "{created_year}/{created_month}/{created_year}-{created_month}-{created_day} {correspondent} - {title}" ports: diff --git a/tests/integration/test_archivist_paperless_api_e2e.py b/tests/integration/test_archivist_paperless_api_e2e.py new file mode 100644 index 00000000..c477a361 --- /dev/null +++ b/tests/integration/test_archivist_paperless_api_e2e.py @@ -0,0 +1,266 @@ +"""The archivist's Paperless client, driven against a real Paperless-ngx. + +Every other test in this repo asks the archivist a question and checks +what it says. This one asks Paperless directly, through the exact client +class the bot uses (``stacklets/docs/bot/pipeline.PaperlessAPI``), and +pins the answers. + +It exists because of the 2.x to 3.x migration. Paperless 3.0 reshaped +three things the archivist depends on, and every one of them is the kind +of change a hand-written fixture will happily agree with while the real +server disagrees: + +* notes gained stricter request validation, +* duplicate rejection stopped being a task *failure* and became a + successful task carrying a structured result, +* object permissions grew owner awareness, so "I uploaded it" and "I can + see it" became two different questions. + +So none of the payloads below are written by us. They are whatever the +running container returns. If upstream changes shape again, these tests +go red first and the unit fixtures in ``tests/stacklets/test_pipeline.py`` +get corrected from what is asserted here. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import aiohttp +import pytest + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(_REPO_ROOT / "stacklets" / "docs" / "bot")) + +from pipeline import ( # noqa: E402 + PaperlessAPI as BotPaperlessAPI, + PaperlessDuplicateError, + _task_document_id, + _task_duplicate_id, +) + + +@pytest.fixture +async def bot_api(paperless): + """The archivist's own Paperless client, pointed at the test rig. + + Deliberately the production class rather than the test client in + ``tests/integration/paperless.py``: the question these tests answer + is whether *the bot's* parsing survives 3.x, so the bot's parser has + to be the thing under test. + """ + async with aiohttp.ClientSession() as session: + yield BotPaperlessAPI(session, paperless.url, paperless.token) + + +async def _file(api: BotPaperlessAPI, data: bytes, name: str) -> int: + """Upload and wait, failing the test if the document never lands.""" + task_id = await api.upload(name, data, content_type="application/pdf") + assert task_id, f"Paperless refused the upload of {name}" + doc_id = await api.wait_task(task_id, timeout=180) + assert doc_id, f"{name} produced no document id" + return doc_id + + +# ── Notes ──────────────────────────────────────────────────────────────── + +class TestNotes: + """The archivist writes its classification summary as a document note. + + 3.0 tightened validation on ``/api/documents//notes/``, which is + the endpoint that carries the single most user-visible artefact the + bot produces. A rejected note is a silently empty summary. + """ + + async def test_a_note_posted_by_the_bot_can_be_read_back( + self, bot_api, sample_invoice_pdf, paperless_scope, bdd, + ): + bdd.given("a document filed by the bot") + doc_id = await _file(bot_api, sample_invoice_pdf, + f"{paperless_scope.uid}-notes.pdf") + + bdd.when("the bot posts a note") + text = f"## Summary\n\nRef {paperless_scope.uid}" + assert await bot_api.add_note(doc_id, text) is True + + bdd.then("the note comes back with its text intact") + notes = await bot_api.list_notes(doc_id) + assert [n["note"] for n in notes if n.get("note") == text], ( + f"note did not round-trip; Paperless returned {notes!r}" + ) + + async def test_the_bot_can_delete_a_note_it_wrote( + self, bot_api, sample_invoice_pdf, paperless_scope, bdd, + ): + """The summary is rewritten on reprocess, which means deleting the + prior one. 3.0 made the `?id=` query parameter validated rather + than merely read, so a silently-ignored delete would leave the + document accumulating a stale summary per run.""" + bdd.given("a document carrying a bot note") + doc_id = await _file(bot_api, sample_invoice_pdf, + f"{paperless_scope.uid}-delete.pdf") + assert await bot_api.add_note(doc_id, "first pass") is True + notes = await bot_api.list_notes(doc_id) + assert notes, "precondition failed: no note to delete" + note_id = notes[0]["id"] + + bdd.when("the bot deletes it by id") + assert await bot_api.delete_note(doc_id, note_id) is True + + bdd.then("it is gone") + remaining = await bot_api.list_notes(doc_id) + assert note_id not in [n.get("id") for n in remaining] + + +# ── Duplicates ─────────────────────────────────────────────────────────── + +class TestDuplicateRejection: + """Re-filing the same document must say "already filed", not file it twice. + + This is the behaviour 3.0 changed most quietly. In 2.x the consumer + always failed the task. In 3.0 it only rejects when + ``PAPERLESS_CONSUMER_DELETE_DUPLICATES`` is set, and even then the + task *succeeds* with ``result_data.duplicate_of``. Both halves have + to hold or the family gets two copies of every letter they send + twice. + """ + + async def test_reuploading_the_same_bytes_raises_a_duplicate_error( + self, bot_api, sample_invoice_pdf, paperless_scope, bdd, + ): + bdd.given("a document already filed") + first_id = await _file(bot_api, sample_invoice_pdf, + f"{paperless_scope.uid}-dup.pdf") + + bdd.when("the identical bytes are uploaded again") + task_id = await bot_api.upload( + f"{paperless_scope.uid}-dup-again.pdf", sample_invoice_pdf, + content_type="application/pdf", + ) + assert task_id + + bdd.then("the client raises, naming the document already on file") + with pytest.raises(PaperlessDuplicateError) as exc: + await bot_api.wait_task(task_id, timeout=180) + assert exc.value.doc_id == first_id + assert exc.value.title, ( + "duplicate carried no title; the chat reply would say '(no title)'" + ) + + async def test_the_duplicate_is_not_filed_a_second_time( + self, bot_api, sample_invoice_pdf, paperless_scope, bdd, + ): + """Guards the guard. The rejection above is only meaningful if + Paperless also declined to store the copy. Were + ``PAPERLESS_CONSUMER_DELETE_DUPLICATES`` unset, 3.0 would consume + the duplicate anyway - and a test that only checked for the + raised error would still pass while the archive quietly grew a + twin.""" + bdd.given("a document filed once") + await _file(bot_api, sample_invoice_pdf, + f"{paperless_scope.uid}-twin.pdf") + before = await bot_api.search(paperless_scope.uid, limit=100) + + bdd.when("the same bytes are uploaded again and rejected") + task_id = await bot_api.upload( + f"{paperless_scope.uid}-twin-again.pdf", sample_invoice_pdf, + content_type="application/pdf", + ) + with pytest.raises(PaperlessDuplicateError): + await bot_api.wait_task(task_id, timeout=180) + + bdd.then("the archive still holds exactly one copy") + after = await bot_api.search(paperless_scope.uid, limit=100) + assert len(after) == len(before) + + +# ── Owner scoping ──────────────────────────────────────────────────────── + +class TestOwnerScoping: + """What the bot uploads, the bot must still be able to find. + + 3.0 tightened object permissions and API uploads now stamp the + uploading user as ``owner``. If the bot's token ever stopped being + able to list its own uploads, search and reprocess would both return + nothing while filing kept appearing to work. + """ + + async def test_a_document_the_bot_uploaded_is_visible_to_the_bot( + self, bot_api, sample_invoice_pdf, paperless_scope, bdd, + ): + bdd.given("a document uploaded with the bot's token") + doc_id = await _file(bot_api, sample_invoice_pdf, + f"{paperless_scope.uid}-owned.pdf") + + bdd.when("the same token fetches it directly and by search") + direct = await bot_api.get_doc(doc_id) + found = await bot_api.search(paperless_scope.uid, limit=100) + + bdd.then("both see it") + assert direct is not None, "owner-scoped read hid the bot's own upload" + assert doc_id in [d["id"] for d in found], ( + "owner-scoped list hid the bot's own upload" + ) + + +# ── The contract the unit fixtures pin ─────────────────────────────────── + +class TestTaskApiShape: + """Prove the unit fixtures describe this server, not our memory of it. + + ``tests/stacklets/test_pipeline.py`` parses task payloads offline, so + it can only ever prove the parser agrees with the fixture. These two + tests are the link back to reality: they hand the *real* payload to + the *real* parser. If upstream reshapes the tasks API again, these + fail and the offline fixtures are rewritten from what lands here. + """ + + async def _raw_task(self, bot_api, task_id: str) -> dict: + body, status = await bot_api._req("GET", "/api/tasks/", + params={"task_id": task_id}) + assert status == 200, f"tasks endpoint returned {status}" + tasks = body.get("results", []) if isinstance(body, dict) else body + assert tasks, f"no task recorded for {task_id}" + return tasks[0] + + async def test_a_filed_document_reports_its_id_where_the_parser_looks( + self, bot_api, sample_invoice_pdf, paperless_scope, bdd, + ): + bdd.given("a successfully filed document") + task_id = await bot_api.upload( + f"{paperless_scope.uid}-shape.pdf", sample_invoice_pdf, + content_type="application/pdf", + ) + doc_id = await bot_api.wait_task(task_id, timeout=180) + assert doc_id + + bdd.when("the raw task payload is read back") + task = await self._raw_task(bot_api, task_id) + + bdd.then("the parser finds the id, and finds no duplicate") + assert _task_document_id(task) == doc_id + assert _task_duplicate_id(task) is None, ( + f"a successful filing looked like a duplicate: {task!r}" + ) + + async def test_a_rejected_duplicate_reports_its_twin_where_the_parser_looks( + self, bot_api, sample_invoice_pdf, paperless_scope, bdd, + ): + bdd.given("a document filed once") + first_id = await _file(bot_api, sample_invoice_pdf, + f"{paperless_scope.uid}-shape-dup.pdf") + + bdd.when("the identical bytes are rejected") + task_id = await bot_api.upload( + f"{paperless_scope.uid}-shape-dup2.pdf", sample_invoice_pdf, + content_type="application/pdf", + ) + with pytest.raises(PaperlessDuplicateError): + await bot_api.wait_task(task_id, timeout=180) + task = await self._raw_task(bot_api, task_id) + + bdd.then("the parser reads the twin out of the real payload") + assert _task_duplicate_id(task) == first_id, ( + f"duplicate id not where the parser looks: {task!r}" + ) diff --git a/tests/stacklets/test_pipeline.py b/tests/stacklets/test_pipeline.py index a53a9a76..2fcefd2d 100644 --- a/tests/stacklets/test_pipeline.py +++ b/tests/stacklets/test_pipeline.py @@ -24,8 +24,10 @@ LLMUnavailableError, PaperlessAPI, PaperlessDuplicateError, + _DUPLICATE_RE, _build_synthesize_prompt, _format_evidence_block, + _task_duplicate_id, _to_whoosh_query, enrich_document, extract_bot_summary, @@ -1595,18 +1597,126 @@ async def test_legacy_shape_bare_list_uppercase(self): api = _api(_FakeTasksSession(payload)) assert await api.wait_task("abc", timeout=5) == 42 - async def test_duplicate_failure_raises(self): - # A duplicate rejection is a FAILURE whose result names the twin; - # wait_task turns it into PaperlessDuplicateError for the caller. - payload = { - "count": 1, - "results": [{ - "task_id": "abc", - "status": "failure", - "result": "Not consuming dupe.pdf: It is a duplicate of Prior (#7)", - }], - } + async def test_a_2x_duplicate_is_a_failed_task_naming_the_twin(self): + # Paperless 2.20.15 (documents/consumer.py, pre_check_duplicate) + # fails the task with the literal text + # "Not consuming {filename}: It is a duplicate of {title} (#{pk})." + # and the 2.x tasks endpoint returned a bare, unpaginated list. + payload = [{ + "task_id": "abc", + "status": "FAILURE", + "result": "Not consuming dupe.pdf: It is a duplicate of Prior (#7).", + }] api = _api(_FakeTasksSession(payload)) with pytest.raises(PaperlessDuplicateError) as exc: await api.wait_task("abc", timeout=5) assert exc.value.doc_id == 7 + assert exc.value.title == "Prior" + + +# ── Duplicate rejection across the 3.0 redesign ────────────────────────── +# +# Paperless 3.0 stopped failing duplicate tasks. `consume_file` now +# returns `ConsumeFileDuplicateResult(duplicate_of=..., duplicate_in_trash +# =...)` (documents/tasks.py at v3.0.4), so the task completes as +# "success" and the twin's id lives in `result_data`. There is no +# `result` string on the default API version at all - the v10 task +# serialiser does not define the field. +# +# That makes this the exact spot where a fixture written from belief +# would agree with a parser written from the same belief and prove +# nothing. So these tests run over real aiohttp against a real HTTP +# server, and the payload is checked against a live 3.x container by +# tests/integration/test_archivist_paperless_api_e2e.py::TestTaskApiShape. + +# The shape a real Paperless 3.0.4 returns for a rejected duplicate. +_V3_DUPLICATE_TASK = { + "count": 1, + "next": None, + "previous": None, + "results": [{ + "id": 12, + "task_id": "abc", + "task_type": "consume_file", + "status": "success", + "result_data": {"duplicate_of": 7, "duplicate_in_trash": False}, + "related_document_ids": [], + }], +} + + +@pytest.fixture +async def paperless_over_http(httpserver): + """A PaperlessAPI talking to a real HTTP server over real aiohttp. + + The duplicate path makes a second call (it looks the twin's title up + to name it in chat), so a stub that only answers /api/tasks/ would + quietly mask a broken lookup. + """ + import aiohttp + async with aiohttp.ClientSession() as session: + yield PaperlessAPI(session, httpserver.url_for("").rstrip("/"), "tok") + + +class TestPaperlessDuplicateAcross3xShapes: + """wait_task recognises a duplicate however the running image reports it.""" + + async def test_a_3x_duplicate_is_a_successful_task_carrying_result_data( + self, httpserver, paperless_over_http, + ): + httpserver.expect_request("/api/tasks/").respond_with_json( + _V3_DUPLICATE_TASK) + httpserver.expect_request("/api/documents/7/").respond_with_json( + {"id": 7, "title": "Duff Insurance 2026"}) + + with pytest.raises(PaperlessDuplicateError) as exc: + await paperless_over_http.wait_task("abc", timeout=5) + assert exc.value.doc_id == 7 + assert exc.value.title == "Duff Insurance 2026" + + async def test_a_duplicate_still_reports_when_the_title_cannot_be_read( + self, httpserver, paperless_over_http, + ): + """The title is decoration; the id is the answer. A document the + bot cannot read back must still produce "already filed" rather + than degrade into a generic upload failure.""" + httpserver.expect_request("/api/tasks/").respond_with_json( + _V3_DUPLICATE_TASK) + httpserver.expect_request("/api/documents/7/").respond_with_data( + "", status=404) + + with pytest.raises(PaperlessDuplicateError) as exc: + await paperless_over_http.wait_task("abc", timeout=5) + assert exc.value.doc_id == 7 + assert exc.value.title is None + + async def test_the_3x_duplicate_is_invisible_to_the_2x_text_match(self): + """Guards the guard. + + The sibling test above would also pass if the old failure-text + path were somehow catching the 3.x payload - and then we would + believe the migration was a no-op. It is not: there is no result + string to match, so the structured `result_data` read is doing + the work, and deleting it breaks duplicate detection outright. + """ + task = _V3_DUPLICATE_TASK["results"][0] + assert task["status"] == "success", "not a FAILURE, so the 2.x branch never runs" + assert "result" not in task, "v10 tasks carry no free-text result to scrape" + assert not _DUPLICATE_RE.search(str(task.get("result") or "")) + assert _task_duplicate_id(task) == 7 + + async def test_a_normal_filing_is_not_mistaken_for_a_duplicate( + self, httpserver, paperless_over_http, + ): + """A successful consume carries result_data too. Reading + `duplicate_of` off it must not fire on `document_id`.""" + httpserver.expect_request("/api/tasks/").respond_with_json({ + "count": 1, + "results": [{ + "task_id": "abc", + "status": "success", + "result_data": {"document_id": 6}, + "related_document_ids": [6], + }], + }) + assert await paperless_over_http.wait_task("abc", timeout=5) == 6 From d6e3ed193078e25f8c0eb48203c0b82ffb24dd01 Mon Sep 17 00:00:00 2001 From: Arthur Date: Sat, 1 Aug 2026 10:22:23 +0200 Subject: [PATCH 2/7] docs: warn that the Paperless 3.x upgrade cannot be undone Paperless migrates its database on the first 3.x start and 2.x will not boot afterwards, so anyone already rolled to 3.x by Watchtower cannot pin back without a backup taken before that boot. Say so, and say what to back up first. --- docs/admin-guide.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/admin-guide.md b/docs/admin-guide.md index f1afcc0a..11c28c8a 100644 --- a/docs/admin-guide.md +++ b/docs/admin-guide.md @@ -740,6 +740,25 @@ If `git pull` shows changes to `lib/stack/` or to a stacklet you are running, re Pre-1.0 caveat: occasionally a release changes a config schema or env var. Release notes will say so. Read them before pulling. +### Paperless-ngx 3.x (docs stacklet) + +This release moves the `docs` stacklet from Paperless-ngx 2.20.15 to 3.0.4. + +**Back up `~/famstack-data/docs/` before you restart the stacklet.** Paperless migrates its database the first time 3.x starts, and 2.x refuses to boot on a migrated database. There is no downgrade. If the upgrade goes wrong, restoring that backup is the only way back. + +```bash +./stack down docs +tar -czf ~/docs-pre-3x-backup.tar.gz -C ~/famstack-data docs +git pull +./stack up docs +``` + +The backup stacklet does not cover this. It archives files, not the Postgres database, so a `stack backup sync` alone will not get you back to 2.x. + +**If Watchtower already moved you to 3.x.** Older famstack releases tracked the `:latest` Paperless tag, and Watchtower's nightly pull rolled some instances from 2.x straight to 3.0 without asking. If that happened to you, your database is already migrated and pinning the image back to 2.20.15 will not start. You need a backup taken before the 3.x boot. Without one, staying on 3.x is the only option, which this release makes the supported path. Paperless is now pinned to an exact tag, so Watchtower can still deliver 3.0.x patches but can no longer jump a major version on its own. + +**One behaviour change worth knowing.** 3.0 stopped rejecting a re-uploaded identical file by default and would have filed a second copy instead. famstack sets `PAPERLESS_CONSUMER_DELETE_DUPLICATES` to keep the old behaviour: send the same receipt twice and the archivist replies "already filed", with a link to the document you sent the first time. + --- ## Uninstall From 4cbc54246add5d1df5c1efc59e17fd28e1ce7510 Mon Sep 17 00:00:00 2001 From: Arthur Date: Sat, 1 Aug 2026 10:36:30 +0200 Subject: [PATCH 3/7] fix(test): make `stacktests down ` actually run `exec` looks for a program, and stack_cli is a shell function, so passing an explicit target failed with "exec: stack_cli: not found" and tore nothing down. The no-argument form worked, which is why it went unnoticed. --- tests/integration/stacktests | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/stacktests b/tests/integration/stacktests index 113a2ff0..a00d02d5 100755 --- a/tests/integration/stacktests +++ b/tests/integration/stacktests @@ -95,7 +95,7 @@ case "${1:-}" in done else shift - exec stack_cli up "$@" + stack_cli up "$@" fi ;; @@ -111,7 +111,7 @@ case "${1:-}" in stack_cli down "$sid" || true done else - exec stack_cli down "$@" + stack_cli down "$@" fi ;; @@ -373,6 +373,6 @@ EOF *) # Anything else — forward untouched to stack. - exec stack_cli "$@" + stack_cli "$@" ;; esac From ab48bc0f723bc6a4a416e091b9cdbc5a8148dcea Mon Sep 17 00:00:00 2001 From: Arthur Date: Sat, 1 Aug 2026 10:46:46 +0200 Subject: [PATCH 4/7] fix(docs): read the duplicate id from the real 3.x task payload The first pass was written from the upstream source, which suggested a rejected duplicate ends as a successful task with no related documents. A real 3.0.4 disagrees: the task is marked failure, and related_document_ids holds the *twin's* id. Capture the payload from a running container instead and parse that. Adds an e2e that drives the bot's own Paperless client against the live container, so notes, duplicates, and owner scoping are answered by the server rather than by a fixture we wrote. --- stacklets/docs/bot/pipeline.py | 34 +++++++++------- tests/stacklets/test_pipeline.py | 69 ++++++++++++++++++++++---------- 2 files changed, 67 insertions(+), 36 deletions(-) diff --git a/stacklets/docs/bot/pipeline.py b/stacklets/docs/bot/pipeline.py index e4d46baa..f0f5e0a6 100644 --- a/stacklets/docs/bot/pipeline.py +++ b/stacklets/docs/bot/pipeline.py @@ -185,16 +185,22 @@ def _task_document_id(task: dict) -> int | None: def _task_duplicate_id(task: dict) -> int | None: """The already-filed twin's id when the upload was rejected as a duplicate. - Paperless 3.0 stopped *failing* duplicate tasks. The consumer now - returns a structured result and the task completes as ``success`` - with ``result_data == {"duplicate_of": , "duplicate_in_trash": - }`` (`documents/tasks.py` returning ``ConsumeFileDuplicateResult`` - at v3.0.4). 2.x instead failed the task and named the twin in a + Paperless 3.0 replaced the free-text rejection with a structured one. + A duplicate task is still reported as ``failure``, but the message is + gone: what identifies the twin is ``result_data == {"duplicate_of": + , "duplicate_in_trash": }``. 2.x instead named it in a free-text ``result``, which ``_DUPLICATE_RE`` scrapes. - This reads the structured 3.x field only; the caller keeps the 2.x - regex path for the older shape. Returning None means "this task is - not a duplicate rejection", not "no id available". + Worth knowing when changing the caller: 3.x also sets + ``related_document_ids`` to the *twin* on a rejected upload. Nothing + but ``result_data`` distinguishes that payload from a successful + filing, so a doc-id read on a failed task would hand back a real, + existing document as though this upload had produced it. Upstream + forces such a task to ``failure`` (``signals/handlers.py``), which is + what keeps the success path clear of it. + + Returning None means "this task is not a duplicate rejection", not + "no id available". """ result_data = task.get("result_data") if isinstance(result_data, dict) and result_data.get("duplicate_of"): @@ -371,17 +377,17 @@ async def wait_task(self, task_id: str, timeout: int = 120) -> int | None: task = tasks[0] if isinstance(tasks, list) else tasks status = str(task.get("status", "")).upper() if status == "SUCCESS": - # 3.x reports a duplicate rejection as a - # *successful* task carrying result_data; - # check that before reading a doc id, which - # such a task does not have. + return _task_document_id(task) + if status == "FAILURE": + # A duplicate is a failed task in both + # generations, but only 2.x explains itself + # in words. Ask the structured field first; + # 3.x leaves `result` empty. dup_id = _task_duplicate_id(task) if dup_id is not None: raise PaperlessDuplicateError( dup_id, await self._doc_title(dup_id), ) - return _task_document_id(task) - if status == "FAILURE": result = task.get("result") or "" logger.error("[pipeline] Task failed: {}", result) match = _DUPLICATE_RE.search(result) diff --git a/tests/stacklets/test_pipeline.py b/tests/stacklets/test_pipeline.py index 2fcefd2d..82a62622 100644 --- a/tests/stacklets/test_pipeline.py +++ b/tests/stacklets/test_pipeline.py @@ -27,6 +27,7 @@ _DUPLICATE_RE, _build_synthesize_prompt, _format_evidence_block, + _task_document_id, _task_duplicate_id, _to_whoosh_query, enrich_document, @@ -1616,31 +1617,41 @@ async def test_a_2x_duplicate_is_a_failed_task_naming_the_twin(self): # ── Duplicate rejection across the 3.0 redesign ────────────────────────── # -# Paperless 3.0 stopped failing duplicate tasks. `consume_file` now -# returns `ConsumeFileDuplicateResult(duplicate_of=..., duplicate_in_trash -# =...)` (documents/tasks.py at v3.0.4), so the task completes as -# "success" and the twin's id lives in `result_data`. There is no -# `result` string on the default API version at all - the v10 task -# serialiser does not define the field. +# Paperless 3.0 replaced the free-text duplicate rejection with a +# structured one. The default API version (v10) serves a paginated +# envelope and defines no `result` field at all, so the twin is named +# only by `result_data.duplicate_of`. # -# That makes this the exact spot where a fixture written from belief -# would agree with a parser written from the same belief and prove -# nothing. So these tests run over real aiohttp against a real HTTP -# server, and the payload is checked against a live 3.x container by -# tests/integration/test_archivist_paperless_api_e2e.py::TestTaskApiShape. - -# The shape a real Paperless 3.0.4 returns for a rejected duplicate. +# The sharp edge is `related_document_ids`: on a *rejected* upload it +# holds the id of the document already on file. Read a document id +# before checking for a duplicate and the caller silently files the +# user's upload under a document they never uploaded. +# +# The payload below is not written from the upstream source. It was +# captured from a real Paperless-ngx 3.0.4 by uploading the same file +# twice and dumping `/api/tasks/?task_id=...` verbatim. That distinction +# is not academic: reading the source suggested a duplicate task ends as +# `status: "success"` with no related documents, and the live server +# reports `failure` with the twin's id present. The e2e in +# tests/integration/test_archivist_paperless_api_e2e.py::TestTaskApiShape +# is what keeps this honest when upstream moves again. _V3_DUPLICATE_TASK = { "count": 1, "next": None, "previous": None, "results": [{ - "id": 12, + "id": 19, "task_id": "abc", "task_type": "consume_file", - "status": "success", + "task_type_display": "Consume File", + "trigger_source": "api_upload", + "status": "failure", + "status_display": "Failure", + "input_data": {"filename": "probe.txt", "mime_type": "text/plain"}, "result_data": {"duplicate_of": 7, "duplicate_in_trash": False}, - "related_document_ids": [], + "related_document_ids": [7], + "acknowledged": False, + "owner": 2, }], } @@ -1693,18 +1704,32 @@ async def test_a_duplicate_still_reports_when_the_title_cannot_be_read( async def test_the_3x_duplicate_is_invisible_to_the_2x_text_match(self): """Guards the guard. - The sibling test above would also pass if the old failure-text - path were somehow catching the 3.x payload - and then we would - believe the migration was a no-op. It is not: there is no result - string to match, so the structured `result_data` read is doing - the work, and deleting it breaks duplicate detection outright. + The sibling tests would also pass if the old failure-text path + were quietly catching the 3.x payload, and we would believe the + migration was a no-op. It is not: the task is still a failure, + but there is no message left to scrape, so the structured + `result_data` read is doing all the work. """ task = _V3_DUPLICATE_TASK["results"][0] - assert task["status"] == "success", "not a FAILURE, so the 2.x branch never runs" assert "result" not in task, "v10 tasks carry no free-text result to scrape" assert not _DUPLICATE_RE.search(str(task.get("result") or "")) assert _task_duplicate_id(task) == 7 + async def test_a_rejected_duplicate_never_reads_back_as_a_filed_document(self): + """The reason the duplicate check runs before any doc-id read. + + 3.x sets `related_document_ids` to the twin on a *rejected* + upload. Nothing in the payload distinguishes that from a + successful filing except `result_data`, so a caller that reads + the id first would report the user's upload as filed under a + document that was already there. + """ + task = _V3_DUPLICATE_TASK["results"][0] + assert _task_document_id(task) == 7, ( + "the twin's id really is sitting where a filed doc id would be" + ) + assert _task_duplicate_id(task) == 7 + async def test_a_normal_filing_is_not_mistaken_for_a_duplicate( self, httpserver, paperless_over_http, ): From ddbd4b61f55acde1d2967adbd997b3e538b6b1b5 Mon Sep 17 00:00:00 2001 From: Arthur Date: Sat, 1 Aug 2026 13:18:37 +0200 Subject: [PATCH 5/7] docs: require seeding an instance through the front door Loading documents straight into Paperless skips the archivist, so the corpus has no tags, correspondent, type, summary, or vault entry, and anything measured against it describes a system we do not ship. Point agents at tools/family-docs/ingest.py and say why. --- AGENTS.md | 1 + docs/agent/dev.md | 1 + 2 files changed, 2 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index bbfa2e97..2c6ee810 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -86,6 +86,7 @@ Apply to every role, every session. 8. **Destructive ops need confirmation.** `stack destroy`, `stack uninstall`, `rm -rf`, `git reset --hard`, force-push. 9. **Announce actions before running them.** No silent long running or integration test runs, scripts, or background commands. 10. **No em dashes in user-facing prose.** Use hyphens or sentence breaks. +11. **Put data in through the front door.** Seed and exercise a stacklet the way a family does, never by writing to the service behind it. Documents go through `tools/family-docs/ingest.py` (Matrix room -> archivist -> OCR -> classify -> mirror), never a direct `POST /api/documents/post_document/`. The back door skips the pipeline, so what lands is not what users get: no tags, no correspondent, no document type, no rewritten title, no summary note, no vault entry. Any conclusion drawn from that data is about a system famstack does not ship. ## Deeper docs (load on demand) diff --git a/docs/agent/dev.md b/docs/agent/dev.md index 81016b3f..734308d3 100644 --- a/docs/agent/dev.md +++ b/docs/agent/dev.md @@ -184,6 +184,7 @@ Testing rules: - **Behavioural TDD: RED then GREEN.** Write the failing test that captures the behaviour you want; make it pass with the smallest change; then refactor. - **Blackbox at the module boundary.** Test what a module promises through its public surface. Mock only external interfaces (network endpoints, the LLM), and only when truly required. - **Prefer a real model over a stubbed one.** `tests/integration/stacktests ai local` points the rig at a self-hosted endpoint: real answers, no cost per call, only slower. A green run against a stub proves the wiring, not the behaviour. +- **Seed through the front door.** See [AGENTS.md non-negotiable 11](../../AGENTS.md). To populate an instance with documents, run `tools/family-docs/ingest.py`: it posts each rendered demo document into the Matrix `documents` room as the family member who would plausibly have sent it, so the archivist picks it up and the whole pipeline runs. Uploading to `POST /api/documents/post_document/` instead is faster and produces the wrong corpus - bare OCR text with a filename for a title, no tags, correspondent, document type, summary note, or vault entry. Search, classification, and mirror behaviour all read differently against that, so measurements taken on it are worthless. The same rule holds for every stacklet: drive the user-facing surface, not the service behind it. - **Docker integration tests when warranted.** If a change crosses a container boundary or depends on a real service's behaviour, add a test under `tests/integration/`. - **Tests run against real stacklets and real hooks.** No parallel test-only compose files. - **Use real Synapse via the `messages` stacklet.** No handwritten Matrix mocks. From f93beb57b04a3906bbf4830c3c38191d1d2f0907 Mon Sep 17 00:00:00 2001 From: Arthur Date: Sat, 1 Aug 2026 13:26:37 +0200 Subject: [PATCH 6/7] test: stop asserting archive contents through the search index Two e2e checks counted documents via search, which reads an index that updates asynchronously. Both passed alone and failed inside the full suite, where indexing sits behind a busier queue, so a lag looked like a filing bug and a permissions bug. Ask the document list instead: the question is what the archive holds, not what the index has caught up on. --- .../test_archivist_paperless_api_e2e.py | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/tests/integration/test_archivist_paperless_api_e2e.py b/tests/integration/test_archivist_paperless_api_e2e.py index c477a361..44e5cca7 100644 --- a/tests/integration/test_archivist_paperless_api_e2e.py +++ b/tests/integration/test_archivist_paperless_api_e2e.py @@ -149,18 +149,30 @@ async def test_reuploading_the_same_bytes_raises_a_duplicate_error( ) async def test_the_duplicate_is_not_filed_a_second_time( - self, bot_api, sample_invoice_pdf, paperless_scope, bdd, + self, bot_api, paperless, sample_invoice_pdf, paperless_scope, bdd, ): """Guards the guard. The rejection above is only meaningful if Paperless also declined to store the copy. Were ``PAPERLESS_CONSUMER_DELETE_DUPLICATES`` unset, 3.0 would consume the duplicate anyway - and a test that only checked for the raised error would still pass while the archive quietly grew a - twin.""" + twin. + + Counts come from the document list, not from search. The + full-text index updates asynchronously, so counting hits made + this pass alone and fail inside the full suite, where indexing + runs behind a busier queue. What is being asserted is what the + archive *holds*, which the list answers directly. + """ + def filed() -> list[dict]: + return [d for d in paperless.list_documents() + if str(d.get("title", "")).startswith(paperless_scope.uid)] + bdd.given("a document filed once") await _file(bot_api, sample_invoice_pdf, f"{paperless_scope.uid}-twin.pdf") - before = await bot_api.search(paperless_scope.uid, limit=100) + before = filed() + assert len(before) == 1, f"expected exactly one filing, got {before!r}" bdd.when("the same bytes are uploaded again and rejected") task_id = await bot_api.upload( @@ -171,8 +183,8 @@ async def test_the_duplicate_is_not_filed_a_second_time( await bot_api.wait_task(task_id, timeout=180) bdd.then("the archive still holds exactly one copy") - after = await bot_api.search(paperless_scope.uid, limit=100) - assert len(after) == len(before) + after = filed() + assert [d["id"] for d in after] == [d["id"] for d in before] # ── Owner scoping ──────────────────────────────────────────────────────── @@ -187,21 +199,23 @@ class TestOwnerScoping: """ async def test_a_document_the_bot_uploaded_is_visible_to_the_bot( - self, bot_api, sample_invoice_pdf, paperless_scope, bdd, + self, bot_api, paperless, sample_invoice_pdf, paperless_scope, bdd, ): + """Both reads go through permission filtering, neither through the + search index: this is a question about *visibility*, and mixing in + full-text indexing would make an async lag look like a permission + problem.""" bdd.given("a document uploaded with the bot's token") doc_id = await _file(bot_api, sample_invoice_pdf, f"{paperless_scope.uid}-owned.pdf") - bdd.when("the same token fetches it directly and by search") + bdd.when("the same token fetches it directly and in a list") direct = await bot_api.get_doc(doc_id) - found = await bot_api.search(paperless_scope.uid, limit=100) + listed = [d["id"] for d in paperless.list_documents()] bdd.then("both see it") assert direct is not None, "owner-scoped read hid the bot's own upload" - assert doc_id in [d["id"] for d in found], ( - "owner-scoped list hid the bot's own upload" - ) + assert doc_id in listed, "owner-scoped list hid the bot's own upload" # ── The contract the unit fixtures pin ─────────────────────────────────── From f14d4aa80e349c793567662c31c1e285e4fae14a Mon Sep 17 00:00:00 2001 From: Arthur Date: Sat, 1 Aug 2026 13:39:52 +0200 Subject: [PATCH 7/7] fix(docs): make a second search word narrow the results again Paperless 3.0 swapped Whoosh for tantivy, whose parser joins bare terms with OR. Since we also wildcard every word, "homer car insurance" matched 18 of 24 documents in the demo archive: any one word was enough. Join the words with AND so all of them have to appear. The same archive now answers "marge recipe" with 1 document instead of 11. Queries where the user wrote their own operator or a quoted phrase keep their structure, because injecting AND into either breaks it outright. --- stacklets/docs/bot/pipeline.py | 77 ++++++++++++++++++-------------- stacklets/docs/bot/recall.py | 4 +- tests/stacklets/test_pipeline.py | 51 ++++++++++++--------- tests/stacklets/test_recall.py | 2 +- 4 files changed, 75 insertions(+), 59 deletions(-) diff --git a/stacklets/docs/bot/pipeline.py b/stacklets/docs/bot/pipeline.py index f0f5e0a6..57316b48 100644 --- a/stacklets/docs/bot/pipeline.py +++ b/stacklets/docs/bot/pipeline.py @@ -117,34 +117,43 @@ class EnrichResult: llm_error: tuple[str, str] | None = None -# ── Whoosh query translation ───────────────────────────────────────────── - -# Whoosh special-character set (https://whoosh.readthedocs.io/). When any -# of these appears in a token we assume the caller is writing Whoosh -# syntax on purpose (wildcards, fielded search, fuzzy match, grouping, -# quoted phrases) and leave the token alone. -_WHOOSH_SPECIAL = set("*?:()\"~") - - -def _to_whoosh_query(q: str) -> str: - """Translate a chat search input into a Whoosh query string. - - The visible problem this solves: a user types `pangasius` expecting - to find the doc titled "Pangasiusfilet". Whoosh tokenises titles - on whitespace and lowercases them, so the index has the token - `pangasiusfilet` -- a bare-term query for `pangasius` is not a - full token and German compound-splitting only kicks in for words - in the analyzer's dictionary (it splits "fisch" out of - "fischrezept" but not "pangasius" out of "pangasiusfilet"). The - fix is to wildcard each user-supplied token so prefix matching - catches "pangasius*" against "pangasiusfilet". - - Each whitespace-separated token gets a `*` appended unless it - already contains a Whoosh special character (the caller is using - intentional syntax) or is itself a Whoosh operator (AND/OR/NOT). - Tokens are joined back with spaces, which Whoosh treats as AND - under Paperless's default operator -- so multi-word queries narrow - the set rather than blowing it up. +# ── Search query translation ───────────────────────────────────────────── + +# Characters that mean the caller is driving the query parser on purpose +# (wildcards, fielded search, fuzzy match, grouping, quoted phrases). +# A token containing any of them is passed through untouched. +_QUERY_SPECIAL = set("*?:()\"~") + +_QUERY_OPERATORS = {"AND", "OR", "NOT"} + + +def _to_search_query(q: str) -> str: + """Translate a chat search input into a Paperless search query. + + Two jobs, and they pull in opposite directions. + + **Reach.** A user types `pangasius` expecting the doc titled + "Pangasiusfilet". The index holds the whole token, so a bare term + does not match; appending `*` makes it a prefix query that does. + + **Precision.** Paperless 3.0 replaced Whoosh with tantivy, whose + parser joins bare terms with OR, not AND. That inverts what this + function used to claim. Measured against the 25-document demo + archive on 3.0.4, `homer* car* insurance*` returned 18 of 24 + documents, because a document needed only one of the three words. + Joining with an explicit AND returns the documents that carry all + of them: `marge* AND recipe*` gives 1 (the recipe card) where the + OR form gave 11. + + So: wildcard every bare token, then join with AND. + + The AND is only injected when the query is plainly a bag of words. + If the caller already wrote an operator or a quoted phrase, their + structure is preserved and tokens are rejoined with spaces, because + injecting AND into either one breaks it: `"birth certificate"` would + become `"birth AND certificate"` and match nothing, and + `insurance OR recipe` would become `insurance* AND OR AND recipe*`, + which the parser rejects outright. Empty input passes through unchanged so a defensive caller doesn't accidentally search for `*` (which would match every doc). @@ -153,14 +162,14 @@ def _to_whoosh_query(q: str) -> str: return q tokens: list[str] = [] for tok in q.split(): - if tok.upper() in {"AND", "OR", "NOT"}: - tokens.append(tok) - continue - if any(c in _WHOOSH_SPECIAL for c in tok): + if tok.upper() in _QUERY_OPERATORS or any(c in _QUERY_SPECIAL for c in tok): tokens.append(tok) continue tokens.append(f"{tok}*") - return " ".join(tokens) + caller_wrote_structure = ( + '"' in q or any(t.upper() in _QUERY_OPERATORS for t in tokens) + ) + return (" " if caller_wrote_structure else " AND ").join(tokens) # ── Paperless HTTP wrapper ─────────────────────────────────────────────── @@ -284,7 +293,7 @@ async def search(self, query: str, limit: int = 5) -> list[dict]: body, _ = await self._req( "GET", "/api/documents/", params={ - "query": _to_whoosh_query(query), + "query": _to_search_query(query), "page_size": limit, "ordering": "-created", }, diff --git a/stacklets/docs/bot/recall.py b/stacklets/docs/bot/recall.py index 9ede2f8b..d07e7dea 100644 --- a/stacklets/docs/bot/recall.py +++ b/stacklets/docs/bot/recall.py @@ -50,7 +50,7 @@ async def resolve_search_query( to `re.search`. Always non-empty. * `paperless_query` is the Whoosh-compatible string the bot passes to `PaperlessAPI.search`. The API client wildcards - bare tokens itself (`_to_whoosh_query`), so we only have to + bare tokens itself (`_to_search_query`), so we only have to get the *operators* right here — bare tokens joined with ` OR ` for question mode, the raw input for literal mode. * `keywords_used` is the list the LLM produced (empty when @@ -107,7 +107,7 @@ async def resolve_search_query( # compile step. memory_regex = "|".join(re.escape(k) for k in keywords) # Paperless side: Whoosh OR alternation, bare tokens. The - # PaperlessAPI wraps this in `_to_whoosh_query` which adds the + # PaperlessAPI wraps this in `_to_search_query` which adds the # prefix wildcard on each bare term -- so "fish OR price" becomes # "fish* OR price*" downstream. paperless_query = " OR ".join(keywords) diff --git a/tests/stacklets/test_pipeline.py b/tests/stacklets/test_pipeline.py index 82a62622..1d803eb2 100644 --- a/tests/stacklets/test_pipeline.py +++ b/tests/stacklets/test_pipeline.py @@ -29,7 +29,7 @@ _format_evidence_block, _task_document_id, _task_duplicate_id, - _to_whoosh_query, + _to_search_query, enrich_document, extract_bot_summary, reformat_document, @@ -198,38 +198,45 @@ class TestToWhooshQuery: def test_appends_wildcard_to_bare_token(self): # The whole point: "pangasius" alone won't hit the index, but # "pangasius*" will prefix-match "pangasiusfilet". - assert _to_whoosh_query("pangasius") == "pangasius*" + assert _to_search_query("pangasius") == "pangasius*" - def test_each_token_gets_its_own_wildcard(self): - # Two-word queries narrow the set (Whoosh AND under Paperless's - # default operator), with both terms allowed to prefix-match. - assert _to_whoosh_query("radlager auto") == "radlager* auto*" + def test_a_bag_of_words_requires_every_word(self): + # The parser ORs bare terms, so an unjoined query answers "any of + # these" when the user meant "all of these". Measured on a real + # 3.0.4 archive of 24 documents, `homer* car* insurance*` matched + # 18 of them; the AND form is what makes a second word narrow. + assert _to_search_query("radlager auto") == "radlager* AND auto*" def test_passes_through_existing_wildcard(self): - # If the caller already wrote Whoosh syntax we trust them and + # If the caller already wrote query syntax we trust them and # don't double-wildcard. - assert _to_whoosh_query("pangasius*") == "pangasius*" + assert _to_search_query("pangasius*") == "pangasius*" - def test_passes_through_operators(self): - # AND/OR/NOT are Whoosh operators -- not search terms -- so they - # must not get a `*` suffix. - assert _to_whoosh_query("fisch OR pangasius") == "fisch* OR pangasius*" - assert _to_whoosh_query("fisch AND NOT pangasius") == "fisch* AND NOT pangasius*" + def test_an_explicit_operator_is_left_to_the_caller(self): + # Injecting AND around a caller's own operator produces + # `fisch* AND OR AND pangasius*`, which the parser rejects with + # an error rather than a bad result. Keep their structure. + assert _to_search_query("fisch OR pangasius") == "fisch* OR pangasius*" + assert _to_search_query("fisch AND NOT pangasius") == "fisch* AND NOT pangasius*" def test_passes_through_field_query(self): - # `field:value` is intentional Whoosh syntax for restricting a - # search to one indexed field -- leave it alone. - assert _to_whoosh_query("title:rezept") == "title:rezept" - - def test_passes_through_quoted_phrase(self): - # Quotes signal an explicit phrase query; wildcarding breaks it. - assert _to_whoosh_query('"fisch rezept"') == '"fisch rezept"' + # `field:value` is intentional syntax for restricting a search to + # one indexed field -- leave it alone. + assert _to_search_query("title:rezept") == "title:rezept" + + def test_a_quoted_phrase_stays_one_phrase(self): + # A phrase splits on whitespace like anything else, so joining + # with AND would emit `"fisch AND rezept"` -- a phrase query for + # words in that literal order, which matched nothing on a real + # 3.0.4 archive. Spaces keep the phrase intact. + assert _to_search_query('"fisch rezept"') == '"fisch rezept"' + assert _to_search_query('"fisch rezept" pangasius') == '"fisch rezept" pangasius*' def test_empty_input_unchanged(self): # Defensive: a stray "" must NOT become "*" -- a wildcard alone # matches every doc, which is the opposite of a no-op. - assert _to_whoosh_query("") == "" - assert _to_whoosh_query(" ") == " " + assert _to_search_query("") == "" + assert _to_search_query(" ") == " " # ── Stub collaborators ──────────────────────────────────────────────────── diff --git a/tests/stacklets/test_recall.py b/tests/stacklets/test_recall.py index 09c01f7f..35d6c11e 100644 --- a/tests/stacklets/test_recall.py +++ b/tests/stacklets/test_recall.py @@ -80,7 +80,7 @@ async def test_question_mark_triggers_rewrite(self): # Memory walker reads Python regex; alternation uses `|`. assert memory == "Impfung|MMR|Auffrischung" # Paperless reads Whoosh; alternation uses ` OR `. The bare - # tokens get wildcarded by PaperlessAPI._to_whoosh_query. + # tokens get wildcarded by PaperlessAPI._to_search_query. assert paperless == "Impfung OR MMR OR Auffrischung" assert len(c.calls) == 1