diff --git a/lib/stack/links.py b/lib/stack/links.py new file mode 100644 index 00000000..133a60d1 --- /dev/null +++ b/lib/stack/links.py @@ -0,0 +1,140 @@ +"""Logical link construction — the emitter half of the `/go` namespace. + +A link a bot posts into a Matrix room is there forever: the timeline is +append-only, so whatever URL was in the message is the URL a family +member clicks two years later. Point it straight at a service and it +dies the day the domain changes, the stack flips between port mode and +domain mode, or Paperless moves. Point it at `home./go/docs/247` +and it re-resolves at click time. + +That indirection has two halves, and they live apart on purpose: + + * this module builds the *logical path* (`/docs/247`) and joins it onto + the configured `/go` base. It knows the shape of the namespace and + nothing else. + * `stacklets/core/tools-server/resolver.py` is the only place that + knows what a logical path resolves *to*. Nothing here may duplicate + that mapping. + +So an emitter never concatenates a service URL. It asks for a logical +path, hands it the base it was given, and posts whatever comes back. +When the base is unset (core has not rendered `LINK_BASE_URL` yet) the +answer is the empty string, and callers fall back to their unlinked +view — a link that cannot resolve is worse than no link. + +Stdlib only: the host CLI imports `lib/stack/` without third-party deps, +and the bot-runner mounts this same tree at `/app/stack`, so host and +containers build links from one implementation. +""" + +from __future__ import annotations + +from urllib.parse import quote + + +# ── Logical paths ───────────────────────────────────────────────────── +# +# Kinds are explicit nouns — the path says whether "camping" is a topic +# or a person, so the resolver never has to guess. They come in two +# families, and which one a new kind belongs to is the whole design: +# +# entities, addressed by NAME — `topic`, `person` +# A name a person could type, for something that has an identity +# beyond any one file. A trailing leaf (`todo`) selects a +# sub-page instead of the overview. +# +# records, addressed by ID — `docs`, `capture` +# One artefact, keyed by something assigned once and never +# re-derived. Never by path: a path encodes where a thing sat on +# the day the link was written, and these things move. +# +# A link posted into chat is permanent, so the cost of putting a kind in +# the wrong family is paid forever. When in doubt, ask what changes when +# a family renames a topic or corrects a title. + +def go_docs(doc_id: int | str) -> str: + """`/docs/` — a document, wherever it is filed right now. + + >>> go_docs(247) + '/docs/247' + """ + return f"/docs/{doc_id}" + + +def go_topic(scope: str, leaf: str | None = None) -> str: + """`/topic/` — a shared or personal topic page. + + `scope` is either a bare slug (`camping`, resolved under the shared + bucket) or an explicit vault path (`family/camping`, `homer/gravel`). + Both forms are passed through verbatim; the resolver takes either. + + >>> go_topic("family/camping", "todo") + '/topic/family/camping/todo' + >>> go_topic("camping") + '/topic/camping' + """ + return _entity_path("topic", scope, leaf) + + +def go_capture(capture_id: str) -> str: + """`/capture/` — one captured note, bookmark or memo. + + A record, so it is addressed by id and never by where it sits. Its + vault path carries the bucket, the topic slug and the title slug, + and each of those changes on its own under ordinary use: a capture + re-scopes when a second person joins the room, a topic gets + renamed, a title is rewritten by a correction. A path-keyed link + would break for three reasons the resolver cannot repair, and break + silently, which is worse than the service URLs it replaced. + + The id is the Matrix event id, assigned once and never rewritten. + It starts with `$` and can carry `/` in older room versions, so it + is percent-encoded into a single path segment. + + >>> go_capture("$abc123") + '/capture/%24abc123' + """ + return f"/capture/{quote(str(capture_id), safe='')}" + + +def go_person(slug: str, leaf: str | None = None) -> str: + """`/person/` — a household member's page. + + >>> go_person("homer") + '/person/homer' + >>> go_person("homer", "todo") + '/person/homer/todo' + """ + return _entity_path("person", slug, leaf) + + +def _entity_path(kind: str, scope: str, leaf: str | None) -> str: + """Join `kind`, the scope segments, and an optional leaf into a path. + + Scopes arrive from vault paths and room bindings, which carry stray + slashes often enough that stripping them here is cheaper than at + every call site. + """ + segments = [s for s in str(scope).strip("/").split("/") if s] + if leaf: + segments.append(leaf.strip("/")) + return "/".join(["", kind, *segments]) + + +# ── Public URL ──────────────────────────────────────────────────────── + +def public(logical: str, base: str) -> str: + """Absolute, clickable form of a logical path. + + `base` is `LINK_BASE_URL` — core's mode-correct home URL with the + `/go` prefix already on it. Empty base means the namespace is not + reachable yet, so there is no honest link to post. + + >>> public(go_docs(247), "https://home.example.org/go") + 'https://home.example.org/go/docs/247' + >>> public(go_docs(247), "") + '' + """ + if not base: + return "" + return f"{base.rstrip('/')}{logical}" diff --git a/stacklets/core/docker-compose.yml b/stacklets/core/docker-compose.yml index c73b7e25..b7a2d9ca 100644 --- a/stacklets/core/docker-compose.yml +++ b/stacklets/core/docker-compose.yml @@ -79,6 +79,12 @@ services: # effect without a rebuild. Keep in sync with tools-server/Dockerfile. - ./tools-server/server.py:/app/server.py:ro - ./tools-server/resolver.py:/app/resolver.py:ro + - ./tools-server/capture_index.py:/app/capture_index.py:ro + # The brain projection, read-only: `/go/capture/` finds where a + # capture is filed now by reading the same tree the wiki serves. + # `${DATA_DIR}` is the host path (BRAIN_REPO_DIR is the *container* + # path the bot-runner sees, so it cannot be a bind source here). + - ${DATA_DIR}/memory/brain:/brain:ro restart: unless-stopped networks: diff --git a/stacklets/core/tools-server/Dockerfile b/stacklets/core/tools-server/Dockerfile index a228174d..a8dd444c 100644 --- a/stacklets/core/tools-server/Dockerfile +++ b/stacklets/core/tools-server/Dockerfile @@ -7,6 +7,7 @@ RUN pip install --no-cache-dir -r requirements.txt COPY server.py . COPY resolver.py . +COPY capture_index.py . # --gecos "" suppresses the interactive Full Name/Room/Phone prompts RUN adduser --disabled-password --gecos "" --uid 1000 tools diff --git a/stacklets/core/tools-server/capture_index.py b/stacklets/core/tools-server/capture_index.py new file mode 100644 index 00000000..0dbd479c --- /dev/null +++ b/stacklets/core/tools-server/capture_index.py @@ -0,0 +1,86 @@ +"""Find where a capture lives right now, by the id it was captured with. + +`/go/capture/` promises a link that survives the file moving. Every +other logical path can be resolved by rewriting the string, but this one +cannot: the id names *which* capture, and says nothing about where it is +today. Something has to look. + +WHY A SCAN AND NOT AN INDEX + +The obvious design is an index the curator maintains during the mirror. +It would be faster and it would be one more thing that can be wrong: an +index is a second copy of the truth, so it can go stale, disagree with +the tree, or be missing on a fresh clone, and every one of those failures +looks like a dead link. + +A scan reads the same files the wiki serves, so it cannot disagree with +them, and there is nothing to rebuild or invalidate. A family vault is +hundreds to low thousands of small markdown files and a redirect happens +when a human clicks something, so the cost lands in the right place. If +a vault ever grows past that, an index can be added *behind this same +function* without touching the resolver or the link format. + +WHY FRONTMATTER AND NOT THE BODY + +`capture_id` is a frontmatter field, and the search path already treats +frontmatter as the machine-readable half of a page. Reading only the head +of each file keeps a scan cheap and avoids matching an id someone quoted +in prose. +""" + +from __future__ import annotations + +from pathlib import Path + +# Frontmatter sits at the top; no capture page has a hundred lines of it. +# Bounding the read keeps a scan over a large vault from paging in whole +# documents to answer a question the header already answers. +_HEAD_BYTES = 4096 + +_FIELD = "capture_id:" + + +def _capture_id_of(path: Path) -> str: + """The `capture_id` declared in a file's frontmatter, or "". + + Only the frontmatter block counts. An id quoted in the body of a + note — a reply discussing another capture, say — must not make this + file answer to it. + """ + try: + with path.open("r", encoding="utf-8", errors="ignore") as fh: + head = fh.read(_HEAD_BYTES) + except OSError: + return "" + lines = head.splitlines() + if not lines or lines[0].strip() != "---": + return "" + for line in lines[1:]: + if line.strip() == "---": + return "" + if line.startswith(_FIELD): + return line[len(_FIELD):].strip().strip("\"'") + return "" + + +def find_capture(capture_id: str, *, brain_dir: Path) -> str | None: + """Wiki-relative target for a capture id, or None if nothing carries it. + + The target is the path the wiki serves, so the `.md` suffix comes off + — same shape `resolve_topic_target` returns for a topic page. + + Returns None rather than a guess when the id is unknown: a 404 tells + the person the capture is gone, while a redirect to something else + would quietly show them the wrong note. + """ + wanted = (capture_id or "").strip() + if not wanted: + return None + brain_dir = Path(brain_dir) + if not brain_dir.is_dir(): + return None + for path in sorted(brain_dir.rglob("*.md")): + if _capture_id_of(path) == wanted: + rel = path.relative_to(brain_dir) + return str(rel.with_suffix("")) + return None diff --git a/stacklets/core/tools-server/resolver.py b/stacklets/core/tools-server/resolver.py index 40896170..49207caf 100644 --- a/stacklets/core/tools-server/resolver.py +++ b/stacklets/core/tools-server/resolver.py @@ -21,6 +21,8 @@ from __future__ import annotations +from typing import Callable + # A trailing `todo`/`todos` segment points at the entity's task list instead of # its overview page. Both spellings accepted — people type either. _TODO_LEAVES = {"todo", "todos"} @@ -65,20 +67,31 @@ def resolve_person_target(segments: list[str]) -> str | None: def build_redirect( kind: str, rest: list[str], *, docs_base: str, wiki_base: str, shared_bucket: str, + find_capture: "Callable[[str], str | None] | None" = None, ) -> str | None: """Full redirect URL for a `///` request, or None. - `kind` is `docs`, `topic`, or `person`; `rest` is the remaining path - segments. `docs_base`/`wiki_base` are the public base URLs of Paperless and - the wiki — already mode-correct, computed once by the HTTP layer from env, - so this stays a pure string join. None (→ 404) for an unknown kind, a + `kind` is `docs`, `topic`, `person`, or `capture`; `rest` is the remaining + path segments. `docs_base`/`wiki_base` are the public base URLs of Paperless + and the wiki — already mode-correct, computed once by the HTTP layer from + env, so this stays a pure string join. None (→ 404) for an unknown kind, a non-numeric doc id, or an entity shape the resolver rejects. + + `capture` is the one kind whose target cannot be computed from the path: + the id says *which* capture, never where it is now. `find_capture` is + injected by the HTTP layer for exactly that lookup, which keeps the I/O out + of here and this module unit-testable without a vault on disk. Left unset, + `/capture/...` 404s rather than guessing. """ if kind == "docs": if len(rest) != 1 or not rest[0].isdigit(): return None return f"{docs_base.rstrip('/')}/documents/{rest[0]}/details" - if kind == "topic": + if kind == "capture": + if len(rest) != 1 or find_capture is None: + return None + target = find_capture(rest[0]) + elif kind == "topic": target = resolve_topic_target(rest, shared_bucket=shared_bucket) elif kind == "person": target = resolve_person_target(rest) diff --git a/stacklets/core/tools-server/server.py b/stacklets/core/tools-server/server.py index 322e3258..54825a92 100644 --- a/stacklets/core/tools-server/server.py +++ b/stacklets/core/tools-server/server.py @@ -12,11 +12,13 @@ import json import os import socket +from pathlib import Path import httpx from fastapi import FastAPI from fastapi.responses import JSONResponse, RedirectResponse +from capture_index import find_capture from resolver import build_redirect @@ -48,6 +50,13 @@ LINK_SHARED_BUCKET = os.environ.get("SHARED_BUCKET", "family") # The one knob: the path namespace persistent links live under (home.tld/go/…). LINK_PREFIX = os.environ.get("LINK_PREFIX", "go").strip("/") +# The brain projection, read-only — the same tree the wiki serves. Only +# `/go/capture/` needs it, to find where a capture sits now. This is the +# mount point, not `BRAIN_REPO_DIR`: that variable is the path the *bot-runner* +# sees, and pointing this at it would name a directory that does not exist in +# this container. Unmounted (memory not installed) simply means capture +# links 404. +BRAIN_DIR = Path("/brain") app = FastAPI( @@ -137,6 +146,7 @@ def _go(kind: str, rest: list[str]): url = build_redirect( kind, rest, docs_base=DOCS_PUBLIC_URL, wiki_base=WIKI_PUBLIC_URL, shared_bucket=LINK_SHARED_BUCKET, + find_capture=lambda cid: find_capture(cid, brain_dir=BRAIN_DIR), ) if not url: return _error("no such resource", status=404) @@ -161,6 +171,19 @@ async def go_person(name: str): return _go("person", [s for s in name.split("/") if s]) +@app.get(f"/{LINK_PREFIX}/capture/{{capture_id:path}}", + summary="Resolve a capture link") +async def go_capture(capture_id: str): + """Redirect a captured note or bookmark to wherever it is filed now. + + Keyed by the capture's id rather than its path, so re-scoping it, + renaming its topic, or correcting its title all leave the link + working. 404 when no file carries that id — better than showing + someone a different note. + """ + return _go("capture", [capture_id]) + + # ── Logs ─────────────────────────────────────────────────────────────────── @app.get("/logs", summary="Get container logs") diff --git a/stacklets/docs/bot/archivist.py b/stacklets/docs/bot/archivist.py index 75eec6a0..60bfb085 100644 --- a/stacklets/docs/bot/archivist.py +++ b/stacklets/docs/bot/archivist.py @@ -63,6 +63,7 @@ ) from stack import resolve_model from stack.email_message import defang_links +from stack.links import go_docs, go_topic, public from stack.ai.client import ( LLMError, LLMUnavailableError, @@ -255,14 +256,20 @@ def __init__(self, homeserver, user_id, password, session_dir, **settings): # chat link is worse than no link at all, since the helper # formatters already render a path-only / bold-title view when # the public URL is empty. + # Links to a *document* are the exception and go one better: + # they are built from `link_base_url` below, so what lands in + # chat is a logical `/go/docs/` that re-resolves at click + # time instead of a Paperless URL that ages badly. self.paperless_url = os.environ.get("PAPERLESS_URL", "") self.paperless_token = os.environ.get("PAPERLESS_TOKEN", "") self.paperless_public_url = os.environ.get("PAPERLESS_PUBLIC_URL", "") self.code_url = os.environ.get("CODE_URL", "") self.code_public_url = os.environ.get("CODE_PUBLIC_URL", "") - # Base for persistent `/go` links the bot posts into chat (e.g. - # `{link_base_url}/topic//todo`). Public/home base, mode-correct. - # Empty when core hasn't rendered it -> the todo link is simply omitted. + # Base for every persistent `/go` link the bot posts into chat or + # into an event envelope -- filing replies, search hits, todo + # pages. Public/home base, mode-correct; `stack.links` joins the + # logical path onto it. Empty when core hasn't rendered it yet -> + # the link is simply omitted and the unlinked view is used. self.link_base_url = os.environ.get("LINK_BASE_URL", "") self.openai_url = os.environ.get("OPENAI_URL", "") self.openai_key = os.environ.get("OPENAI_KEY", "") @@ -424,6 +431,7 @@ async def start(self) -> None: vision_max_pdf_pages=self.vision_max_pdf_pages, reformat_max_pdf_pages=self.reformat_max_pdf_pages, paperless_public_url=self.paperless_public_url, + link_base_url=self.link_base_url, actor=self.user_id, vault=self._vault, ) @@ -434,7 +442,7 @@ async def start(self) -> None: language=self.language, code_public_url=self.code_public_url, mirror_org=self.mirror_org, - paperless_public_url=self.paperless_public_url, + link_base_url=self.link_base_url, shared_bucket=self.shared_bucket, vault=self._vault, ) @@ -544,8 +552,7 @@ def _duplicate_reply(self, name: str, e: PaperlessDuplicateError) -> str: Points the user at the original doc's Paperless page so they can verify the match instead of wondering why the upload 'failed'. """ - link = (f"{self.paperless_public_url}/documents/{e.doc_id}/details" - if e.doc_id and self.paperless_public_url else "") + link = public(go_docs(e.doc_id), self.link_base_url) if e.doc_id else "" return self.t("already_filed", name=name, doc_id=e.doc_id if e.doc_id is not None else "?", @@ -2358,11 +2365,11 @@ def _todo_link(self, o: CaptureOutcome) -> str: the explicit `family/camping` path form as readily as a bare slug. """ scope = (o.scope or "").strip("/") - if not self.link_base_url or "/" not in scope: + if "/" not in scope: return "" if not (o.classification.get("action_items") or []): return "" - return f"{self.link_base_url}/topic/{scope}/todo" + return public(go_topic(scope, "todo"), self.link_base_url) # ── URL archiving (documents room — feeds Paperless) ───────────────── @@ -2466,7 +2473,7 @@ async def _handle_show(self, room_id: str, doc_id: int, reply_to: str | None = N title = doc.get("title", "Untitled") content = doc.get("content", "").strip() - link = f"{self.paperless_public_url}/documents/{doc_id}/details" if self.paperless_public_url else "" + link = public(go_docs(doc_id), self.link_base_url) if not content: await self._send(room_id, f"**{title}** — no text content available.\n\n {link}", reply_to) diff --git a/stacklets/docs/bot/document_pipeline.py b/stacklets/docs/bot/document_pipeline.py index e943f019..bca2e757 100644 --- a/stacklets/docs/bot/document_pipeline.py +++ b/stacklets/docs/bot/document_pipeline.py @@ -38,6 +38,7 @@ reformat_document, ) from stack import resolve_model +from stack.links import go_docs, public # Text-like extensions skip reformat (the content is already clean) but # still classify + mirror. Paperless only parses text/plain and text/csv, @@ -140,6 +141,7 @@ def __init__( vision_max_pdf_pages: int, reformat_max_pdf_pages: int, paperless_public_url: str, + link_base_url: str, actor: str, vault, ): @@ -154,6 +156,7 @@ def __init__( self.vision_max_pdf_pages = vision_max_pdf_pages self.reformat_max_pdf_pages = reformat_max_pdf_pages self.paperless_public_url = paperless_public_url + self.link_base_url = link_base_url self.actor = actor self._vault = vault @@ -211,10 +214,7 @@ async def process( if not doc_id: return FilingOutcome(status="ocr_failed", display_name=display_name) - link = ( - f"{self.paperless_public_url}/documents/{doc_id}/details" - if self.paperless_public_url else "" - ) + link = public(go_docs(doc_id), self.link_base_url) doc = await self._paperless.get_doc(doc_id) if not doc: # Filed but unreadable — still mirror a minimal entry so @@ -286,7 +286,7 @@ async def process( resolved_persons=result.resolved_persons, resolved_correspondent=result.resolved_correspondent, resolved_type=result.resolved_type, - paperless_url=self.paperless_public_url, + link_base_url=self.link_base_url, actor=self.actor, ts=utc_now_isoformat(), ) @@ -451,7 +451,7 @@ async def reprocess( resolved_persons=result.resolved_persons, resolved_correspondent=result.resolved_correspondent, resolved_type=result.resolved_type, - paperless_url=self.paperless_public_url, + link_base_url=self.link_base_url, actor=self.actor, ts=utc_now_isoformat(), ) diff --git a/stacklets/docs/bot/matching.py b/stacklets/docs/bot/matching.py index 359c2368..47ab6d11 100644 --- a/stacklets/docs/bot/matching.py +++ b/stacklets/docs/bot/matching.py @@ -21,6 +21,8 @@ import re from typing import TYPE_CHECKING, Any, NamedTuple +from stack.links import go_docs, public + if TYPE_CHECKING: from stack.ontology import Ontology @@ -316,7 +318,7 @@ def build_document_event( resolved_persons: list[str] | None = None, resolved_correspondent: str | None = None, resolved_type: str | None = None, - paperless_url: str = "", + link_base_url: str = "", actor: str | None = None, ts: str | None = None, ) -> dict: @@ -362,8 +364,10 @@ def build_document_event( "facts": classification.get("facts", []), "action_items": classification.get("action_items", []), } - if paperless_url: - data["url"] = f"{paperless_url}/documents/{doc_id}/details" + # The envelope rides on a Matrix message, so its link outlives every + # hosting change the stack goes through: logical, not a Paperless URL. + if url := public(go_docs(doc_id), link_base_url): + data["url"] = url summary = f"{title} filed (#{doc_id})" if title else f"Document #{doc_id} filed" diff --git a/stacklets/docs/bot/nl_query.py b/stacklets/docs/bot/nl_query.py index e19f7155..eca5a16b 100644 --- a/stacklets/docs/bot/nl_query.py +++ b/stacklets/docs/bot/nl_query.py @@ -43,7 +43,7 @@ import re from typing import Any -from search_format import memory_doc_url, paperless_doc_url +from search_format import memory_hit_url, paperless_doc_url from pipeline import extract_bot_summary # Memory lib is a sibling stacklet. The archivist already wires the @@ -68,7 +68,7 @@ def build_evidence( *, code_public_url: str = "", mirror_org: str = "family", - paperless_public_url: str = "", + link_base_url: str = "", limit: int = EVIDENCE_LIMIT, ) -> list[dict]: """Merge memory + Paperless hits into the LLM's evidence list. @@ -106,8 +106,9 @@ def build_evidence( # Prefer the structured summary; fall back to the excerpt # for vault files that predate the classifier. "summary": (r.get("summary") or r.get("excerpt") or "").strip(), - "url": memory_doc_url( - r.get("rel") or "", + "url": memory_hit_url( + r, + link_base_url=link_base_url, code_public_url=code_public_url, mirror_org=mirror_org, ), @@ -127,7 +128,7 @@ def build_evidence( "persons": [], "summary": extract_bot_summary(doc), "url": paperless_doc_url( - doc_id, public_url=paperless_public_url, + doc_id, link_base_url=link_base_url, ), "doc_id": doc_id, }) diff --git a/stacklets/docs/bot/search_format.py b/stacklets/docs/bot/search_format.py index 4376ba78..59dc3037 100644 --- a/stacklets/docs/bot/search_format.py +++ b/stacklets/docs/bot/search_format.py @@ -25,6 +25,8 @@ from typing import Optional +from stack.links import go_capture, go_docs, public + def memory_doc_url( rel: str, *, @@ -47,18 +49,52 @@ def memory_doc_url( def paperless_doc_url( doc_id: Optional[int], *, - public_url: str = "", + link_base_url: str = "", ) -> str: - """Build a Paperless detail-page URL for a doc id.""" - if not (public_url and doc_id): + """Build a persistent `/go/docs/` link for a doc id. + + Search results live in Matrix history as long as the room does, so + the link is logical rather than a Paperless URL: it resolves to the + document wherever it lives at click time. Returns "" without a link + base, and the formatter falls back to a bold title. + """ + if not (link_base_url and doc_id): return "" - return f"{public_url.rstrip('/')}/documents/{doc_id}/details" + return public(go_docs(doc_id), link_base_url) + + +def memory_hit_url( + r: dict, *, + link_base_url: str = "", + code_public_url: str = "", + mirror_org: str = "family", +) -> str: + """The best durable link for one memory hit, or "" for none. + + A capture is a record, so when the hit carries the id it was + captured with, the link is `/go/capture/` and survives the file + being re-scoped, its topic renamed, or its title corrected. + + Everything else falls back to the Forgejo blob URL. That link + freezes today's address and today's path, which is exactly what the + `/go` namespace exists to avoid — but a hand-written wiki page has + no id to key on, and a worse link still beats none while that is + true. Anything that grows a stable id should move to a record kind + rather than widening this fallback. + """ + if capture_id := (r.get("capture_id") or "").strip(): + if url := public(go_capture(capture_id), link_base_url): + return url + return memory_doc_url( + r.get("rel", ""), code_public_url=code_public_url, mirror_org=mirror_org, + ) def format_memory_hit( r: dict, n: int, *, code_public_url: str = "", mirror_org: str = "family", + link_base_url: str = "", ) -> str: """Render one memory hit as a Matrix-markdown block. @@ -79,7 +115,10 @@ def format_memory_hit( # paragraph separation, and a loose `
    ` ends up rendering each # marker on its own line in Element; plain "1. Foo" text avoids # the list machinery entirely. - url = memory_doc_url(rel, code_public_url=code_public_url, mirror_org=mirror_org) + url = memory_hit_url( + r, link_base_url=link_base_url, + code_public_url=code_public_url, mirror_org=mirror_org, + ) if url: head = f"{n}\\. [{title}]({url})" else: @@ -100,7 +139,7 @@ def format_memory_hit( def format_paperless_hit( doc: dict, n: int, *, - public_url: str = "", + link_base_url: str = "", ) -> str: """Render one Paperless hit as a single Matrix-markdown line. @@ -114,7 +153,7 @@ def format_paperless_hit( created = (doc.get("created") or "")[:10] meta = " · ".join(p for p in [created, f"#{doc_id}" if doc_id else ""] if p) - url = paperless_doc_url(doc_id, public_url=public_url) + url = paperless_doc_url(doc_id, link_base_url=link_base_url) if url: head = f"{n}\\. [{title}]({url})" else: diff --git a/stacklets/docs/bot/search_service.py b/stacklets/docs/bot/search_service.py index e3d1030c..a6793fa8 100644 --- a/stacklets/docs/bot/search_service.py +++ b/stacklets/docs/bot/search_service.py @@ -54,7 +54,7 @@ def __init__( language: str, code_public_url: str, mirror_org: str, - paperless_public_url: str, + link_base_url: str, shared_bucket: str, vault, ): @@ -64,7 +64,7 @@ def __init__( self.language = language self.code_public_url = code_public_url self.mirror_org = mirror_org - self.paperless_public_url = paperless_public_url + self.link_base_url = link_base_url self.shared_bucket = shared_bucket self._vault = vault @@ -146,13 +146,14 @@ async def run( blocks.append(self._t("search_memory_results", query=query)) for n, r in enumerate(memory_results, start=1): blocks.append(_format_memory_hit( - r, n, code_public_url=self.code_public_url, mirror_org=self.mirror_org, + r, n, code_public_url=self.code_public_url, + mirror_org=self.mirror_org, link_base_url=self.link_base_url, )) if paperless_results: blocks.append(self._t("search_paperless_results", query=query)) for n, doc in enumerate(paperless_results, start=1): blocks.append(_format_paperless_hit( - doc, n, public_url=self.paperless_public_url, + doc, n, link_base_url=self.link_base_url, )) return "\n\n".join(blocks) @@ -190,7 +191,7 @@ async def _synthesize(self, query, keywords, memory_results, paperless_results): memory_results, paperless_results, code_public_url=self.code_public_url, mirror_org=self.mirror_org, - paperless_public_url=self.paperless_public_url, + link_base_url=self.link_base_url, ) answer = await self._classifier.synthesize_answer(query, evidence, lang=self.language) logger.info( diff --git a/stacklets/memory/lib.py b/stacklets/memory/lib.py index 2026ba8d..dada05da 100644 --- a/stacklets/memory/lib.py +++ b/stacklets/memory/lib.py @@ -1575,6 +1575,14 @@ def search_memory( # when the file isn't a Paperless mirror (e.g. capture # notes, hand-written wiki entries). "paperless_id": fm.get("paperless_id") or "", + # The Matrix event this file was captured from. Assigned + # once and never rewritten, which is what makes it the only + # safe key for a link that outlives the file: `rel` carries + # the bucket, the topic slug and the title slug, and all + # three change under ordinary use (a re-scope, a topic + # rename, a corrected title). Empty for anything not + # captured from chat. + "capture_id": fm.get("capture_id") or "", }) results.sort( diff --git a/tests/framework/test_links.py b/tests/framework/test_links.py new file mode 100644 index 00000000..cc054699 --- /dev/null +++ b/tests/framework/test_links.py @@ -0,0 +1,137 @@ +"""The one link builder — logical paths, and the rule that keeps it one. + +`stack.links` builds the `/go` paths bots post into Matrix. The point of +the module is not the string formatting, which is trivial; it is that no +emitter anywhere builds a service URL by hand, because a link frozen in +chat history has to survive a domain change, a hosting-mode flip, and a +moved backend. + +So this file has two halves: what the builder promises, and a guard that +the promise stays singular. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +from stack.links import go_docs, go_person, go_topic, public + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent + + +# ── What the builder promises ──────────────────────────────────────────── + +class TestLogicalPaths: + """Paths name the entity kind, so the resolver never has to guess.""" + + def test_document(self): + assert go_docs(247) == "/docs/247" + + def test_shared_topic_by_slug(self): + # A bare slug resolves under the shared bucket; the emitter does + # not need to know what that bucket is called. + assert go_topic("camping") == "/topic/camping" + + def test_topic_with_explicit_bucket(self): + assert go_topic("family/camping") == "/topic/family/camping" + + def test_topic_todo_leaf(self): + assert go_topic("family/camping", "todo") == "/topic/family/camping/todo" + + def test_person(self): + assert go_person("homer") == "/person/homer" + + def test_person_todo_leaf(self): + assert go_person("homer", "todo") == "/person/homer/todo" + + def test_scope_slashes_are_tolerated(self): + # Scopes arrive from vault paths and room bindings, which carry + # stray slashes; the caller should not have to trim them. + assert go_topic("/family/camping/") == "/topic/family/camping" + + +class TestPublicUrl: + """Joining a logical path onto the configured `/go` base.""" + + def test_joins_base_and_path(self): + assert public(go_docs(247), "https://home.example.org/go") == \ + "https://home.example.org/go/docs/247" + + def test_trailing_slash_on_base_does_not_double(self): + assert public(go_docs(247), "https://home.example.org/go/") == \ + "https://home.example.org/go/docs/247" + + def test_port_mode_base(self): + # Port mode hands out ip:port with no vanity host; same join. + assert public(go_topic("family/camping", "todo"), "http://10.0.0.5:42000/go") == \ + "http://10.0.0.5:42000/go/topic/family/camping/todo" + + def test_no_base_means_no_link(self): + # Core has not rendered LINK_BASE_URL yet. An unresolvable link + # is worse than none, so emitters get "" and fall back to their + # unlinked rendering. + assert public(go_docs(247), "") == "" + + +# ── The rule: exactly one builder ──────────────────────────────────────── +# +# Generic guard for a "there MUST be exactly one implementation" rule. +# The URL builder is not the only place we have written that sentence in +# a spec and then grown a second implementation anyway (vault-format.md +# §8 says one frontmatter writer; there are three). Nothing in the suite +# catches that class of drift, because each duplicate is individually +# correct — only the count is wrong. Walking the tree for the shape is +# inelegant and would have caught both. Copy this test, change the +# pattern and the allowlist. + +# Paperless's document detail page. Every hand-built copy of this string +# was a link that died on the next domain change. +DOC_URL_SHAPE = re.compile(r"/documents/.*?/details") + +# Only the two halves of the link seam may name that shape: `links.py` +# builds the logical path an emitter posts, `resolver.py` maps it to +# wherever the document lives right now. +ALLOWED = { + "lib/stack/links.py", + "stacklets/core/tools-server/resolver.py", + # S6 of the memory-architecture pass replaces this with the logical + # `/go/docs/` path. It writes `resource` into vault frontmatter, + # so it needs a vault-format.md §5 spec change plus a migration of + # entries already on disk. Delete this line when S6 lands. + "stacklets/docs/bot/vault_entry.py", +} + +SCANNED_ROOTS = ("lib", "stacklets", "tools") + + +def _offenders() -> list[str]: + hits = [] + for root in SCANNED_ROOTS: + for path in sorted((REPO_ROOT / root).rglob("*.py")): + rel = path.relative_to(REPO_ROOT).as_posix() + if rel in ALLOWED: + continue + for n, line in enumerate(path.read_text().splitlines(), start=1): + if DOC_URL_SHAPE.search(line): + hits.append(f"{rel}:{n}: {line.strip()}") + return hits + + +class TestExactlyOneDocumentUrlBuilder: + + def test_no_module_builds_a_document_url_by_hand(self): + offenders = _offenders() + assert not offenders, ( + "A document URL is built outside the link seam:\n " + + "\n ".join(offenders) + + "\n\nEmit `stack.links.public(go_docs(id), link_base_url)` instead. " + "Only lib/stack/links.py and the tools-server resolver may know " + "what a document URL looks like." + ) + + def test_the_guard_actually_looks_at_files(self): + # A tree walk that silently matches nothing passes forever. Pin + # that the pattern still fires on the one file we allow. + resolver = REPO_ROOT / "stacklets/core/tools-server/resolver.py" + assert DOC_URL_SHAPE.search(resolver.read_text()) diff --git a/tests/stacklets/test_archivist_matching.py b/tests/stacklets/test_archivist_matching.py index 846a3567..534d5f27 100644 --- a/tests/stacklets/test_archivist_matching.py +++ b/tests/stacklets/test_archivist_matching.py @@ -561,12 +561,14 @@ def test_resolved_fields_land_in_data(self): assert evt["data"]["correspondent"] == "Duff Insurance" assert evt["data"]["document_type"] == "Invoice" - def test_includes_paperless_url(self): + def test_includes_persistent_document_link(self): + # The envelope lives in Matrix history, so its link is the + # logical one -- it still resolves after a domain change. evt = build_document_event( 42, {}, - paperless_url="http://localhost:42020", + link_base_url="http://localhost:42000/go", ) - assert evt["data"]["url"] == "http://localhost:42020/documents/42/details" + assert evt["data"]["url"] == "http://localhost:42000/go/docs/42" def test_no_url_when_empty(self): evt = build_document_event(42, {}) diff --git a/tests/stacklets/test_capture_links.py b/tests/stacklets/test_capture_links.py new file mode 100644 index 00000000..809d4264 --- /dev/null +++ b/tests/stacklets/test_capture_links.py @@ -0,0 +1,179 @@ +"""Links to a captured note survive the note moving. + +A link posted into a Matrix room is permanent, so the question that +decides its design is not "where is this today" but "what about it will +still be true in two years". For a capture the answer is its id and +almost nothing else: the vault path carries the bucket, the topic slug +and the title slug, and every one of those changes under ordinary use -- +a capture re-scopes when a second person joins the room, a topic gets +renamed, a title is rewritten by a correction. + +These tests pin that promise end to end: the emitter keys on the id, the +resolver finds the file wherever it now sits, and moving the file does +not change the answer. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(_REPO_ROOT / "lib")) +sys.path.insert(0, str(_REPO_ROOT / "stacklets" / "core" / "tools-server")) +sys.path.insert(0, str(_REPO_ROOT / "stacklets" / "docs" / "bot")) + +from capture_index import find_capture # noqa: E402 +from resolver import build_redirect # noqa: E402 +from search_format import memory_hit_url # noqa: E402 +from stack.links import go_capture, public # noqa: E402 + +CAPTURE_ID = "$Efjml6ySCYyWM6xsNwRCSSEund-u2CIywGpgC8u6tvY" + +PAGE = f"""--- +type: note +title: Campsite booked at Lake Springfield +capture_id: {CAPTURE_ID} +--- + +# Campsite booked at Lake Springfield +""" + + +def _write(brain: Path, rel: str, text: str = PAGE) -> Path: + path = brain / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +class TestFindingACaptureWhereverItSits: + + def test_a_capture_is_found_by_its_id(self, tmp_path): + _write(tmp_path, "family/camping/notes/2026/08/campsite-3a338e.md") + + assert find_capture(CAPTURE_ID, brain_dir=tmp_path) == ( + "family/camping/notes/2026/08/campsite-3a338e" + ) + + def test_the_answer_follows_the_file_when_it_moves(self, tmp_path): + """The whole point, in one test. + + Same capture, re-scoped from a personal bucket to the shared one + and re-slugged by a corrected title. A path-keyed link would now + be dead; the id-keyed one still lands. + """ + before = _write(tmp_path, "homer/trip/notes/2026/08/old-title-3a338e.md") + assert find_capture(CAPTURE_ID, brain_dir=tmp_path).startswith("homer/") + + before.unlink() + _write(tmp_path, "family/camping/notes/2026/08/new-title-9f0011.md") + + assert find_capture(CAPTURE_ID, brain_dir=tmp_path) == ( + "family/camping/notes/2026/08/new-title-9f0011" + ) + + def test_an_unknown_id_is_not_found(self, tmp_path): + """404 beats a redirect to whatever happened to be nearby.""" + _write(tmp_path, "family/camping/notes/2026/08/campsite-3a338e.md") + + assert find_capture("$nope", brain_dir=tmp_path) is None + + def test_an_id_quoted_in_the_body_does_not_count(self, tmp_path): + """Only frontmatter declares what a page *is*. + + A note discussing another capture would otherwise answer to that + capture's link. + """ + _write( + tmp_path, "family/camping/notes/2026/08/chatter.md", + f"---\ntype: note\n---\n\nSee capture_id: {CAPTURE_ID} for details.\n", + ) + + assert find_capture(CAPTURE_ID, brain_dir=tmp_path) is None + + def test_a_missing_brain_is_not_an_error(self, tmp_path): + """The projection may not exist yet on a fresh install.""" + assert find_capture(CAPTURE_ID, brain_dir=tmp_path / "nothing") is None + + +class TestResolvingACaptureLink: + + def test_it_redirects_to_the_wiki_page(self, tmp_path): + _write(tmp_path, "family/camping/notes/2026/08/campsite-3a338e.md") + + url = build_redirect( + "capture", [CAPTURE_ID], + docs_base="http://docs.example", wiki_base="http://wiki.example", + shared_bucket="family", + find_capture=lambda cid: find_capture(cid, brain_dir=tmp_path), + ) + + assert url == ( + "http://wiki.example/family/camping/notes/2026/08/campsite-3a338e" + ) + + def test_without_a_lookup_it_declines_rather_than_guessing(self): + """Core running without the projection mounted must 404, not + invent a path from the id.""" + assert build_redirect( + "capture", [CAPTURE_ID], + docs_base="http://docs.example", wiki_base="http://wiki.example", + shared_bucket="family", + ) is None + + def test_the_existing_kinds_are_untouched(self): + """Guards against the new branch swallowing the old ones.""" + common = dict( + docs_base="http://docs.example", wiki_base="http://wiki.example", + shared_bucket="family", + ) + assert build_redirect("docs", ["247"], **common) == ( + "http://docs.example/documents/247/details" + ) + assert build_redirect("topic", ["camping"], **common) == ( + "http://wiki.example/family/camping/about" + ) + assert build_redirect("person", ["homer", "todo"], **common) == ( + "http://wiki.example/homer/todos" + ) + + +class TestWhichLinkASearchHitGets: + + BASE = "https://home.example.org/go" + + def test_a_capture_is_linked_by_id(self): + url = memory_hit_url( + {"rel": "family/camping/notes/2026/08/campsite-3a338e.md", + "capture_id": CAPTURE_ID}, + link_base_url=self.BASE, code_public_url="http://code.example", + ) + + assert url == public(go_capture(CAPTURE_ID), self.BASE) + assert "camping" not in url, "the path must not be baked into the link" + + def test_a_page_without_an_id_keeps_the_old_link(self): + """No regression for hand-written wiki pages. + + They have no id to key on, so they keep the Forgejo blob URL. + It rots, but a worse link beats none until they grow an id. + """ + url = memory_hit_url( + {"rel": "family/correspondents/README.md"}, + link_base_url=self.BASE, code_public_url="http://code.example", + ) + + assert url == ( + "http://code.example/family/memory/src/branch/main/" + "family/correspondents/README.md" + ) + + def test_no_configured_base_falls_back_rather_than_dropping_the_link(self): + """Before core renders LINK_BASE_URL there is no /go to point at.""" + url = memory_hit_url( + {"rel": "family/camping/notes/x.md", "capture_id": CAPTURE_ID}, + link_base_url="", code_public_url="http://code.example", + ) + + assert url.startswith("http://code.example/") diff --git a/tests/stacklets/test_core_link_resolver.py b/tests/stacklets/test_core_link_resolver.py index 7ea7b19b..366ba2aa 100644 --- a/tests/stacklets/test_core_link_resolver.py +++ b/tests/stacklets/test_core_link_resolver.py @@ -128,3 +128,63 @@ def test_port_mode_base_no_trailing_slash(self): "topic", ["camping", "todo"], docs_base="http://10.0.0.5:42000", wiki_base="http://10.0.0.5:42070", shared_bucket=_SHARED, ) == "http://10.0.0.5:42070/family/camping/todos" + + +# ── Round trip: what emitters build lands where it used to ────────────── +# +# The bots used to paste `{PAPERLESS_PUBLIC_URL}/documents//details` +# straight into chat; they now post a logical `/go` link built by +# `stack.links`. Nothing about the destination was supposed to move, so +# these pin the old URLs as literals and walk a freshly built logical +# path through the same two hops a click takes: the HTTP route splits it, +# the resolver maps it. A drift here is a family clicking a link in an +# old message and landing somewhere else. + +from stack.links import go_docs, go_person, go_topic, public # noqa: E402 + +_DOCS_BASE = "http://localhost:42020" +_WIKI_BASE = "http://localhost:42070" + +# Verbatim from the pre-change emitters (archivist.py, document_pipeline.py, +# search_format.py, matching.py) — the contract this refactor must not break. +_LEGACY_DOC_URL = f"{_DOCS_BASE}/documents/247/details" + + +def _click(logical: str) -> str | None: + """Resolve a logical path the way a browser hitting `/go/…` does. + + Mirrors the split in `server.py`'s three routes: the first segment + is the entity kind, the rest is the entity path. + """ + kind, *rest = [s for s in logical.split("/") if s] + return build_redirect( + kind, rest, docs_base=_DOCS_BASE, wiki_base=_WIKI_BASE, + shared_bucket=_SHARED, + ) + + +class TestEmittedLinksResolveWhereTheyUsedTo: + + def test_document_link_lands_on_the_paperless_detail_page(self): + assert _click(go_docs(247)) == _LEGACY_DOC_URL + + def test_document_link_survives_a_string_doc_id(self): + # Paperless ids arrive as ints from the API and as strings from + # chat commands; both must build the same link. + assert _click(go_docs("247")) == _LEGACY_DOC_URL + + def test_topic_todo_link_is_unchanged(self): + # The archivist already emitted this one by hand as + # f"{base}/topic/{scope}/todo" — same path, same destination. + assert go_topic("family/camping", "todo") == "/topic/family/camping/todo" + assert _click(go_topic("family/camping", "todo")) == \ + f"{_WIKI_BASE}/family/camping/todos" + + def test_person_link_resolves_to_the_member_page(self): + assert _click(go_person("homer")) == f"{_WIKI_BASE}/homer/about" + + def test_public_form_is_what_goes_into_the_message(self): + # The base carries the `/go` prefix (core's LINK_BASE_URL), so + # the emitted link is base + logical path, no separator surprises. + assert public(go_docs(247), "https://home.example.org/go") == \ + "https://home.example.org/go/docs/247" diff --git a/tests/stacklets/test_document_pipeline.py b/tests/stacklets/test_document_pipeline.py index d9e85859..a1573c95 100644 --- a/tests/stacklets/test_document_pipeline.py +++ b/tests/stacklets/test_document_pipeline.py @@ -80,6 +80,7 @@ def _pipeline(paperless, *, mirror=None, classify_enabled=True, reformat_enabled vision_max_pdf_pages=5, reformat_max_pdf_pages=5, paperless_public_url="http://paperless", + link_base_url="http://home.test/go", actor="@archivist-bot:test.local", vault=_FakeVault(), ) @@ -118,7 +119,7 @@ async def test_filed_no_details_when_doc_unreadable(self): out = await _process(_pipeline(FakePaperless(doc_id=7, doc=None), mirror=mirror)) assert out.status == "filed_no_details" assert out.doc_id == 7 - assert out.link == "http://paperless/documents/7/details" + assert out.link == "http://home.test/go/docs/7" # Mirror still reached so Paperless ⇄ mirror stay 1:1. assert len(mirror.published) == 1 assert mirror.published[0]["fallback_title"] == "note.txt" diff --git a/tests/stacklets/test_memory_search.py b/tests/stacklets/test_memory_search.py index 82c17465..23f6b396 100644 --- a/tests/stacklets/test_memory_search.py +++ b/tests/stacklets/test_memory_search.py @@ -347,7 +347,7 @@ def test_returns_result_dicts_with_expected_keys(self, vault): assert set(r.keys()) == { "path", "rel", "title", "date", "persons", "tags", "excerpt", "summary", - "paperless_id", + "paperless_id", "capture_id", } assert r["rel"].endswith("radlager.md") assert r["persons"] == ["Homer"] diff --git a/tests/stacklets/test_nl_query.py b/tests/stacklets/test_nl_query.py index 390b212d..ccdd3ac3 100644 --- a/tests/stacklets/test_nl_query.py +++ b/tests/stacklets/test_nl_query.py @@ -79,13 +79,13 @@ def test_memory_url_built_when_code_url_set(self): assert ev[0]["url"].startswith("https://code.example/") assert "family/memory/src/branch/main/family/notes/x.md" in ev[0]["url"] - def test_paperless_url_built_when_public_url_set(self): + def test_paperless_url_built_when_link_base_set(self): ev = build_evidence( memory_results=[], paperless_results=[{"id": 42, "title": "T", "created": "2026-01-01T00:00:00Z"}], - paperless_public_url="https://paperless.example", + link_base_url="https://home.example/go", ) - assert ev[0]["url"] == "https://paperless.example/documents/42/details" + assert ev[0]["url"] == "https://home.example/go/docs/42" def test_summary_falls_back_to_excerpt(self): # A vault file older than the classifier has no `> [!summary]` diff --git a/tests/stacklets/test_search_format.py b/tests/stacklets/test_search_format.py index bb3ecae8..992155c5 100644 --- a/tests/stacklets/test_search_format.py +++ b/tests/stacklets/test_search_format.py @@ -56,17 +56,20 @@ def test_respects_custom_org(self): class TestPaperlessDocUrl: - def test_returns_empty_without_public_url(self): + def test_returns_empty_without_link_base(self): assert paperless_doc_url(42) == "" def test_returns_empty_without_doc_id(self): # A doc without an id is malformed; surface that as "no link" # so the formatter falls back to a bold title. - assert paperless_doc_url(None, public_url="https://p") == "" + assert paperless_doc_url(None, link_base_url="https://home.example/go") == "" - def test_builds_detail_page_url(self): - url = paperless_doc_url(42, public_url="https://paperless.example") - assert url == "https://paperless.example/documents/42/details" + def test_builds_persistent_go_link(self): + # A search result sits in Matrix history forever, so the link + # is logical: the resolver points it at the document wherever + # it lives when someone finally clicks it. + url = paperless_doc_url(42, link_base_url="https://home.example/go") + assert url == "https://home.example/go/docs/42" # ── Memory hit formatting ──────────────────────────────────────────────── @@ -165,15 +168,15 @@ def _doc(**overrides) -> dict: "created": "2026-01-15T08:00:00Z", } | overrides - def test_includes_linked_title_with_public_url(self): + def test_includes_linked_title_with_link_base(self): out = format_paperless_hit( - self._doc(), 1, public_url="https://paperless.example", + self._doc(), 1, link_base_url="https://home.example/go", ) - assert "[Globex KFZ Versicherung 2026](https://paperless.example/documents/42/details)" in out + assert "[Globex KFZ Versicherung 2026](https://home.example/go/docs/42)" in out def test_meta_carries_date_and_doc_id(self): out = format_paperless_hit( - self._doc(), 1, public_url="https://p", + self._doc(), 1, link_base_url="https://home.example/go", ) # The created stamp is truncated to YYYY-MM-DD; #id is the # second metadata segment. diff --git a/tests/stacklets/test_search_service.py b/tests/stacklets/test_search_service.py index c11dd2f6..7ae113d1 100644 --- a/tests/stacklets/test_search_service.py +++ b/tests/stacklets/test_search_service.py @@ -58,7 +58,7 @@ def _service(paperless, *, shared_bucket="family"): language="en", code_public_url="http://code", mirror_org="family", - paperless_public_url="http://paperless", + link_base_url="http://home.test/go", shared_bucket=shared_bucket, vault=_FakeVault(), )