diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18162a9..aeeb5a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,6 +92,23 @@ jobs: -d '{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"search","arguments":{"query":"litestream"}}}' \ | grep -q 'Moving Tidewater from Postgres to SQLite' + - name: Upload an export as a visitor, then download and delete it + run: | + curl -fs -c visitor.txt http://127.0.0.1:7860/library/import \ + -H 'X-ChatLore: 1' -H 'X-Filename: conversations.json' \ + --data-binary @tests/fixtures/claude/conversations.json + for attempt in $(seq 60); do + curl -fs -b visitor.txt http://127.0.0.1:7860/library | grep -q '"running":false' && break + sleep 2 + done + curl -fs -b visitor.txt http://127.0.0.1:7860/library | grep -q '"stage":"done"' + curl -fs -b visitor.txt http://127.0.0.1:7860/stats | grep -q '"conversations":3,' + curl -fs http://127.0.0.1:7860/stats | grep -q '"conversations":32,' + curl -fs -b visitor.txt -o library.zip http://127.0.0.1:7860/library/export + unzip -l library.zip | grep -q manifest.json + curl -fs -b visitor.txt -X DELETE -H 'X-ChatLore: 1' http://127.0.0.1:7860/library + curl -fs -b visitor.txt http://127.0.0.1:7860/library | grep -q '"own":false' + - name: Show the server's output if: always() run: docker logs demo || true diff --git a/CHANGELOG.md b/CHANGELOG.md index b223a6c..b640677 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A `Dockerfile` for the hosted demo, with the demo library and embedding model built in. CI builds it, starts it, and checks the web interface, the API, and a tool call over MCP. Guide in `docs/hosting.md`. +- **Your data** in the web interface: upload a ChatGPT, Claude, or Gemini export, Markdown + notes, or a ChatLore archive, and watch it be imported, embedded, and read into the knowledge + graph in the background; download the library as a ChatLore archive or as Markdown. + `POST /library/import`, `GET /library`, `GET /library/export`, and `DELETE /library` do the + same over the API. Guide in `docs/web.md`. +- `chatlore serve --public --uploads` lets visitors try ChatLore on their own conversations: + each who uploads gets a private library tied to their browser, deleted after 24 hours + (`--keep-hours`) or when they choose, while everyone else sees the server's library. + `--max-upload-mb` and `--extract-limit` set how large an upload may be and how much of it the + model reads. The hosted demo image turns it on. Guide in `docs/hosting.md`. +- `export_archive` and `import_archive` take a store that is already open. - A FalkorDB store, chosen with `CHATLORE_STORE=falkordb`, with `CHATLORE_FALKORDB_URL` and `CHATLORE_FALKORDB_GRAPH` saying where. Every command, the web interface, and the MCP server work on it as on SQLite, and it passes the same contract tests. The client is an optional diff --git a/Dockerfile b/Dockerfile index 72aa83e..6e07976 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ # The hosted ChatLore demo: the made-up demo library behind the public web -# interface, API, and MCP endpoint. See docs/hosting.md. +# interface, API, and MCP endpoint, where visitors can also upload their own +# export into a private library that is deleted after 24 hours. See +# docs/hosting.md. # # docker build -t chatlore-demo . # docker run -p 7860:7860 -e OPENROUTER_API_KEY=... chatlore-demo @@ -28,4 +30,4 @@ RUN chatlore demo --no-serve \ EXPOSE 7860 # PORT is set by hosts such as Render and Fly.io; Hugging Face Spaces uses 7860. -CMD ["sh", "-c", "exec chatlore --home \"$HOME/.chatlore-demo\" serve --public --host 0.0.0.0 --port \"${PORT:-7860}\""] +CMD ["sh", "-c", "exec chatlore --home \"$HOME/.chatlore-demo\" serve --public --uploads --host 0.0.0.0 --port \"${PORT:-7860}\""] diff --git a/README.md b/README.md index 185e794..00b25ee 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,9 @@ chatlore stats ``` The library is kept in `~/.chatlore`; `--home ` or `CHATLORE_HOME` picks another. +The web interface can do the same without the terminal: after `chatlore serve`, +**Your data** uploads an export, builds everything from it, and downloads the +library again. The source is detected from the file. Importing is idempotent, so re-running it after a fresh export only adds what changed. How to get each export, what is @@ -98,6 +101,7 @@ extraction is an optional enrichment you can re-run with a better model later. | M8 | Easy to try: PyPI package, demo library, export and import | done, v0.1.0 | | M9 | Hosted demo | public mode, MCP over HTTP, and Docker image done; deployment next | | M10 | FalkorDB backend | done | +| M11 | Your own data in the web interface: upload, export, private visitor libraries | done | ## Development setup diff --git a/docs/hosting.md b/docs/hosting.md index 562f9ea..a1a0d0c 100644 --- a/docs/hosting.md +++ b/docs/hosting.md @@ -56,8 +56,44 @@ front of a hosting service: through a visitor's browser. - The web interface says it is a demo, and `/health` reports `"public": true`. -The library is only ever read. Never serve your own library this way: anyone -who finds the address can read every conversation in it. +The server's library is only ever read. Never serve your own library this way: +anyone who finds the address can read every conversation in it. + +## Visitors' own data + +The image also runs with `--uploads`, which lets visitors try ChatLore on their +own conversations: + +```bash +chatlore --home ~/.chatlore-demo serve --public --uploads --host 0.0.0.0 --port 7860 +``` + +- A visitor who uploads an export under **Your data** gets a library of their + own, and everything they then see, search, and ask is theirs alone. Everyone + else still sees the server's library. +- The library is tied to the visitor's browser by a random token in a cookie + (HttpOnly, SameSite=Lax, and Secure over HTTPS). Its folder is named after a + hash of the token, so the server's files do not give the token away. +- It is deleted after 24 hours, or `--keep-hours`, and at once when the visitor + chooses **Delete my library**; an import still running is stopped first. The + server looks for expired libraries every ten minutes. They are kept in + `~/.chatlore-spaces`, or `--spaces-dir`, each with its graph in its own SQLite + file, even when the server's library is on FalkorDB. +- Uploads are imported one at a time, so the server keeps answering while one + runs, and each visitor has one import at a time. An upload may be up to + 200 MB, or `--max-upload-mb`; a zip may unpack to at most 2 GB of text. +- Every change (an upload, a deletion) must carry the header the web interface + sends, `X-ChatLore: 1`, so another website cannot make a visitor's browser + upload or delete. +- Building the knowledge graph from an upload reads all of it with the + language model on the server's key, as `chatlore extract` does locally, so a + large export costs as much as extracting it at home. `--extract-limit` caps + how many passages of each upload the model reads; the rest stays searchable. + The dialog tells visitors that their conversations go to that model. +- Visitors download their library as a ChatLore archive or as Markdown. + +Assistants connected over `/mcp` see the server's library, since they carry no +visitor's cookie. ## Where to host it @@ -65,7 +101,9 @@ Any service that builds and runs a Dockerfile from a Git repository works, and gives the container an HTTPS address. ChatGPT only connects to HTTPS. CI builds the image on every pull request, starts it, and checks the web -interface, the API, and a tool call over MCP. +interface, the API, and a tool call over MCP. It then uploads an export as a +visitor, waits for the import, checks that only that visitor sees it, downloads +it, and deletes it. ## Connecting assistants to it diff --git a/docs/web.md b/docs/web.md index 805224e..8f20046 100644 --- a/docs/web.md +++ b/docs/web.md @@ -28,6 +28,27 @@ follows the system's light or dark setting. is a circle sized by how often it is mentioned and coloured by its topic, and related entities are linked. +## Your data + +**Your data**, in the navigation, imports an export and downloads the library: + +- Drop a file on the dialog, or choose one: a ChatGPT or Claude export (.zip), + Gemini Takeout (.zip or MyActivity.json), Markdown notes (.zip or .md), or a + ChatLore archive. Uploads may be up to 200 MB, or what `--max-upload-mb` sets. +- The import runs in the background, like `chatlore import`, `chatlore + process`, and `chatlore extract` one after another, and the dialog shows each + step with its progress: importing, preparing search, reading with the language + model, summarising, linking names for the same thing, and topics. Without a + model key, everything but the knowledge graph is built. When it is done, + **Show the library** reloads the page on it. +- **ChatLore archive** downloads the whole library, graph and caches included, + to import anywhere; **Markdown** downloads one readable file per conversation. + See [export.md](export.md). + +On your own machine the upload goes into the library itself. On a public server +that takes uploads, it goes into a private library of the visitor's own; see +[hosting.md](hosting.md#visitors-own-data). + ## Exploring the graph The graph fills the page, with a floating toolbar, a legend of the largest diff --git a/src/chatlore/api.py b/src/chatlore/api.py index a584071..a2e3f64 100644 --- a/src/chatlore/api.py +++ b/src/chatlore/api.py @@ -9,6 +9,12 @@ The MCP server is served at /mcp too, over streamable HTTP, for assistants that connect to a URL instead of starting ``chatlore mcp``. +The web interface can also upload an export, which is imported, embedded, and +read into the knowledge graph in the background, and download the library as +an archive or Markdown. On a public server that takes uploads, each visitor who +uploads gets a private library of their own that is deleted after a while +(``chatlore.spaces``); everyone else sees the server's library. + Each request opens the store and closes it again, so requests never share a database connection across threads. The server listens on 127.0.0.1 unless told otherwise, because the library is private. A public server, such as the hosted @@ -18,29 +24,43 @@ from __future__ import annotations +import functools +import shutil +import tempfile import threading import time +import uuid from collections import deque from collections.abc import AsyncIterator, Callable, Iterator from contextlib import asynccontextmanager +from contextvars import ContextVar +from http.cookies import SimpleCookie from pathlib import Path from typing import Annotated, Any +from urllib.parse import unquote +import anyio from fastapi import FastAPI, HTTPException, Query, Request +from fastapi.responses import FileResponse from fastapi.sse import EventSourceResponse, ServerSentEvent from fastapi.staticfiles import StaticFiles from mcp.server.transport_security import TransportSecuritySettings from pydantic import BaseModel, Field +from starlette.background import BackgroundTask from starlette.responses import Response -from starlette.types import Scope +from starlette.types import ASGIApp, Receive, Scope, Send from chatlore import __version__ +from chatlore.archive import export_archive, export_markdown from chatlore.chat import MAX_SOURCES, ChatLimits, Context, Source, answer, cited, retrieve from chatlore.embeddings import Embedder, EmbeddingError, make_embedder, normalise +from chatlore.imports import Imports, upload_suffix +from chatlore.library import Library from chatlore.llm import LLMError, make_llm from chatlore.mcp_server import create_server from chatlore.paths import default_home from chatlore.search import hybrid_search +from chatlore.spaces import COOKIE, Space, Spaces from chatlore.store import Edge, EdgeType, GraphStore, Label, Node, open_store WEB = Path(__file__).parent / "web" @@ -149,8 +169,62 @@ def _topic(node: Node, entities: int | None = None) -> dict[str, Any]: } +MAX_UPLOAD = 200_000_000 +"""Bytes an upload may have unless the server says otherwise.""" +REQUEST_HEADER = "X-ChatLore" +"""Sent by the web interface with every change. Another website cannot send it +without the browser first asking this server, which does not allow it, so it +cannot upload or delete through a visitor's browser.""" +_SWEEP_SECONDS = 600 + +# The visitor's own library, when the request carries a token for one. +_visitor: ContextVar[Space | None] = ContextVar("chatlore_visitor", default=None) + + +class _Visitors: + """Serve each request from its visitor's own library, when it has one.""" + + def __init__(self, app: ASGIApp, spaces: Spaces) -> None: + self.app = app + self.spaces = spaces + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + space = None + if scope["type"] == "http": + space = self.spaces.find(_cookie(scope, COOKIE)) + reset = _visitor.set(space) + try: + await self.app(scope, receive, send) + finally: + _visitor.reset(reset) + + +def _cookie(scope: Scope, name: str) -> str | None: + for key, value in scope.get("headers", []): + if key == b"cookie": + cookies: SimpleCookie = SimpleCookie() + try: + cookies.load(value.decode("latin-1")) + except Exception: # a malformed cookie header is no token + return None + if name in cookies: + return cookies[name].value + return None + + +def _changes_allowed(request: Request) -> None: + if request.headers.get(REQUEST_HEADER) != "1": + raise HTTPException(403, f"changes need the {REQUEST_HEADER} header") + + def create_app( - home: Path | None = None, *, public: bool = False, limits: ChatLimits | None = None + home: Path | None = None, + *, + public: bool = False, + limits: ChatLimits | None = None, + spaces: Spaces | None = None, + max_upload: int = MAX_UPLOAD, + extract_limit: int | None = None, ) -> FastAPI: """The API over the library in ``home``, or the default ChatLore home. @@ -159,8 +233,13 @@ def create_app( be reached from the internet. Otherwise /mcp only answers requests addressed to this machine, which keeps other websites from reaching it through the browser. ``limits`` caps the questions sent to the language model. + + Uploads go into the library on a private server. A public one takes them + only with ``spaces``, and puts each visitor's in a library of their own. + ``extract_limit`` caps how many passages of an upload the model reads. """ library = home or default_home() + uploads = not public or spaces is not None assistants = create_server(library) security = TransportSecuritySettings(enable_dns_rebinding_protection=False) if public else None # Stateless, with plain JSON replies: any worker can answer any request, and @@ -169,10 +248,45 @@ def create_app( stateless_http=True, json_response=True, transport_security=security ) + embedders: list[Embedder] = [] + loading = threading.Lock() + + def embedder() -> Embedder: + """One embedding model for the whole server, loaded on first use.""" + with loading: + if not embedders: + embedders.append(make_embedder(library)) + return embedders[0] + + # make_llm is looked up when an import needs it, so it can be replaced in tests. + imports = Imports(embedder, lambda: make_llm(), extract_limit=extract_limit) + + def sweep() -> None: + """Delete the visitors' libraries whose time is up.""" + if spaces is None: + return + for space in spaces.expired(): + imports.cancel(space.home, then=functools.partial(_forget, space)) + + def _forget(space: Space) -> None: + assert spaces is not None + spaces.delete(space) + imports.forget(space.home) + + async def sweeping() -> None: + while True: + await anyio.to_thread.run_sync(sweep) + await anyio.sleep(_SWEEP_SECONDS) + @asynccontextmanager async def lifespan(_: FastAPI) -> AsyncIterator[None]: - async with assistants.session_manager.run(): - yield + async with anyio.create_task_group() as tasks: + if spaces is not None: + tasks.start_soon(sweeping) + async with assistants.session_manager.run(): + yield + tasks.cancel_scope.cancel() + await anyio.to_thread.run_sync(imports.shutdown) app = FastAPI( title="ChatLore", @@ -180,26 +294,149 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: description="All your AI conversations, one graph, one chat.", lifespan=lifespan, ) - embedders: list[Embedder] = [] allowance = _Allowance(limits) if limits is not None else None + if spaces is not None: + app.add_middleware(_Visitors, spaces=spaces) def store() -> GraphStore: - return open_store(library) + space = _visitor.get() + return space.open_store() if space is not None else open_store(library) + + def current_home() -> Path: + space = _visitor.get() + return space.home if space is not None else library def embed(text: str, graph: GraphStore) -> list[float] | None: """The query's embedding, or None when the library has no embeddings.""" if graph.count_embeddings() == 0: return None - if not embedders: - embedders.append(make_embedder(library)) try: - return normalise(embedders[0].embed_query(text)) + return normalise(embedder().embed_query(text)) except EmbeddingError as error: raise HTTPException(503, str(error)) from error @app.get("/health") def health() -> dict[str, Any]: - return {"status": "ok", "version": __version__, "public": public} + return {"status": "ok", "version": __version__, "public": public, "uploads": uploads} + + @app.get("/library") + def library_info() -> dict[str, Any]: + """Whose library this is, whether uploads are taken, and the latest import.""" + space = _visitor.get() + status = imports.status(current_home()) + return { + "own": space is not None or not public, + "uploads": uploads, + "public": public, + "expires_at": space.expires_at.isoformat() if space is not None else None, + "keep_hours": spaces.keep.total_seconds() / 3600 if spaces is not None else None, + "max_upload_mb": max_upload // 1_000_000, + "import": status.as_dict() if status is not None else None, + } + + @app.post("/library/import", status_code=202) + async def import_upload(request: Request, response: Response) -> dict[str, Any]: + """Take an export as the request body and import it in the background. + + Send the file's name in ``X-Filename``. On a public server the upload goes + into the visitor's own library, made on the first upload, and a cookie + remembers it. ``GET /library`` shows how far the import got. + """ + _changes_allowed(request) + if not uploads: + raise HTTPException(403, "this server does not take uploads") + declared = request.headers.get("content-length") + if declared is not None and declared.isdigit() and int(declared) > max_upload: + raise HTTPException(413, f"uploads may be up to {max_upload // 1_000_000} MB") + space = _visitor.get() + created: Space | None = None + if public: + assert spaces is not None + if space is None: + token, space = spaces.create() + created = space + response.set_cookie( + COOKIE, + token, + max_age=int(spaces.keep.total_seconds()), + httponly=True, + samesite="lax", + secure=request.url.scheme == "https", + ) + target, opener = space.home, space.open_store + else: + target, opener = library, lambda: open_store(library) + try: + if imports.running(target): + raise HTTPException(409, "an import is already running for this library") + name = unquote(request.headers.get("x-filename") or "upload")[:200] + folder = target / "uploads" + folder.mkdir(parents=True, exist_ok=True) + path = folder / f"{uuid.uuid4().hex}{upload_suffix(name)}" + size = 0 + try: + with path.open("wb") as handle: + async for piece in request.stream(): + size += len(piece) + if size > max_upload: + raise HTTPException( + 413, f"uploads may be up to {max_upload // 1_000_000} MB" + ) + handle.write(piece) + if size == 0: + raise HTTPException(400, "the upload is empty") + except BaseException: + path.unlink(missing_ok=True) + raise + except BaseException: + if created is not None and spaces is not None: + spaces.delete(created) + raise + return imports.start(target, path, name, opener).as_dict() + + @app.get("/library/export") + def export_library( + format: Annotated[str, Query(pattern="^(archive|markdown)$")] = "archive", + ) -> FileResponse: + """The library as a ChatLore archive, or as a zip of Markdown files.""" + home = current_home() + if not Library(home).stats(): + raise HTTPException(404, "the library is empty") + folder = Path(tempfile.mkdtemp(prefix="chatlore-export-")) + try: + if format == "archive": + target = folder / "chatlore-library.zip" + with store() as graph: + export_archive(home, target, store=graph) + else: + export_markdown(Library(home), folder / "markdown") + target = Path( + shutil.make_archive( + str(folder / "chatlore-markdown"), "zip", folder / "markdown" + ) + ) + except BaseException: + shutil.rmtree(folder, ignore_errors=True) + raise + return FileResponse( + target, + media_type="application/zip", + filename=target.name, + background=BackgroundTask(shutil.rmtree, folder, ignore_errors=True), + ) + + @app.delete("/library") + def delete_library(request: Request, response: Response) -> dict[str, str]: + """Delete the visitor's own library now, stopping any import into it first.""" + _changes_allowed(request) + space = _visitor.get() + if space is None: + if not public: + raise HTTPException(403, "the server's library cannot be deleted from here") + raise HTTPException(404, "you have no library of your own on this server") + imports.cancel(space.home, then=lambda: _forget(space)) + response.delete_cookie(COOKIE) + return {"status": "deleted"} @app.get("/stats") def stats() -> dict[str, int]: diff --git a/src/chatlore/archive.py b/src/chatlore/archive.py index ebdfa44..6f0db3c 100644 --- a/src/chatlore/archive.py +++ b/src/chatlore/archive.py @@ -24,7 +24,7 @@ import zipfile from collections import Counter from collections.abc import Iterable, Iterator -from contextlib import closing +from contextlib import AbstractContextManager, closing, nullcontext from dataclasses import dataclass, field from datetime import UTC, datetime from pathlib import Path @@ -89,25 +89,26 @@ class ImportReport: # -- archives ---------------------------------------------------------------- -def export_archive(home: Path, target: Path) -> ExportReport: +def export_archive(home: Path, target: Path, store: GraphStore | None = None) -> ExportReport: """Write the library at ``home`` into one archive at ``target``. - The file is written next to ``target`` and moved into place at the end, so - a failed export never leaves half an archive behind. + The graph is read from ``store``, or the library's own store when none is + given. The file is written next to ``target`` and moved into place at the + end, so a failed export never leaves half an archive behind. """ partial = target.with_name(target.name + ".partial") conversations = 0 caches: list[str] = [] try: with ( - open_store(home) as store, + _opened(home, store) as graph, zipfile.ZipFile(partial, "w", zipfile.ZIP_DEFLATED) as archive, ): with archive.open(CONVERSATIONS, "w") as handle: for conversation in Library(home): handle.write(_line(conversation.model_dump(mode="json"))) conversations += 1 - nodes, edges = _derived_graph(store) + nodes, edges = _derived_graph(graph) with archive.open(GRAPH, "w") as handle: for node in nodes: handle.write(_line({"node": [node.id, node.label, node.props]})) @@ -197,13 +198,16 @@ def read_graph(path: Path) -> tuple[list[Node], list[Edge]]: return nodes, edges -def import_archive(path: Path, home: Path, dry_run: bool = False) -> ImportReport: +def import_archive( + path: Path, home: Path, dry_run: bool = False, store: GraphStore | None = None +) -> ImportReport: """Add an archive's conversations, graph, and caches to the library at ``home``. Conversations are added like any import, so importing twice changes nothing and a conversation already in the library is replaced by the archive's copy - only when they differ. Nodes and edges are merged into the graph, and cache - entries the library lacks are added to its caches. + only when they differ. Nodes and edges are merged into the graph, in + ``store`` or the library's own store, and cache entries the library lacks + are added to its caches. """ read_manifest(path) report = ImportReport() @@ -215,19 +219,24 @@ def import_archive(path: Path, home: Path, dry_run: bool = False) -> ImportRepor report.nodes, report.edges = len(nodes), len(edges) return report - with Library(home) as library, open_store(home) as store, store.transaction(): - with ConversationWriter(store) as writer: + with Library(home) as library, _opened(home, store) as graph, graph.transaction(): + with ConversationWriter(graph) as writer: for conversation in read_conversations(path): outcome = library.add(conversation) report.outcomes[outcome.value] += 1 report.messages += len(conversation.messages) if outcome is not AddOutcome.UNCHANGED: writer.add(conversation) - report.nodes, report.edges = restore_graph(store, *read_graph(path)) + report.nodes, report.edges = restore_graph(graph, *read_graph(path)) restore_caches(path, home) return report +def _opened(home: Path, store: GraphStore | None) -> AbstractContextManager[GraphStore]: + """The store given, left open for its owner, or the library's own, closed after.""" + return nullcontext(store) if store is not None else open_store(home) + + def restore_graph(store: GraphStore, nodes: list[Node], edges: list[Edge]) -> tuple[int, int]: """Merge nodes and edges into ``store``. Returns how many of each were written. diff --git a/src/chatlore/cli.py b/src/chatlore/cli.py index 55c974a..faa80bc 100644 --- a/src/chatlore/cli.py +++ b/src/chatlore/cli.py @@ -11,6 +11,7 @@ from collections import Counter from collections.abc import Iterator from contextlib import contextmanager +from datetime import timedelta from pathlib import Path from typing import Annotated, Any @@ -41,7 +42,7 @@ ) from chatlore.library import AddOutcome, Library from chatlore.llm import OPENROUTER_KEY_ENV, LLMError, llm_settings, make_llm -from chatlore.paths import HOME_ENV, default_home, demo_home +from chatlore.paths import HOME_ENV, default_home, demo_home, spaces_home from chatlore.pipeline import ( EmbeddingModelMismatchError, pending_duplicates, @@ -707,6 +708,7 @@ def ask( _LOCAL_HOSTS = ("127.0.0.1", "localhost", "::1") +MAX_UPLOAD_MB = 200 @app.command() @@ -743,16 +745,66 @@ def serve( help="With --public: questions all visitors may ask per day.", ), ] = ChatLimits().per_day, + uploads: Annotated[ + bool, + typer.Option( + "--uploads", + help=( + "With --public: let each visitor upload an export into a private library " + "of their own, deleted after --keep-hours." + ), + ), + ] = False, + keep_hours: Annotated[ + int, + typer.Option( + "--keep-hours", min=1, help="With --uploads: hours a visitor's library stays." + ), + ] = 24, + spaces_dir: Annotated[ + Path | None, + typer.Option( + "--spaces-dir", + help="With --uploads: where visitors' libraries are kept (~/.chatlore-spaces).", + show_default=False, + ), + ] = None, + max_upload_mb: Annotated[ + int, typer.Option("--max-upload-mb", min=1, help="Largest upload taken, in MB.") + ] = MAX_UPLOAD_MB, + extract_limit: Annotated[ + int | None, + typer.Option( + "--extract-limit", + min=1, + help="Passages of each upload the model reads into the graph; all by default.", + show_default=False, + ), + ] = None, ) -> None: - """Serve the web interface, the REST API, and MCP at /mcp. API docs are at /docs.""" + """Serve the web interface, the REST API, and MCP at /mcp. API docs are at /docs. + + The web interface can upload exports into the library and download it. On a + public server, --uploads gives each visitor who uploads a private library. + """ import uvicorn # loaded here so other commands start without the web stack from chatlore.api import create_app + from chatlore.spaces import Spaces options: dict[str, Any] = {} + settings: dict[str, Any] = { + "max_upload": max_upload_mb * 1_000_000, + "extract_limit": extract_limit, + } if public: limits = ChatLimits(per_visitor_hour=questions_per_hour, per_day=questions_per_day) - application = create_app(public=True, limits=limits) + visitors = None + if uploads: + visitors = Spaces( + (spaces_dir or spaces_home()).expanduser(), keep=timedelta(hours=keep_hours) + ) + application = create_app(public=True, limits=limits, spaces=visitors, **settings) # The proxy in front says who the visitor is; without it every visitor # would share the proxy's address, and its limit. options = {"proxy_headers": True, "forwarded_allow_ips": "*"} @@ -761,12 +813,17 @@ def serve( f"this library and ask {questions_per_hour} questions an hour each, " f"{questions_per_day} a day in all." ) + if visitors is not None: + console.print( + f"Visitors can upload their own exports: each gets a private library in " + f"{display_path(visitors.root)}, deleted after {keep_hours} hours." + ) else: - application = create_app() + application = create_app(**settings) if host not in _LOCAL_HOSTS: console.print( "[yellow]Listening beyond this machine: anyone who can reach this address " - "can read your library.[/yellow]" + "can read your library and import into it.[/yellow]" ) console.print(f"ChatLore API on http://{host}:{port} (docs: http://{host}:{port}/docs)") uvicorn.run(application, host=host, port=port, log_level="warning", **options) diff --git a/src/chatlore/imports.py b/src/chatlore/imports.py new file mode 100644 index 0000000..81c9f12 --- /dev/null +++ b/src/chatlore/imports.py @@ -0,0 +1,323 @@ +"""Importing an uploaded export in the background: import, embed, and build the graph. + +The web interface hands an upload to ``Imports``, which runs one import at a +time on a worker thread and keeps its status, so the page can show how far it +got while the server goes on answering other requests. The steps are the ones +the command line runs, in the same order: ``chatlore import``, ``chatlore +process``, and ``chatlore extract``. Everything each step finishes is kept, so +an import that fails or is cancelled halfway leaves what it had done. +""" + +from __future__ import annotations + +import logging +import tempfile +import threading +import zipfile +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from dataclasses import asdict, dataclass, field +from datetime import UTC, datetime +from pathlib import Path, PurePosixPath +from typing import Any, Protocol + +from chatlore.archive import ArchiveError, import_archive, is_archive +from chatlore.embeddings import Embedder, EmbeddingCache, EmbeddingError +from chatlore.extraction import ExtractionCache +from chatlore.importers import ImporterError, detect_source, get_importer +from chatlore.library import AddOutcome, Library +from chatlore.llm import LLM, LLMError +from chatlore.models import SourceKind +from chatlore.pipeline import ( + EmbeddingModelMismatchError, + pending_duplicates, + pending_extractions, + pending_summaries, + pending_topics, + sync_chunks, + sync_duplicates, + sync_embeddings, + sync_entities, + sync_extractions, + sync_summaries, + sync_topics, +) +from chatlore.store import ConversationWriter, GraphStore + +logger = logging.getLogger(__name__) + +SUFFIXES = frozenset({".zip", ".json", ".md", ".markdown", ".txt"}) +"""File types an upload may have; they tell the importers what they are reading.""" +MAX_UNPACKED = 2_000_000_000 +"""Bytes of text a zip may unpack to, so a small zip cannot fill the disk.""" +_TEXT = (".json", ".jsonl", ".md", ".markdown", ".txt") + + +class _ClosableLLM(LLM, Protocol): + def close(self) -> None: ... + + +class ImportCancelledError(Exception): + """The import was stopped on request.""" + + +@dataclass(slots=True) +class ImportStatus: + """Where an import is: from ``waiting`` to ``done``, ``failed``, or ``cancelled``. + + The stages in between are ``importing``, ``embedding``, ``reading`` (the + model reads the chunks), ``summarising``, ``linking`` (possible duplicates), + and ``topics``, each with ``done`` out of ``total``. + """ + + file: str + stage: str = "waiting" + done: int = 0 + total: int = 0 + conversations: int = 0 + messages: int = 0 + skipped: int = 0 + notes: list[str] = field(default_factory=list) + error: str | None = None + started_at: str = field(default_factory=lambda: datetime.now(UTC).isoformat()) + finished_at: str | None = None + + @property + def running(self) -> bool: + return self.finished_at is None + + def as_dict(self) -> dict[str, Any]: + return {**asdict(self), "running": self.running} + + +class Imports: + """Runs uploads through the pipeline, one at a time, and keeps each library's latest.""" + + def __init__( + self, + embedder: Callable[[], Embedder], + llm: Callable[[], _ClosableLLM], + extract_limit: int | None = None, + ) -> None: + self._embedder = embedder + self._llm = llm + self.extract_limit = extract_limit + self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="chatlore-import") + self._lock = threading.Lock() + self._status: dict[Path, ImportStatus] = {} + self._cancelled: set[Path] = set() + self._finished: dict[Path, list[Callable[[], None]]] = {} + + def status(self, home: Path) -> ImportStatus | None: + with self._lock: + return self._status.get(home) + + def running(self, home: Path) -> bool: + status = self.status(home) + return status is not None and status.running + + def start( + self, home: Path, upload: Path, name: str, store: Callable[[], GraphStore] + ) -> ImportStatus: + """Queue ``upload`` for the library at ``home``, whose graph ``store`` opens.""" + status = ImportStatus(file=name) + with self._lock: + self._status[home] = status + self._cancelled.discard(home) + self._executor.submit(self._run, home, upload, store, status) + return status + + def cancel(self, home: Path, then: Callable[[], None] | None = None) -> bool: + """Stop the library's import at its next step. Returns whether one was running. + + ``then`` runs once the import has stopped, or at once when none runs. + """ + with self._lock: + status = self._status.get(home) + running = status is not None and status.running + if running: + self._cancelled.add(home) + if then is not None: + self._finished.setdefault(home, []).append(then) + if not running and then is not None: + then() + return running + + def forget(self, home: Path) -> None: + with self._lock: + self._status.pop(home, None) + + def shutdown(self) -> None: + with self._lock: + self._cancelled.update(home for home, status in self._status.items() if status.running) + self._executor.shutdown(wait=True, cancel_futures=True) + + def _run( + self, home: Path, upload: Path, store: Callable[[], GraphStore], status: ImportStatus + ) -> None: + def check() -> None: + with self._lock: + if home in self._cancelled: + raise ImportCancelledError + + def advance(count: int) -> None: + status.done += count + check() + + graph: GraphStore | None = None + try: + check() + graph = store() + status.stage = "importing" + _import(upload, home, graph, status) + check() + self._embed(home, graph, status, advance) + check() + self._build_graph(home, graph, status, advance) + status.stage = "done" + except ImportCancelledError: + status.stage = "cancelled" + except ( + ImporterError, + ArchiveError, + EmbeddingError, + EmbeddingModelMismatchError, + LLMError, + ) as error: + status.stage, status.error = "failed", str(error) + except Exception as error: # a bad upload must not stop the worker + logger.exception("import of %s failed", status.file) + status.stage, status.error = "failed", f"the import stopped unexpectedly: {error}" + finally: + if graph is not None: + graph.close() + upload.unlink(missing_ok=True) + status.finished_at = datetime.now(UTC).isoformat() + with self._lock: + self._cancelled.discard(home) + waiting = self._finished.pop(home, []) + for then in waiting: + then() + + def _embed( + self, home: Path, graph: GraphStore, status: ImportStatus, advance: Callable[[int], None] + ) -> None: + chunks = sync_chunks(graph, Library(home)) + status.stage, status.done = "embedding", 0 + status.total = chunks.total - graph.count_embeddings() + with EmbeddingCache(home / "cache" / "embeddings.db") as cache: + sync_embeddings(graph, self._embedder(), cache, on_progress=advance) + + def _build_graph( + self, home: Path, graph: GraphStore, status: ImportStatus, advance: Callable[[int], None] + ) -> None: + """As ``chatlore extract``: entities, summaries, duplicates, then topics.""" + try: + llm = self._llm() + except LLMError as error: + status.notes.append(f"The knowledge graph was not built: {error}") + return + try: + with ExtractionCache(home / "cache" / "extractions.db") as cache: + pending = len(pending_extractions(graph, cache, llm.name)) + limit = self.extract_limit + status.stage, status.done = "reading", 0 + status.total = pending if limit is None else min(pending, limit) + if limit is not None and pending > limit: + status.notes.append( + f"The model read {limit} of {pending} passages; the rest are " + "searchable but not in the knowledge graph." + ) + try: + sync_extractions(graph, llm, cache, limit=limit, on_progress=advance) + except LLMError as error: + # As on the command line: keep what was read so far. + sync_entities(graph, cache, llm.name) + status.notes.append(f"The model stopped answering: {error}") + return + sync_entities(graph, cache, llm.name) + try: + status.stage, status.done = "summarising", 0 + status.total = len(pending_summaries(graph)) + sync_summaries(graph, llm, cache, on_progress=advance) + status.stage, status.done = "linking", 0 + status.total = pending_duplicates(graph, cache, llm.name) + sync_duplicates(graph, llm, cache, on_progress=advance) + status.stage, status.done = "topics", 0 + status.total = pending_topics(graph, cache, llm.name) + sync_topics(graph, llm, cache, on_progress=advance) + except LLMError as error: + status.notes.append( + f"The model stopped answering while {status.stage}: {error}" + ) + finally: + llm.close() + + +def upload_suffix(name: str) -> str: + """The suffix to save an upload with, taken from its name when it is one ChatLore reads.""" + suffix = PurePosixPath(name.replace("\\", "/")).suffix.lower() + return suffix if suffix in SUFFIXES else "" + + +def _import(upload: Path, home: Path, graph: GraphStore, status: ImportStatus) -> None: + """Add the upload's conversations to the library and its graph.""" + if zipfile.is_zipfile(upload): + _check_unpacked_size(upload) + if is_archive(upload): + report = import_archive(upload, home, store=graph) + status.conversations = report.outcomes[AddOutcome.NEW] + report.outcomes[AddOutcome.UPDATED] + status.messages = report.messages + return + with tempfile.TemporaryDirectory() as folder: + try: + path, kind = upload, detect_source(upload) + except ImporterError: + unpacked = _markdown_folder(upload, Path(folder)) + if unpacked is None: + raise + path, kind = unpacked, SourceKind.MARKDOWN + issues: list[object] = [] + with ( + Library(home) as library, + graph.transaction(), + ConversationWriter(graph) as writer, + ): + for conversation in get_importer(kind.value).parse(path, issues.append): + status.messages += len(conversation.messages) + if library.add(conversation) is not AddOutcome.UNCHANGED: + status.conversations += 1 + writer.add(conversation) + status.skipped = len(issues) + if status.conversations == 0 and status.messages == 0: + raise ImporterError("the upload holds no conversations ChatLore can read") + + +def _check_unpacked_size(upload: Path) -> None: + with zipfile.ZipFile(upload) as archive: + size = sum( + member.file_size + for member in archive.infolist() + if member.filename.lower().endswith(_TEXT) + ) + if size > MAX_UNPACKED: + raise ImporterError("the zip unpacks to more text than this server accepts") + + +def _markdown_folder(upload: Path, folder: Path) -> Path | None: + """Unpack a zip of Markdown or text notes, as a vault is shared; None for any other zip.""" + if not zipfile.is_zipfile(upload): + return None + with zipfile.ZipFile(upload) as archive: + notes = [ + member + for member in archive.infolist() + if not member.is_dir() + and member.filename.lower().endswith((".md", ".markdown", ".txt")) + ] + if not notes: + return None + target = folder / "notes" + for member in notes: + archive.extract(member, target) # zipfile keeps members inside the target + return target diff --git a/src/chatlore/paths.py b/src/chatlore/paths.py index c33416b..75fd136 100644 --- a/src/chatlore/paths.py +++ b/src/chatlore/paths.py @@ -17,6 +17,11 @@ def default_home() -> Path: return Path(override).expanduser() if override else Path.home() / ".chatlore" +def spaces_home() -> Path: + """Return where a public server keeps its visitors' own libraries.""" + return Path.home() / ".chatlore-spaces" + + def demo_home() -> Path: """Return where ``chatlore demo`` puts its made-up library, apart from the real one.""" return Path.home() / ".chatlore-demo" diff --git a/src/chatlore/spaces.py b/src/chatlore/spaces.py new file mode 100644 index 0000000..2b78039 --- /dev/null +++ b/src/chatlore/spaces.py @@ -0,0 +1,116 @@ +"""Private, temporary libraries for the visitors of a public server. + +A visitor who brings their own export gets a library of their own, tied to +their browser by a random token kept in a cookie. Only the token's holder can +reach it: its folder is named after a hash of the token, never the token +itself, so the folder names on the server give nothing away. Each library is +deleted when it expires, 24 hours after it was made by default, or earlier when +its visitor deletes it. + +A visitor's library always keeps its graph in a SQLite file in its folder, even +when the server's own library is on FalkorDB, so that deleting the folder +deletes everything. +""" + +from __future__ import annotations + +import hashlib +import json +import secrets +import shutil +from collections.abc import Callable +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from chatlore.store import DATABASE_NAME, GraphStore, SQLiteStore + +COOKIE = "chatlore_library" +"""The cookie holding a visitor's token.""" +DEFAULT_KEEP = timedelta(hours=24) +_META = "space.json" +_TOKEN_LENGTH = 200 + + +@dataclass(frozen=True, slots=True) +class Space: + """One visitor's library.""" + + id: str + home: Path + created_at: datetime + expires_at: datetime + + def open_store(self) -> GraphStore: + """The library's graph, in a SQLite file in its folder.""" + self.home.mkdir(parents=True, exist_ok=True) + return SQLiteStore(self.home / DATABASE_NAME) + + +class Spaces: + """The visitors' libraries under one folder.""" + + def __init__( + self, + root: Path, + keep: timedelta = DEFAULT_KEEP, + clock: Callable[[], datetime] = lambda: datetime.now(UTC), + ) -> None: + self.root = root + self.keep = keep + self._clock = clock + + def create(self) -> tuple[str, Space]: + """A new library and the token that reaches it.""" + token = secrets.token_urlsafe(32) + created = self._clock() + space = Space(_id(token), self.root / _id(token), created, created + self.keep) + space.home.mkdir(parents=True, exist_ok=True) + meta = {"created_at": created.isoformat(), "expires_at": space.expires_at.isoformat()} + (space.home / _META).write_text(json.dumps(meta), encoding="utf-8") + return token, space + + def find(self, token: str | None) -> Space | None: + """The library a token reaches, unless it expired or was deleted.""" + if not token or len(token) > _TOKEN_LENGTH: + return None + space = self._read(self.root / _id(token)) + if space is None or space.expires_at <= self._clock(): + return None + return space + + def expired(self) -> list[Space]: + """Every library whose time is up, or whose details cannot be read.""" + if not self.root.exists(): + return [] + found: list[Space] = [] + for home in sorted(self.root.iterdir()): + if not home.is_dir(): + continue + space = self._read(home) + if space is None: + now = self._clock() + found.append(Space(home.name, home, now, now)) + elif space.expires_at <= self._clock(): + found.append(space) + return found + + def delete(self, space: Space) -> None: + """Delete a library's folder with everything in it.""" + shutil.rmtree(space.home, ignore_errors=True) + + def _read(self, home: Path) -> Space | None: + try: + meta = json.loads((home / _META).read_text(encoding="utf-8")) + return Space( + home.name, + home, + datetime.fromisoformat(meta["created_at"]), + datetime.fromisoformat(meta["expires_at"]), + ) + except (OSError, ValueError, KeyError, TypeError): + return None + + +def _id(token: str) -> str: + return hashlib.sha256(token.encode("utf-8")).hexdigest()[:32] diff --git a/src/chatlore/web/app.js b/src/chatlore/web/app.js index df323d2..f1211bd 100644 --- a/src/chatlore/web/app.js +++ b/src/chatlore/web/app.js @@ -115,7 +115,7 @@ function show(view) { if (view === "ask") $("#ask-input").focus(); } -document.querySelectorAll("nav button").forEach((button) => button.addEventListener("click", () => show(button.dataset.view))); +document.querySelectorAll("nav button[data-view]").forEach((button) => button.addEventListener("click", () => show(button.dataset.view))); document.querySelectorAll("dialog [data-close]").forEach((button) => button.addEventListener("click", () => button.closest("dialog").close())); document.querySelectorAll("dialog").forEach((dialog) => dialog.addEventListener("click", (event) => event.target === dialog && dialog.close()), @@ -128,15 +128,177 @@ api("/stats") }) .catch(() => {}); -// A public server, such as the hosted demo, says so. -api("/health") - .then((health) => { - if (!health.public) return; - $("#ask-hero p").textContent = "A demo on made-up conversations. Every answer cites the chats it came from."; - $("#demo-note").hidden = false; +// A public server, such as the hosted demo, says so, unless the visitor brought their own library. +api("/library") + .then((library) => { + Data.info = library; + Data.render(); + if (library.import?.running) Data.poll(); + if (!library.public) return; + if (library.own) { + $("#ask-hero p").textContent = "Your own conversations, kept on this server only for you. Every answer cites its sources."; + } else { + $("#ask-hero p").textContent = "A demo on made-up conversations. Every answer cites the chats it came from."; + $("#demo-note").hidden = false; + } }) .catch(() => {}); +// -- your data ------------------------------------------------------------------------- + +const STAGES = { + waiting: "Waiting to start", + importing: "Importing conversations", + embedding: "Preparing search", + reading: "Reading with the language model", + summarising: "Summarising entities", + linking: "Linking names for the same thing", + topics: "Writing topic reports", + done: "Done", + failed: "The import failed", + cancelled: "Stopped", +}; + +const Data = { + info: null, + timer: null, + + hoursLeft() { + const left = (new Date(this.info.expires_at) - Date.now()) / 3_600_000; + return left < 1 ? "in less than an hour" : `in about ${Math.round(left)} hours`; + }, + + render() { + const info = this.info; + if (!info) return; + $("#data-open").hidden = false; + $("#data-max").textContent = number(info.max_upload_mb); + $("#data-drop").hidden = !info.uploads; + $("#data-delete").hidden = !(info.public && info.own); + const privacy = $("#data-privacy"); + if (info.public && info.uploads) { + privacy.hidden = false; + privacy.textContent = + `Your upload goes to this server only, into a library nobody else can see, tied to this browser. ` + + `It is deleted after ${info.keep_hours} hours, or now with Delete. Building the knowledge graph ` + + `sends your conversations to the language model this server uses.`; + } + $("#data-meta").textContent = !info.public + ? "Import exports into this library, or download it." + : info.own + ? `Your private library on this server, deleted ${this.hoursLeft()}.` + : info.uploads + ? "Try ChatLore on your own conversations." + : "Download the demo library."; + const status = info.import; + $("#data-progress").hidden = !status; + if (status) this.show(status); + }, + + show(status, uploaded) { + $("#data-drop").classList.toggle("busy", Boolean(status.running)); + $("#data-stage").textContent = uploaded === undefined ? STAGES[status.stage] || status.stage : "Uploading"; + const bar = $("#data-bar"); + if (uploaded !== undefined) { + bar.value = uploaded; + $("#data-count").textContent = `${Math.round(uploaded * 100)}%`; + } else if (status.total > 0 && status.running) { + bar.value = Math.min(1, status.done / status.total); + $("#data-count").textContent = `${number(status.done)} of ${number(status.total)}`; + } else { + bar.value = status.running ? 0 : 1; + bar.toggleAttribute("value", !status.running || status.total > 0); + $("#data-count").textContent = ""; + } + const summary = []; + if (status.file) summary.push(status.file); + if (status.conversations) summary.push(`${number(status.conversations)} conversations added`); + if (status.skipped) summary.push(`${number(status.skipped)} records skipped`); + $("#data-summary").innerHTML = esc(summary.join(" ยท ")) + (status.error ? `
${esc(status.error)}` : ""); + const notes = [...(status.notes || [])]; + $("#data-notes").innerHTML = notes.map((note) => `
  • ${esc(note)}
    • `).join(""); + if (status.stage === "done" && !status.running) { + $("#data-notes").insertAdjacentHTML("beforeend", '
    • '); + $("#data-show").addEventListener("click", () => location.reload()); + } + }, + + async refresh() { + this.info = await api("/library"); + this.render(); + }, + + poll() { + clearTimeout(this.timer); + this.timer = setTimeout(async () => { + try { + await this.refresh(); + } catch { + // try again on the next tick + } + if (this.info?.import?.running) this.poll(); + }, 1500); + }, + + upload(file) { + if (!file) return; + if (file.size > this.info.max_upload_mb * 1_000_000) { + $("#data-progress").hidden = false; + this.show({ file: file.name, stage: "failed", running: false, error: `The file is larger than ${this.info.max_upload_mb} MB.` }); + return; + } + $("#data-progress").hidden = false; + const request = new XMLHttpRequest(); + request.open("POST", "library/import"); + request.setRequestHeader("X-ChatLore", "1"); + request.setRequestHeader("X-Filename", encodeURIComponent(file.name)); + request.setRequestHeader("Content-Type", "application/octet-stream"); + request.upload.onprogress = (event) => { + if (event.lengthComputable) this.show({ file: file.name, running: true }, event.loaded / event.total); + }; + request.onload = () => { + let body = {}; + try { + body = JSON.parse(request.responseText); + } catch { + // an empty or plain-text answer + } + if (request.status === 202) { + this.show(body); + this.poll(); + } else { + this.show({ file: file.name, stage: "failed", running: false, error: body.detail || `The server answered ${request.status}.` }); + } + }; + request.onerror = () => this.show({ file: file.name, stage: "failed", running: false, error: "The upload did not reach the server." }); + request.send(file); + }, + + async remove() { + if (!confirm("Delete your library from this server now? This cannot be undone.")) return; + const response = await fetch("library", { method: "DELETE", headers: { "X-ChatLore": "1" } }); + if (response.ok) location.reload(); + }, +}; + +$("#data-open").addEventListener("click", () => { + Data.refresh().catch(() => {}); + $("#data").showModal(); +}); +$("#data-file").addEventListener("change", (event) => Data.upload(event.target.files[0])); +$("#data-delete").addEventListener("click", () => Data.remove()); +const drop = $("#data-drop"); +drop.addEventListener("dragover", (event) => { + event.preventDefault(); + drop.classList.add("over"); +}); +drop.addEventListener("dragleave", () => drop.classList.remove("over")); +drop.addEventListener("drop", (event) => { + event.preventDefault(); + drop.classList.remove("over"); + Data.upload(event.dataTransfer.files[0]); +}); + // -- conversations --------------------------------------------------------------------- async function openConversation(id, messageId) { @@ -193,7 +355,7 @@ const Conversations = { ${esc(date(c.updated_at || c.created_at))}`, ) .join("") - : `

    ${this.all.length ? "No conversation title matches." : "Nothing imported yet. Run chatlore import."}

    `; + : `

    ${this.all.length ? "No conversation title matches." : "Nothing imported yet. Import an export under Your data."}

    `; }, }; diff --git a/src/chatlore/web/index.html b/src/chatlore/web/index.html index cb81526..15281ec 100644 --- a/src/chatlore/web/index.html +++ b/src/chatlore/web/index.html @@ -43,6 +43,10 @@ Graph +
    @@ -182,6 +186,41 @@

    + +
    +
    +

    Your data

    +

    +
    + +
    +
    + + + +

    Download

    +

    Everything in this library, to import into ChatLore anywhere, or as Markdown files to read.

    +
    + ChatLore archive + Markdown + +
    +
    +     + diff --git a/src/chatlore/web/style.css b/src/chatlore/web/style.css index 7550c7a..2aeb55e 100644 --- a/src/chatlore/web/style.css +++ b/src/chatlore/web/style.css @@ -387,6 +387,26 @@ button.chip:hover { border-color: var(--accent); color: var(--accent-text); back .sheet-body li { margin: 4px 0; } .sheet-actions { display: flex; gap: 8px; margin-top: 18px; } +/* -- your data ---------------------------------------------------------------------- */ + +a.button { display: inline-flex; align-items: center; padding: 8px 14px; border: 1px solid var(--border); border-radius: var(--radius-sm); background: var(--surface); color: var(--text); text-decoration: none; font-size: 14px; } +a.button:hover { border-color: var(--border-strong); background: var(--surface-2); } +button.danger { margin-left: auto; color: var(--danger); } +button.danger:hover { border-color: var(--danger); } +.dropzone { display: grid; justify-items: center; gap: 6px; padding: 26px 20px; border: 1.5px dashed var(--border-strong); border-radius: var(--radius); text-align: center; cursor: pointer; color: var(--text-2); font-size: 13px; } +.dropzone:hover, .dropzone.over { border-color: var(--accent); background: var(--accent-soft); } +.dropzone.busy { pointer-events: none; opacity: .5; } +.dropzone svg { width: 28px; height: 28px; fill: none; stroke: var(--accent); stroke-width: 1.8; stroke-linecap: round; stroke-linejoin: round; } +.dropzone strong { color: var(--text); font-size: 14px; } +.dropzone input { position: absolute; width: 1px; height: 1px; opacity: 0; } +.data-note { margin: 12px 0 0; font-size: 12.5px; line-height: 1.5; color: var(--muted); } +.data-progress { margin-top: 16px; padding: 14px; border: 1px solid var(--border); border-radius: var(--radius); background: var(--bg); } +.data-stage { display: flex; justify-content: space-between; gap: 12px; font-size: 14px; font-weight: 550; } +.data-progress progress { width: 100%; height: 6px; margin: 10px 0 6px; accent-color: var(--accent); } +.data-progress p { margin: 0; font-size: 13px; } +.data-progress ul { margin: 8px 0 0; font-size: 13px; } +.data-help { margin: 0; font-size: 13px; } + .chat { display: flex; flex-direction: column; gap: 12px; background: var(--bg); } .bubble { max-width: 88%; padding: 10px 14px; border-radius: 16px; background: var(--surface); border: 1px solid var(--border); color: var(--text-2); overflow-wrap: anywhere; } .bubble.user { align-self: flex-end; background: var(--accent-soft); border-color: transparent; color: var(--text); border-bottom-right-radius: 4px; } diff --git a/tests/test_uploads.py b/tests/test_uploads.py new file mode 100644 index 0000000..577a888 --- /dev/null +++ b/tests/test_uploads.py @@ -0,0 +1,357 @@ +"""Tests for uploading, exporting, and visitors' own libraries.""" + +from __future__ import annotations + +import io +import threading +import time +import zipfile +from collections.abc import Iterator +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import pytest +from fastapi.testclient import TestClient +from typer.testing import CliRunner + +from chatlore.api import create_app +from chatlore.archive import export_archive, read_manifest +from chatlore.cli import app as cli +from chatlore.llm import LLMError +from chatlore.spaces import COOKIE, Spaces +from tests.conftest import make_zip +from tests.fakes import FakeEmbedder, FakeLLM + +runner = CliRunner() +CHANGE = {"X-ChatLore": "1"} + + +@pytest.fixture +def home(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: + """A small library to serve, with its graph built.""" + target = tmp_path / "home" + monkeypatch.setenv("CHATLORE_HOME", str(target)) + monkeypatch.setenv("COLUMNS", "120") + for module in ("cli", "api", "mcp_server"): + monkeypatch.setattr(f"chatlore.{module}.make_embedder", lambda home: FakeEmbedder()) + fake = FakeLLM() + monkeypatch.setattr("chatlore.cli.make_llm", lambda: fake) + monkeypatch.setattr("chatlore.api.make_llm", lambda: FakeLLM()) + runner.invoke(cli, ["note", "Lisbon and Sintra make a good weekend.", "--title", "Weekend"]) + runner.invoke(cli, ["process"]) + runner.invoke(cli, ["extract"]) + return target + + +@pytest.fixture +def private(home: Path) -> Iterator[TestClient]: + with TestClient(create_app(home)) as client: + yield client + + +@pytest.fixture +def spaces(tmp_path: Path) -> Spaces: + return Spaces(tmp_path / "spaces") + + +@pytest.fixture +def public(home: Path, spaces: Spaces) -> Iterator[TestClient]: + with TestClient(create_app(home, public=True, spaces=spaces)) as client: + yield client + + +def upload(client: TestClient, path: Path, name: str | None = None) -> Any: + return client.post( + "/library/import", + content=path.read_bytes(), + headers={**CHANGE, "X-Filename": name or path.name}, + ) + + +def finished(client: TestClient) -> dict[str, Any]: + """The import's status once it stops.""" + for _ in range(400): + status = client.get("/library").json()["import"] + if status is not None and not status["running"]: + return dict(status) + time.sleep(0.025) + raise AssertionError("the import did not finish") + + +def test_an_upload_is_imported_embedded_and_read_into_the_graph( + private: TestClient, home: Path, fixtures: Path +) -> None: + before = private.get("/stats").json() + + started = upload(private, fixtures / "claude" / "conversations.json") + status = finished(private) + after = private.get("/stats").json() + + assert started.status_code == 202 + assert started.json()["file"] == "conversations.json" + assert status["stage"] == "done", status + assert status["conversations"] == 3 + assert after["conversations"] == before["conversations"] + 3 + assert after["chunks"] == after["embeddings"] > before["chunks"] + assert after["entities"] > before["entities"] + assert list((home / "uploads").iterdir()) == [] # the upload is not kept + + +def test_the_server_says_whether_it_takes_uploads(private: TestClient, home: Path) -> None: + info = private.get("/library").json() + with TestClient(create_app(home, public=True)) as demo: + closed = demo.get("/library").json() + refused = demo.post("/library/import", content=b"x", headers=CHANGE) + + assert info["own"] is True and info["uploads"] is True and info["import"] is None + assert private.get("/health").json()["uploads"] is True + assert closed["uploads"] is False and closed["own"] is False + assert refused.status_code == 403 + + +def test_changes_need_the_interface_header(private: TestClient, fixtures: Path) -> None: + path = fixtures / "claude" / "conversations.json" + + without = private.post("/library/import", content=path.read_bytes()) + delete = private.delete("/library") + + assert without.status_code == 403 + assert "X-ChatLore" in without.json()["detail"] + assert delete.status_code == 403 + + +def test_empty_and_oversized_uploads_are_refused(home: Path, fixtures: Path) -> None: + with TestClient(create_app(home, max_upload=100)) as client: + empty = client.post("/library/import", content=b"", headers=CHANGE) + large = upload(client, fixtures / "claude" / "conversations.json") + + assert empty.status_code == 400 + assert large.status_code == 413 + assert list((home / "uploads").iterdir()) == [] + + +def test_an_unreadable_upload_fails_with_a_reason(private: TestClient, tmp_path: Path) -> None: + path = tmp_path / "photo.zip" + make_zip(path, {"photo.jpg": "not an export"}) + + upload(private, path) + status = finished(private) + + assert status["stage"] == "failed" + assert "could not recognise" in status["error"] + + +def test_a_zip_of_markdown_notes_and_an_archive_are_imported( + private: TestClient, home: Path, fixtures: Path, tmp_path: Path +) -> None: + vault = fixtures / "markdown" / "vault" + notes = make_zip( + tmp_path / "vault.zip", + {str(path.relative_to(vault)): path for path in vault.rglob("*") if path.is_file()}, + ) + upload(private, notes) + from_notes = finished(private) + + other = tmp_path / "other" + export_archive(home, tmp_path / "library.zip") + with TestClient(create_app(other)) as client: + upload(client, tmp_path / "library.zip") + from_archive = finished(client) + stats = client.get("/stats").json() + + assert from_notes["stage"] == "done" and from_notes["conversations"] > 0 + assert from_archive["stage"] == "done", from_archive + assert stats["conversations"] == from_notes["conversations"] + 1 + assert stats["entities"] > 0 + + +def test_without_a_model_key_everything_but_the_graph_is_built( + home: Path, fixtures: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def no_key() -> FakeLLM: + raise LLMError("no API key for OpenRouter") + + monkeypatch.setattr("chatlore.api.make_llm", no_key) + with TestClient(create_app(home)) as client: + upload(client, fixtures / "claude" / "conversations.json") + status = finished(client) + stats = client.get("/stats").json() + + assert status["stage"] == "done" + assert status["notes"] == ["The knowledge graph was not built: no API key for OpenRouter"] + assert stats["chunks"] == stats["embeddings"] + + +def test_the_model_reads_only_as_many_passages_as_allowed(home: Path, fixtures: Path) -> None: + with TestClient(create_app(home, extract_limit=1)) as client: + upload(client, fixtures / "claude" / "conversations.json") + status = finished(client) + + assert status["stage"] == "done" + assert status["notes"][0].startswith("The model read 1 of ") + + +def test_the_library_downloads_as_an_archive_or_markdown(private: TestClient) -> None: + archive = private.get("/library/export") + markdown = private.get("/library/export", params={"format": "markdown"}) + + assert archive.status_code == 200 + assert "chatlore-library.zip" in archive.headers["content-disposition"] + with zipfile.ZipFile(io.BytesIO(archive.content)) as zipped: + assert {"manifest.json", "conversations.jsonl", "graph.jsonl"} <= set(zipped.namelist()) + with zipfile.ZipFile(io.BytesIO(markdown.content)) as zipped: + names = zipped.namelist() + assert any(name.endswith("Weekend.md") for name in names) + assert private.get("/library/export", params={"format": "pdf"}).status_code == 422 + + +def test_an_empty_library_has_nothing_to_download(tmp_path: Path) -> None: + with TestClient(create_app(tmp_path / "empty")) as client: + assert client.get("/library/export").status_code == 404 + + +def test_each_visitor_gets_a_private_library( + public: TestClient, home: Path, spaces: Spaces, fixtures: Path +) -> None: + demo = public.get("/stats").json() + + started = upload(public, fixtures / "claude" / "conversations.json") + status = finished(public) + own = public.get("/stats").json() + info = public.get("/library").json() + stranger = TestClient(public.app) # another browser, on the app already running + theirs = stranger.get("/stats").json() + stranger_info = stranger.get("/library").json() + + cookie = started.headers["set-cookie"] + assert status["stage"] == "done" + assert f"{COOKIE}=" in cookie and "HttpOnly" in cookie and "samesite=lax" in cookie.lower() + assert own["conversations"] == 3 # only the visitor's upload, not the demo + assert theirs == demo + assert info["own"] is True and info["expires_at"] is not None and info["keep_hours"] == 24 + assert stranger_info["own"] is False and stranger_info["import"] is None + homes = [path for path in spaces.root.iterdir()] + assert len(homes) == 1 + token = public.cookies[COOKIE] + assert token not in homes[0].name # the folder does not give the token away + + +def test_a_visitor_can_delete_their_library( + public: TestClient, spaces: Spaces, fixtures: Path +) -> None: + upload(public, fixtures / "claude" / "conversations.json") + finished(public) + + deleted = public.delete("/library", headers=CHANGE) + again = public.delete("/library", headers=CHANGE) + + assert deleted.status_code == 200 + assert list(spaces.root.iterdir()) == [] + assert again.status_code == 404 + assert public.get("/library").json()["own"] is False + + +def test_the_servers_own_library_cannot_be_deleted(private: TestClient) -> None: + assert private.delete("/library", headers=CHANGE).status_code == 403 + + +def test_deleting_stops_an_import_first( + home: Path, spaces: Spaces, fixtures: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + release = threading.Event() + + class Slow(FakeEmbedder): + def embed_passages(self, texts: Any) -> list[list[float]]: + release.wait(5) + return super().embed_passages(texts) + + monkeypatch.setattr("chatlore.api.make_embedder", lambda home: Slow()) + with TestClient(create_app(home, public=True, spaces=spaces)) as client: + upload(client, fixtures / "claude" / "conversations.json") + for _ in range(200): + if client.get("/library").json()["import"]["stage"] == "embedding": + break + time.sleep(0.02) + busy = upload(client, fixtures / "claude" / "conversations.json") + client.delete("/library", headers=CHANGE) + release.set() + for _ in range(200): + if not list(spaces.root.iterdir()): + break + time.sleep(0.02) + + assert busy.status_code == 409 + assert list(spaces.root.iterdir()) == [] + + +def test_libraries_expire(tmp_path: Path) -> None: + now = [datetime(2026, 9, 1, tzinfo=UTC)] + spaces = Spaces(tmp_path / "spaces", keep=timedelta(hours=24), clock=lambda: now[0]) + + token, space = spaces.create() + found = spaces.find(token) + now[0] += timedelta(hours=25) + + assert found == space + assert spaces.find(token) is None + assert spaces.find("not a token") is None + assert spaces.find(None) is None + assert spaces.expired() == [space] + (spaces.root / "leftover").mkdir() + assert [expired.id for expired in spaces.expired()] == [space.id, "leftover"] + spaces.delete(space) + assert not space.home.exists() + + +def test_expired_libraries_are_swept_away(home: Path, tmp_path: Path) -> None: + now = [datetime.now(UTC) - timedelta(hours=30)] + spaces = Spaces(tmp_path / "spaces", clock=lambda: now[0]) + _, old = spaces.create() + now[0] = datetime.now(UTC) + + with TestClient(create_app(home, public=True, spaces=spaces)): + for _ in range(200): + if not old.home.exists(): + break + time.sleep(0.02) + + assert not old.home.exists() + + +def test_serve_takes_uploads_on_request( + home: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + started: list[tuple[Any, dict[str, Any]]] = [] + monkeypatch.setattr("uvicorn.run", lambda app, **options: started.append((app, options))) + + result = runner.invoke( + cli, + [ + "serve", + "--public", + "--uploads", + "--keep-hours", + "2", + "--spaces-dir", + str(tmp_path / "v"), + ], + ) + + assert result.exit_code == 0, result.output + assert "each gets a private library" in " ".join(result.output.split()) + with TestClient(started[0][0]) as client: + info = client.get("/library").json() + assert info["uploads"] is True and info["keep_hours"] == 2 + + +def test_the_archive_downloaded_from_a_visitor_library_imports_anywhere( + public: TestClient, fixtures: Path, tmp_path: Path +) -> None: + upload(public, fixtures / "claude" / "conversations.json") + finished(public) + + download = public.get("/library/export") + (tmp_path / "mine.zip").write_bytes(download.content) + + assert read_manifest(tmp_path / "mine.zip")["conversations"] == 3