From 2f97be27387949709328d5f6e4630470080839e2 Mon Sep 17 00:00:00 2001 From: bndct-devops Date: Mon, 8 Jun 2026 21:31:26 +0200 Subject: [PATCH] feat(admin): bake metadata into source files on disk Add an admin "Bake to File" tool that writes Tome's metadata (title, author, series, description, cover, tags) directly into the library files on disk, complementing the existing lazy download-time cache. Useful when files are read outside Tome (Syncthing, Calibre, NAS). Runs as a single serial, cancellable background job with byte-weighted progress + ETA; it survives a tab close and the page reconnects to a running job on return. Per-file safety: validate the embed parses -> atomic tmp write + os.replace -> recompute sha256 -> set BookFile.metadata_synced_at. get_baked_path() serves the raw file once synced, so re-runs skip already-current files and a restart resumes. Disabled on read-only mounts; TOME_ALLOW_INFILE_BAKE is a hard off-switch. CBZ pages are stored uncompressed during embed to cut bake time on large comic archives. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 17 ++ backend/api/users.py | 72 +++++ backend/core/config.py | 5 + backend/main.py | 6 + backend/models/book.py | 6 + backend/services/bake_job.py | 262 +++++++++++++++++ backend/services/metadata_embed.py | 148 +++++++++- frontend/src/components/BakeToFile.tsx | 319 +++++++++++++++++++++ frontend/src/lib/bake.ts | 45 +++ frontend/src/pages/AdminPage.tsx | 5 +- tests/test_bake_to_file.py | 208 ++++++++++++++ website/src/pages/docs/admin.astro | 31 ++ website/src/pages/docs/changelog.astro | 11 + website/src/pages/docs/configuration.astro | 1 + 14 files changed, 1133 insertions(+), 3 deletions(-) create mode 100644 backend/services/bake_job.py create mode 100644 frontend/src/components/BakeToFile.tsx create mode 100644 frontend/src/lib/bake.ts create mode 100644 tests/test_bake_to_file.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b1f227..12942f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ All notable changes to Tome are documented here. Format loosely follows ## [Unreleased] ### Added +- **Bake metadata to files (admin).** A new **Admin → Bake to File** page writes + Tome's metadata — title, author, series, description, cover and tags — directly + into the source library files on disk, rather than only into the lazy + download-time cache. This makes the embedded metadata visible to tools that read + the files outside Tome (Syncthing, a Calibre library pointed at the same folder, + direct NAS browsing). It runs as a single background job with a live, + byte-weighted progress bar, current-file display, baked/skipped/failed counters + and an ETA; closing the tab or navigating away does not stop it, and reopening + the page reconnects to the run in progress. When it finishes you get a summary + screen listing any files that were skipped or failed. The action is destructive + and irreversible (it recomputes each file's content hash), so it is admin-only, + confirm-gated, and disabled on read-only library mounts. Files already carrying + current metadata are skipped, so re-running is cheap. EPUB/CBZ get full metadata + plus cover (CBZ pages are now stored uncompressed during embedding, cutting bake + time on large comic archives); PDF gets title/author/subject. New env var + `TOME_ALLOW_INFILE_BAKE` (default `true`) is a hard off-switch for operators who + never want their files mutated. - **Send to KOReader (beta).** Queue a book from the web straight to your e-reader — no email, no Amazon Send-to-Kindle. It's the KOReader-native counterpart to email send-to-device: the original EPUB/CBZ arrives in your diff --git a/backend/api/users.py b/backend/api/users.py index 02a6b70..acab22a 100644 --- a/backend/api/users.py +++ b/backend/api/users.py @@ -830,3 +830,75 @@ def clear_covers_cache( os.remove(fp) deleted += 1 return {"deleted": deleted} + + +# ── Bake metadata to file (admin) ────────────────────────────────────────────── +# Whole-library, in-place rewrite of source files with Tome's metadata. Runs as a +# single background job; the UI polls /admin/bake/status. See services/bake_job.py. + +@router.get("/admin/bake/status") +def bake_status(current_user: User = Depends(get_current_user)) -> dict: + require_role(current_user, "admin") + from backend.services import bake_job + from backend.services.metadata_embed import library_writable + from backend.core.config import settings + return { + **bake_job.get_status(), + "library_writable": library_writable(), + "enabled": settings.allow_infile_bake, + } + + +@router.get("/admin/bake/preflight") +def bake_preflight(current_user: User = Depends(get_current_user)) -> dict: + require_role(current_user, "admin") + from backend.services import bake_job + from backend.services.metadata_embed import library_writable + from backend.core.config import settings + return { + **bake_job.preflight(), + "library_writable": library_writable(), + "enabled": settings.allow_infile_bake, + } + + +@router.post("/admin/bake/start") +def bake_start( + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +) -> dict: + require_role(current_user, "admin") + from backend.services import bake_job + from backend.services.metadata_embed import library_writable + from backend.core.config import settings + if not settings.allow_infile_bake: + raise HTTPException(403, "In-file baking is disabled (TOME_ALLOW_INFILE_BAKE=false)") + if not library_writable(): + raise HTTPException(409, "Library directory is read-only") + try: + state = bake_job.start(username=current_user.username) + except bake_job.BakeAlreadyRunning: + raise HTTPException(409, "A bake is already running") + audit( + db, + "books.metadata_baked_started", + user_id=current_user.id, + username=current_user.username, + details={"total_files": state.get("total_files"), "total_bytes": state.get("total_bytes")}, + ) + return state + + +@router.post("/admin/bake/cancel") +def bake_cancel(current_user: User = Depends(get_current_user)) -> dict: + require_role(current_user, "admin") + from backend.services import bake_job + cancelled = bake_job.request_cancel() + return {"cancelling": cancelled, **bake_job.get_status()} + + +@router.post("/admin/bake/dismiss") +def bake_dismiss(current_user: User = Depends(get_current_user)) -> dict: + require_role(current_user, "admin") + from backend.services import bake_job + return bake_job.dismiss() diff --git a/backend/core/config.py b/backend/core/config.py index e786f60..3d26b27 100644 --- a/backend/core/config.py +++ b/backend/core/config.py @@ -64,6 +64,11 @@ class Settings(BaseSettings): # switch thereafter. send_to_koreader: bool = False + # Bake-to-file: write Tome's metadata into the source library files on disk + # (vs the lazy download cache). Destructive + admin-gated. Hard off-switch for + # operators who never want their files mutated. env TOME_ALLOW_INFILE_BAKE. + allow_infile_bake: bool = True + # JWT settings jwt_algorithm: str = "HS256" jwt_expire_minutes: int = 60 * 24 * 7 # 7 days diff --git a/backend/main.py b/backend/main.py index 3c56a84..6c14475 100644 --- a/backend/main.py +++ b/backend/main.py @@ -130,6 +130,12 @@ async def lifespan(app: FastAPI): if "scope" not in at_cols: conn.execute(text("ALTER TABLE api_tokens ADD COLUMN scope VARCHAR(16) NOT NULL DEFAULT 'full'")) conn.commit() + # Bake-to-file: track when a file's on-disk bytes were last written with + # Tome's metadata. NULL for every pre-existing row (= never baked in place). + bf_cols = {r[1] for r in conn.execute(text("PRAGMA table_info(book_files)")).fetchall()} + if "metadata_synced_at" not in bf_cols: + conn.execute(text("ALTER TABLE book_files ADD COLUMN metadata_synced_at DATETIME")) + conn.commit() init_fts(engine) backfill_fts(engine) settings.ensure_dirs() diff --git a/backend/models/book.py b/backend/models/book.py index 016938f..7b0514c 100644 --- a/backend/models/book.py +++ b/backend/models/book.py @@ -66,6 +66,12 @@ class BookFile(Base): file_size: Mapped[Optional[int]] = mapped_column(Integer) content_hash: Mapped[Optional[str]] = mapped_column(String(64), index=True) added_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, nullable=False) + # Set to the owning Book.updated_at at the moment this file was baked to disk + # (in-file metadata write). When it equals Book.updated_at, the bytes on disk + # already carry the current metadata → download path can serve the raw file + # and skip the lazy bake. Goes stale (≠ updated_at) the next time metadata is + # edited, transparently re-enabling lazy baking. NULL = never baked in place. + metadata_synced_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) book: Mapped["Book"] = relationship("Book", back_populates="files") diff --git a/backend/services/bake_job.py b/backend/services/bake_job.py new file mode 100644 index 0000000..07bb717 --- /dev/null +++ b/backend/services/bake_job.py @@ -0,0 +1,262 @@ +"""Whole-library 'bake metadata to file' background job. + +A single, serial, cancellable job rewrites the source files on disk with Tome's +metadata (see ``metadata_embed.bake_to_file``). It runs in a daemon thread so it +survives the request that started it — closing the browser tab or navigating +away never cancels it. Progress lives in module-global state (guarded by a lock) +and is read back via the status endpoint, so the UI is just a poller and can +reconnect at any time. + +Single-process model: state is in-memory, so a server restart drops the job and +its progress. That is safe and cheap to recover from — ``metadata_synced_at`` +makes a re-run skip everything already baked, so the admin just starts it again +and it resumes where it left off. (Tome runs a single uvicorn worker; SQLite is +single-writer, so there is no multi-process state to reconcile.) + +Progress is weighted by *bytes*, not file count: a 300 MB CBZ costs ~150× an +EPUB, so a file-count bar/ETA would lie. We sum file sizes up front and track +bytes processed; the ETA is ``elapsed * bytes_left / bytes_done``. +""" +from __future__ import annotations + +import logging +import threading +import time +from typing import Optional + +from backend.core.database import SessionLocal +from backend.models.book import Book, BookFile +from backend.services.metadata_embed import BAKEABLE_FORMATS, bake_to_file + +log = logging.getLogger(__name__) + +# Cap on how many per-file issues (skipped/failed) we retain for the end screen. +_MAX_ISSUES = 500 + +_lock = threading.Lock() +_cancel = threading.Event() +_thread: Optional[threading.Thread] = None + +# The single job's state. ``status`` is one of: +# idle | running | done | cancelled | error +_state: dict = {"status": "idle"} + + +class BakeAlreadyRunning(Exception): + pass + + +def _needs_bake(synced, updated) -> bool: + return not (synced is not None and updated is not None and synced == updated) + + +def preflight() -> dict: + """Cheap, DB-only summary of what a run would do, using stored file_size + (no disk stat). Safe to call on page load.""" + with SessionLocal() as db: + rows = ( + db.query( + BookFile.file_size, + BookFile.format, + BookFile.metadata_synced_at, + Book.updated_at, + ) + .join(Book, BookFile.book_id == Book.id) + .all() + ) + bakeable = already = pending = 0 + pending_bytes = 0 + for size, fmt, synced, updated in rows: + if (fmt or "").lower() not in BAKEABLE_FORMATS: + continue + bakeable += 1 + if _needs_bake(synced, updated): + pending += 1 + pending_bytes += size or 0 + else: + already += 1 + return { + "bakeable_total": bakeable, + "already_current": already, + "pending": pending, + "pending_bytes": pending_bytes, + } + + +def _snapshot() -> dict: + """A JSON-able copy of state with derived elapsed/ETA filled in.""" + s = dict(_state) + if s.get("status") == "running" and s.get("started_at"): + elapsed = time.time() - s["started_at"] + elif s.get("started_at") and s.get("finished_at"): + elapsed = s["finished_at"] - s["started_at"] + else: + elapsed = 0.0 + s["elapsed_seconds"] = round(elapsed, 1) + + eta = None + if s.get("status") == "running": + done_bytes = s.get("done_bytes", 0) + total_bytes = s.get("total_bytes", 0) + if done_bytes > 0 and total_bytes > done_bytes: + eta = elapsed * (total_bytes - done_bytes) / done_bytes + s["eta_seconds"] = round(eta, 1) if eta is not None else None + return s + + +def get_status() -> dict: + with _lock: + return _snapshot() + + +def dismiss() -> dict: + """Clear a finished run's summary so the page resets to idle. No-op while + a run is in progress.""" + global _state + with _lock: + if _state.get("status") != "running": + _state = {"status": "idle"} + return _snapshot() + + +def request_cancel() -> bool: + with _lock: + if _state.get("status") == "running": + _cancel.set() + return True + return False + + +def start(*, username: Optional[str] = None) -> dict: + """Begin a whole-library bake. Raises BakeAlreadyRunning if one is active.""" + global _state, _thread + with _lock: + if _state.get("status") == "running": + raise BakeAlreadyRunning() + + # Build the work list now (file_id, book_id, size) for files that need it. + with SessionLocal() as db: + rows = ( + db.query( + BookFile.id, + BookFile.book_id, + BookFile.file_size, + BookFile.format, + BookFile.metadata_synced_at, + Book.updated_at, + ) + .join(Book, BookFile.book_id == Book.id) + .all() + ) + work: list[tuple[int, int, int]] = [] + total_bytes = 0 + for file_id, book_id, size, fmt, synced, updated in rows: + if (fmt or "").lower() not in BAKEABLE_FORMATS: + continue + if _needs_bake(synced, updated): + size = size or 0 + work.append((file_id, book_id, size)) + total_bytes += size + + _cancel.clear() + _state = { + "status": "running", + "started_at": time.time(), + "finished_at": None, + "triggered_by": username, + "total_files": len(work), + "total_bytes": total_bytes, + "done_files": 0, + "done_bytes": 0, + "baked": 0, + "skipped": 0, + "failed": 0, + "current_file": None, + "issues": [], + "error": None, + } + _thread = threading.Thread( + target=_run, args=(work,), name="bake-to-file", daemon=True + ) + _thread.start() + return _snapshot() + + +def _set_current(path: str) -> None: + with _lock: + _state["current_file"] = path + + +def _record(result, size: int) -> None: + with _lock: + _state["done_files"] += 1 + _state["done_bytes"] += size + st = result.status + if st == "baked": + _state["baked"] += 1 + elif st == "failed": + _state["failed"] += 1 + else: # skipped | readonly + _state["skipped"] += 1 + if st != "baked" and len(_state["issues"]) < _MAX_ISSUES: + _state["issues"].append( + {"path": result.file_path, "status": st, "reason": result.reason} + ) + + +def _finish(status: str, error: Optional[str] = None) -> None: + with _lock: + _state["status"] = status + _state["finished_at"] = time.time() + _state["current_file"] = None + if error: + _state["error"] = error + + +def _run(work: list[tuple[int, int, int]]) -> None: + try: + for file_id, book_id, size in work: + if _cancel.is_set(): + _finish("cancelled") + return + try: + with SessionLocal() as db: + bf = db.get(BookFile, file_id) + if bf is None: + _record(_Missing(f"file #{file_id}"), size) + continue + _set_current(bf.file_path) + book = db.get(Book, book_id) + if book is None: + _record(_Missing(bf.file_path), size) + continue + result = bake_to_file(book, bf) + if result.ok: + db.commit() + else: + db.rollback() + _record(result, size) + except Exception as e: # noqa: BLE001 — one bad file never aborts the run + log.exception("bake: unexpected error on file #%s", file_id) + _record(_Failed(str(file_id), str(e)), size) + _finish("done") + except Exception as e: # noqa: BLE001 + log.exception("bake: job crashed") + _finish("error", error=str(e)) + + +# Lightweight stand-ins so _record can treat every outcome uniformly. +class _Missing: + status = "skipped" + reason = "file missing" + + def __init__(self, path: str): + self.file_path = path + + +class _Failed: + status = "failed" + + def __init__(self, path: str, reason: str): + self.file_path = path + self.reason = reason diff --git a/backend/services/metadata_embed.py b/backend/services/metadata_embed.py index fd48893..7137a67 100644 --- a/backend/services/metadata_embed.py +++ b/backend/services/metadata_embed.py @@ -12,18 +12,36 @@ import io import logging +import os import shutil import zipfile +from dataclasses import dataclass +from datetime import datetime from pathlib import Path from typing import Optional from xml.etree import ElementTree as ET from backend.core.config import settings from backend.models.book import Book, BookFile +from backend.services.metadata import sha256_file from backend.services.xml_ns import namespaces log = logging.getLogger(__name__) +# Formats whose bytes we can rewrite with embedded metadata. Everything else is +# served/left as-is. +BAKEABLE_FORMATS = {"epub", "cbz", "pdf"} + +# Image entries are already compressed; re-deflating them is pure CPU cost for +# zero size win (and, for big CBZ archives, the bulk of the bake time). Store +# them uncompressed instead. Text-ish entries (XHTML, CSS, XML) still deflate. +_STORED_EXTS = {".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".avif", ".jxl"} + + +def _compress_for(name: str) -> int: + ext = Path(name).suffix.lower() + return zipfile.ZIP_STORED if ext in _STORED_EXTS else zipfile.ZIP_DEFLATED + OPF_NS = "http://www.idpf.org/2007/opf" DC_NS = "http://purl.org/dc/elements/1.1/" CONTAINER_NS = "urn:oasis:names:tc:opendocument:xmlns:container" @@ -75,6 +93,15 @@ def get_baked_path(book: Book, book_file: BookFile) -> Path: if not src.exists(): return src + # If the on-disk file was already baked in place for this exact metadata + # revision, its bytes are current — serve the raw file, skip embed + cache. + if ( + book_file.metadata_synced_at is not None + and book.updated_at is not None + and book_file.metadata_synced_at == book.updated_at + ): + return src + key = _cache_key(book) baked = _baked_dir() / f"{key}.{book_file.format}" if baked.exists() and baked.stat().st_size > 0: @@ -308,7 +335,7 @@ def _embed_cbz(book: Book, src: Path, cover_bytes: Optional[bytes]) -> Optional[ buf = io.BytesIO() with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zout: if cover_bytes is not None: - zout.writestr(COVER_NAME, cover_bytes) + zout.writestr(COVER_NAME, cover_bytes, compress_type=zipfile.ZIP_STORED) for n in names: # Skip the old ComicInfo — we rewrite it below if n.lower() == "comicinfo.xml": @@ -316,7 +343,8 @@ def _embed_cbz(book: Book, src: Path, cover_bytes: Optional[bytes]) -> Optional[ # Skip the slot we're writing the Tome cover into (if any) if cover_bytes is not None and n == COVER_NAME: continue - zout.writestr(n, zf.read(n)) + # Pages are already-compressed images → store, don't re-deflate. + zout.writestr(n, zf.read(n), compress_type=_compress_for(n)) zout.writestr(COMIC_INFO_NAME, comic_info) return buf.getvalue() @@ -377,3 +405,119 @@ def _embed_pdf(book: Book, src: Path) -> Optional[bytes]: finally: doc.close() return buf + + +# ── In-file bake (write metadata into the source file on disk) ───────────────── +# +# Unlike get_baked_path (which writes a disposable cache copy), bake_to_file +# mutates the real library file. It is destructive + irreversible, so the heavy +# lifting here is safety: validate the freshly-embedded bytes parse, write to a +# sibling tmp, fsync, then atomically os.replace the original. A buggy embed must +# never clobber a good source. + +@dataclass +class BakeFileResult: + """Outcome of baking one BookFile in place.""" + file_path: str + status: str # "baked" | "skipped" | "readonly" | "failed" + reason: Optional[str] = None + old_hash: Optional[str] = None + new_hash: Optional[str] = None + + @property + def ok(self) -> bool: + return self.status == "baked" + + +def library_writable() -> bool: + """Is the library root writable? Used to gate the bake UI on a `:ro` mount.""" + try: + return os.access(settings.library_dir, os.W_OK) + except OSError: + return False + + +def _validate_baked(out: bytes, fmt: str) -> bool: + """Cheap sanity check that the embedded bytes are a usable file of `fmt` + before we let them replace the original.""" + fmt = (fmt or "").lower() + if fmt in ("epub", "cbz"): + try: + with zipfile.ZipFile(io.BytesIO(out), "r") as zf: + if zf.testzip() is not None: + return False + if fmt == "epub" and "META-INF/container.xml" not in zf.namelist(): + return False + return True + except (zipfile.BadZipFile, OSError): + return False + if fmt == "pdf": + return out[:5] == b"%PDF-" + return False + + +def bake_to_file(book: Book, book_file: BookFile) -> BakeFileResult: + """Write Tome's metadata into ``book_file`` on disk, in place. + + Mutates ``book_file.content_hash`` + ``book_file.metadata_synced_at`` on the + attached ORM objects **without committing** — the caller owns the transaction. + Per-file isolated: any failure leaves the original file untouched, never + leaks a ``.tmp``, and returns a non-``baked`` status rather than raising. + """ + fmt = (book_file.format or "").lower() + src = Path(book_file.file_path) + + if fmt not in BAKEABLE_FORMATS: + return BakeFileResult(str(src), "skipped", reason="unsupported format") + if not src.exists(): + return BakeFileResult(str(src), "skipped", reason="file missing") + if not os.access(src.parent, os.W_OK): + return BakeFileResult(str(src), "readonly", reason="directory not writable") + + try: + cover_bytes = _load_cover(book) + out = _embed(book, src, fmt, cover_bytes) + except Exception as e: # noqa: BLE001 — embed must never abort a bulk run + log.warning("bake: embed failed for book_id=%s file=%s: %s", book.id, src.name, e) + return BakeFileResult(str(src), "failed", reason=f"embed error: {e}") + + if out is None: + # _embed returns None for formats it can't handle (e.g. PDF without + # PyMuPDF, or a malformed EPUB with no OPF). + return BakeFileResult(str(src), "skipped", reason="nothing to embed") + + if not _validate_baked(out, fmt): + return BakeFileResult(str(src), "failed", reason="validation failed — original kept") + + old_hash = book_file.content_hash + tmp = src.with_suffix(src.suffix + ".bake.tmp") + try: + with open(tmp, "wb") as f: + f.write(out) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, src) # atomic within the same directory + except OSError as e: + log.warning("bake: write failed for book_id=%s file=%s: %s", book.id, src.name, e) + try: + if tmp.exists(): + tmp.unlink() + except OSError: + pass + return BakeFileResult(str(src), "failed", reason=f"write error: {e}") + + new_hash = sha256_file(src) + book_file.content_hash = new_hash + book_file.metadata_synced_at = book.updated_at + try: + book_file.file_size = src.stat().st_size + except OSError: + pass + + # The lazy cache for this book is now redundant. + try: + _purge_stale(book.id) + except OSError: + pass + + return BakeFileResult(str(src), "baked", old_hash=old_hash, new_hash=new_hash) diff --git a/frontend/src/components/BakeToFile.tsx b/frontend/src/components/BakeToFile.tsx new file mode 100644 index 0000000..94f3743 --- /dev/null +++ b/frontend/src/components/BakeToFile.tsx @@ -0,0 +1,319 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import { + HardDriveDownload, AlertTriangle, Loader2, Check, X, + FileWarning, Play, Ban, RotateCcw, CircleSlash, +} from 'lucide-react' +import { cn } from '@/lib/utils' +import { + getBakeStatus, getBakePreflight, startBake, cancelBake, dismissBake, + type BakeState, type BakePreflight, +} from '@/lib/bake' + +const POLL_MS = 1500 + +function formatBytes(n: number): string { + if (!n) return '0 B' + const units = ['B', 'KB', 'MB', 'GB', 'TB'] + let i = 0 + let v = n + while (v >= 1024 && i < units.length - 1) { v /= 1024; i++ } + return `${v.toFixed(i === 0 ? 0 : 1)} ${units[i]}` +} + +function formatDuration(secs: number | null): string { + if (secs == null) return '—' + const s = Math.max(0, Math.round(secs)) + if (s < 60) return `${s}s` + const m = Math.floor(s / 60) + const rem = s % 60 + if (m < 60) return `${m}m ${rem}s` + const h = Math.floor(m / 60) + return `${h}h ${m % 60}m` +} + +const TERMINAL = new Set(['done', 'cancelled', 'error']) + +export function BakeToFileTab() { + const [preflight, setPreflight] = useState(null) + const [state, setState] = useState(null) + const [confirming, setConfirming] = useState(false) + const [busy, setBusy] = useState(false) + const [error, setError] = useState(null) + const pollRef = useRef | null>(null) + + const stopPolling = useCallback(() => { + if (pollRef.current) { clearInterval(pollRef.current); pollRef.current = null } + }, []) + + const poll = useCallback(async () => { + try { + const s = await getBakeStatus() + setState(s) + if (TERMINAL.has(s.status) || s.status === 'idle') { + stopPolling() + // refresh the pending/already-current counts after a run + getBakePreflight().then(setPreflight).catch(() => {}) + } + } catch { /* keep last state */ } + }, [stopPolling]) + + const startPolling = useCallback(() => { + stopPolling() + pollRef.current = setInterval(poll, POLL_MS) + }, [poll, stopPolling]) + + // On mount: load preflight + current status; reconnect to a running job. + useEffect(() => { + let alive = true + Promise.all([getBakePreflight(), getBakeStatus()]) + .then(([pf, s]) => { + if (!alive) return + setPreflight(pf) + setState(s) + if (s.status === 'running') startPolling() + }) + .catch(() => { if (alive) setError('Failed to load bake status') }) + return () => { alive = false; stopPolling() } + }, [startPolling, stopPolling]) + + async function handleStart() { + setBusy(true) + setError(null) + try { + const s = await startBake() + setState(s) + setConfirming(false) + startPolling() + } catch (e) { + setError(e instanceof Error ? e.message : 'Failed to start') + } finally { + setBusy(false) + } + } + + async function handleCancel() { + setBusy(true) + try { + const s = await cancelBake() + setState(s) + } catch { /* ignore */ } finally { setBusy(false) } + } + + async function handleDismiss() { + setBusy(true) + try { + const s = await dismissBake() + setState(s) + getBakePreflight().then(setPreflight).catch(() => {}) + } catch { /* ignore */ } finally { setBusy(false) } + } + + const running = state?.status === 'running' + const terminal = state ? TERMINAL.has(state.status) : false + const writable = preflight?.library_writable ?? state?.library_writable ?? true + const enabled = preflight?.enabled ?? state?.enabled ?? true + const pending = preflight?.pending ?? 0 + const pct = state && state.total_bytes > 0 + ? Math.min(100, (state.done_bytes / state.total_bytes) * 100) + : 0 + + return ( +
+
+
+ +
+
+

Bake metadata to files

+

+ Write Tome's metadata (title, author, series, description, cover, tags) directly + into the source files on disk. Useful when files are read outside Tome — Syncthing, + a Calibre library on the same folder, or direct NAS browsing. +

+
+
+ + {/* Caveat box */} +
+ +
+ This rewrites the actual library files and recomputes their content hash. + It cannot be undone. EPUB and CBZ get full + metadata + cover; PDF gets title/author/subject. Files already current are skipped. +
+
+ + {error && ( +
+ {error} + +
+ )} + + {!enabled && ( +
+ + In-file baking is disabled on this server (TOME_ALLOW_INFILE_BAKE=false). +
+ )} + {enabled && !writable && ( +
+ + The library directory is mounted read-only — files cannot be modified. +
+ )} + + {/* ── Idle: pre-flight + start ───────────────────────────────────────── */} + {!running && !terminal && ( +
+
+ + + 0} sub={preflight ? formatBytes(preflight.pending_bytes) : undefined} /> +
+
+ {confirming ? ( +
+

+ Rewrite {pending} file{pending !== 1 ? 's' : ''} on disk + {preflight ? <> ({formatBytes(preflight.pending_bytes)}) : null}? + This cannot be undone. +

+
+ + +
+
+ ) : ( + + )} +
+
+ )} + + {/* ── Running: progress ──────────────────────────────────────────────── */} + {running && state && ( +
+
+
+ + + Baking… {state.done_files} / {state.total_files} files + + {pct.toFixed(1)}% +
+
+
+
+
+ {formatBytes(state.done_bytes)} / {formatBytes(state.total_bytes)} + Elapsed {formatDuration(state.elapsed_seconds)} · ~{formatDuration(state.eta_seconds)} left +
+
+
+
+ Current:{' '} + {state.current_file?.split('/').pop() ?? '…'} +
+
+ + + 0} /> +
+ +
+
+ )} + + {/* ── Terminal: end summary ──────────────────────────────────────────── */} + {terminal && state && ( +
+
+ {state.status === 'done' && } + {state.status === 'cancelled' && } + {state.status === 'error' && } + + {state.status === 'done' && 'Bake complete'} + {state.status === 'cancelled' && 'Bake stopped'} + {state.status === 'error' && 'Bake failed'} + + Took {formatDuration(state.elapsed_seconds)} +
+
+ + + 0} /> +
+ {state.error && ( +
{state.error}
+ )} + {state.issues.length > 0 && ( +
+

+ Files not baked ({state.issues.length}) +

+
+ {state.issues.map((it, i) => ( +
+ + {it.status} + + {it.path.split('/').pop()} + {it.reason && {it.reason}} +
+ ))} +
+
+ )} +
+ + {pending > 0 && ( + + )} +
+
+ )} +
+ ) +} + +function Stat({ label, value, sub, accent, danger }: { + label: string; value: number | string; sub?: string; accent?: boolean; danger?: boolean +}) { + return ( +
+

+ {value} +

+

{label}

+ {sub &&

{sub}

} +
+ ) +} diff --git a/frontend/src/lib/bake.ts b/frontend/src/lib/bake.ts new file mode 100644 index 0000000..b8e66ed --- /dev/null +++ b/frontend/src/lib/bake.ts @@ -0,0 +1,45 @@ +import { api } from './api' + +export type BakeStatus = 'idle' | 'running' | 'done' | 'cancelled' | 'error' + +export interface BakeIssue { + path: string + status: 'skipped' | 'readonly' | 'failed' + reason: string | null +} + +export interface BakeState { + status: BakeStatus + started_at: number | null + finished_at: number | null + triggered_by: string | null + total_files: number + total_bytes: number + done_files: number + done_bytes: number + baked: number + skipped: number + failed: number + current_file: string | null + issues: BakeIssue[] + error: string | null + elapsed_seconds: number + eta_seconds: number | null + library_writable: boolean + enabled: boolean +} + +export interface BakePreflight { + bakeable_total: number + already_current: number + pending: number + pending_bytes: number + library_writable: boolean + enabled: boolean +} + +export const getBakeStatus = () => api.get('/admin/bake/status') +export const getBakePreflight = () => api.get('/admin/bake/preflight') +export const startBake = () => api.post('/admin/bake/start') +export const cancelBake = () => api.post('/admin/bake/cancel') +export const dismissBake = () => api.post('/admin/bake/dismiss') diff --git a/frontend/src/pages/AdminPage.tsx b/frontend/src/pages/AdminPage.tsx index 12a416a..9c0a79b 100644 --- a/frontend/src/pages/AdminPage.tsx +++ b/frontend/src/pages/AdminPage.tsx @@ -11,6 +11,7 @@ import { import { DOCS, docsLink } from '@/lib/docs' import { MetadataManager } from '@/components/MetadataManager' import { LibraryHealthTab } from '@/components/LibraryHealth' +import { BakeToFileTab } from '@/components/BakeToFile' import { SeriesCoverageStrip } from '@/components/SeriesCoverageStrip' import { useAuth, isAdmin } from '@/contexts/AuthContext' import { api } from '@/lib/api' @@ -2003,7 +2004,7 @@ function WishlistTab() { // ── AdminPage ───────────────────────────────────────────────────────────── -type Tab = 'users' | 'scanner' | 'server' | 'types' | 'audit' | 'metadata' | 'library' | 'sync' | 'duplicates' | 'email' | 'wishlist' +type Tab = 'users' | 'scanner' | 'server' | 'types' | 'audit' | 'metadata' | 'library' | 'bake' | 'sync' | 'duplicates' | 'email' | 'wishlist' export function AdminPage() { const { user } = useAuth() @@ -2027,6 +2028,7 @@ export function AdminPage() { { id: 'audit', label: 'Audit Log' }, { id: 'metadata', label: 'Metadata' }, { id: 'library', label: 'Library' }, + { id: 'bake', label: 'Bake to File' }, { id: 'sync', label: 'Sync Status' }, { id: 'duplicates', label: 'Duplicates' }, { id: 'email', label: 'Email' }, @@ -2070,6 +2072,7 @@ export function AdminPage() { {tab === 'audit' && } {tab === 'metadata' && } {tab === 'library' && } + {tab === 'bake' && } {tab === 'sync' && } {tab === 'duplicates' && } {tab === 'email' && } diff --git a/tests/test_bake_to_file.py b/tests/test_bake_to_file.py new file mode 100644 index 0000000..d887457 --- /dev/null +++ b/tests/test_bake_to_file.py @@ -0,0 +1,208 @@ +"""Tests for in-file metadata baking (backend/services/metadata_embed.bake_to_file). + +The embed itself is covered by test_metadata_embed.py; this focuses on the +safety machinery: validate-before-replace, atomic write, hash recompute, +metadata_synced_at semantics, the read-only guard, and per-file failure +isolation (a bad embed must never clobber a good source). +""" +import zipfile +from datetime import timedelta + +import pytest + +from backend.services import metadata_embed as me +from backend.services.metadata_embed import ( + bake_to_file, get_baked_path, _compress_for, _validate_baked, +) +from backend.services.metadata import sha256_file + + +# ── builders ─────────────────────────────────────────────────────────────────── + +def _make_epub(path): + with zipfile.ZipFile(path, "w") as z: + z.writestr("mimetype", "application/epub+zip", compress_type=zipfile.ZIP_STORED) + z.writestr( + "META-INF/container.xml", + '' + '' + '', + ) + z.writestr( + "content.opf", + '' + '' + 'Original Title' + '' + '' + '', + ) + z.writestr("t.xhtml", "hi") + + +def _make_cbz(path): + with zipfile.ZipFile(path, "w") as z: + z.writestr("001.jpg", b"\xff\xd8\xff\xe0" + b"fakejpeg" * 50) + z.writestr("002.jpg", b"\xff\xd8\xff\xe0" + b"moredata" * 50) + + +# ── unit: helpers ──────────────────────────────────────────────────────────── + +def test_compress_for_images_stored(): + assert _compress_for("001.jpg") == zipfile.ZIP_STORED + assert _compress_for("cover.PNG") == zipfile.ZIP_STORED + assert _compress_for("page.xhtml") == zipfile.ZIP_DEFLATED + assert _compress_for("ComicInfo.xml") == zipfile.ZIP_DEFLATED + + +def test_validate_baked(): + assert _validate_baked(b"%PDF-1.7 ...", "pdf") is True + assert _validate_baked(b"not a pdf", "pdf") is False + assert _validate_baked(b"not a zip", "epub") is False + assert _validate_baked(b"not a zip", "cbz") is False + + +# ── bake: EPUB happy path ────────────────────────────────────────────────────── + +def test_bake_epub_rewrites_metadata_and_hash(db, make_book, tmp_path): + epub = tmp_path / "book.epub" + _make_epub(epub) + book = make_book(title="Baked Title", author="Jane Doe", series="My Series", + series_index=3, file_path=str(epub), file_format="epub", + content_hash="oldhash") + bf = book.files[0] + + result = bake_to_file(book, bf) + db.commit() + + assert result.status == "baked" + # File is still a valid EPUB carrying Tome's title. + with zipfile.ZipFile(epub, "r") as z: + assert z.testzip() is None + opf = z.read("content.opf").decode() + assert "Baked Title" in opf + assert "calibre:series" in opf and "My Series" in opf + # Hash recomputed to match the new bytes, and the file is marked synced. + assert bf.content_hash == sha256_file(epub) + assert bf.content_hash != "oldhash" + assert bf.metadata_synced_at == book.updated_at + + +# ── bake: CBZ stores pages uncompressed ──────────────────────────────────────── + +def test_bake_cbz_embeds_comicinfo_and_stores_images(db, make_book, tmp_path): + cbz = tmp_path / "comic.cbz" + _make_cbz(cbz) + book = make_book(title="Vol 1", series="Comic Series", series_index=1, + file_path=str(cbz), file_format="cbz") + bf = book.files[0] + + result = bake_to_file(book, bf) + db.commit() + + assert result.status == "baked" + with zipfile.ZipFile(cbz, "r") as z: + assert z.testzip() is None + names = z.namelist() + assert "ComicInfo.xml" in names + assert "Comic Series" in z.read("ComicInfo.xml").decode() + # Image pages must be stored, not re-deflated. + assert z.getinfo("001.jpg").compress_type == zipfile.ZIP_STORED + + +# ── get_baked_path early-return ──────────────────────────────────────────────── + +def test_get_baked_path_serves_raw_when_synced(db, make_book, tmp_path): + epub = tmp_path / "synced.epub" + _make_epub(epub) + book = make_book(title="Synced", file_path=str(epub), file_format="epub") + bf = book.files[0] + + bake_to_file(book, bf) + db.commit() + + # Already current → raw path, no cache copy. + assert get_baked_path(book, bf) == epub + + # Simulate a later metadata edit bumping updated_at → stale → lazy-bake again. + book.updated_at = book.updated_at + timedelta(seconds=5) + db.flush() + out = get_baked_path(book, bf) + assert out != epub + assert out.exists() + + +# ── safety: read-only directory ──────────────────────────────────────────────── + +def test_bake_readonly_dir_leaves_file_untouched(db, make_book, tmp_path, monkeypatch): + epub = tmp_path / "ro.epub" + _make_epub(epub) + before = epub.read_bytes() + book = make_book(title="RO", file_path=str(epub), file_format="epub") + bf = book.files[0] + bf.content_hash = "keepme" + db.flush() + + monkeypatch.setattr(me.os, "access", lambda p, mode: False) + result = bake_to_file(book, bf) + + assert result.status == "readonly" + assert epub.read_bytes() == before + assert bf.content_hash == "keepme" + assert bf.metadata_synced_at is None + + +# ── safety: corrupt embed must not replace the original ───────────────────────── + +def test_bake_corrupt_embed_fails_safe(db, make_book, tmp_path, monkeypatch): + epub = tmp_path / "corrupt.epub" + _make_epub(epub) + before = epub.read_bytes() + book = make_book(title="Corrupt", file_path=str(epub), file_format="epub") + bf = book.files[0] + + monkeypatch.setattr(me, "_embed", lambda *a, **k: b"not a real zip") + result = bake_to_file(book, bf) + + assert result.status == "failed" + assert "validation" in (result.reason or "") + assert epub.read_bytes() == before # original intact + assert not (tmp_path / "corrupt.epub.bake.tmp").exists() # no leftover tmp + assert bf.metadata_synced_at is None + + +def test_bake_embed_raises_fails_safe(db, make_book, tmp_path, monkeypatch): + epub = tmp_path / "boom.epub" + _make_epub(epub) + before = epub.read_bytes() + book = make_book(title="Boom", file_path=str(epub), file_format="epub") + bf = book.files[0] + + def _boom(*a, **k): + raise RuntimeError("kaboom") + monkeypatch.setattr(me, "_embed", _boom) + result = bake_to_file(book, bf) + + assert result.status == "failed" + assert epub.read_bytes() == before + + +# ── skips ─────────────────────────────────────────────────────────────────── + +def test_bake_unsupported_format_skipped(db, make_book, tmp_path): + mobi = tmp_path / "x.mobi" + mobi.write_bytes(b"whatever") + book = make_book(title="Mobi", file_path=str(mobi), file_format="mobi") + result = bake_to_file(book, book.files[0]) + assert result.status == "skipped" + assert "format" in (result.reason or "") + + +def test_bake_missing_file_skipped(db, make_book, tmp_path): + book = make_book(title="Gone", file_path=str(tmp_path / "nope.epub"), + file_format="epub") + result = bake_to_file(book, book.files[0]) + assert result.status == "skipped" + assert "missing" in (result.reason or "") diff --git a/website/src/pages/docs/admin.astro b/website/src/pages/docs/admin.astro index 52d7040..125476a 100644 --- a/website/src/pages/docs/admin.astro +++ b/website/src/pages/docs/admin.astro @@ -87,6 +87,37 @@ import { withBase } from '../../lib/path' read-write first. +

Bake metadata to files

+

+ By default Tome never modifies your source files: it embeds its metadata (title, author, + series, description, cover, tags) into a disposable copy at download time, leaving the + originals in TOME_LIBRARY_DIR untouched. That copy is only seen by things that + download through Tome. The Bake to File tab writes that same metadata + directly into the source files on disk — useful when the files are read by something + outside Tome: Syncthing to another device, a Calibre library pointed at the same folder, or + plain NAS browsing. +

+

+ Clicking Write metadata to files shows how many files need baking (files that + already carry current metadata are skipped) and asks you to confirm. The run happens in the + background with a live progress bar, the file currently being written, and baked / skipped / + failed counts plus an estimated time remaining. You can leave the page or close the tab — the + job keeps running, and reopening the page reconnects to it. Stop halts cleanly + between files. When it finishes you get a summary, including any files that were skipped or + failed. +

+
    +
  • EPUB / CBZ — full metadata plus cover
    • +
  • PDF — title, author and subject
    • +
+ + + Baking replaces the bytes of each source file and recomputes its content hash. There is no undo. + It is disabled when TOME_LIBRARY_DIR is mounted read-only, and can be turned off + entirely with TOME_ALLOW_INFILE_BAKE=false. Editing a book's metadata later simply + marks its file as out of date again, and a re-run re-bakes only what changed. + +

Email (Send to Device)

The Email tab shows SMTP status (configured or not, with host/port/from), a diff --git a/website/src/pages/docs/changelog.astro b/website/src/pages/docs/changelog.astro index 1beb488..bc7677d 100644 --- a/website/src/pages/docs/changelog.astro +++ b/website/src/pages/docs/changelog.astro @@ -24,6 +24,17 @@ import { withBase } from '../../lib/path'

Unreleased

    +
  • + Bake metadata to files (admin). A new Admin → Bake to File + page writes Tome's metadata straight into your source library files on disk (not just the + lazy download cache), so it's visible to Syncthing, Calibre, or direct NAS browsing. Runs as a + background job with a live byte-weighted progress bar, current file, baked/skipped/failed + counts and an ETA; closing the tab doesn't stop it and reopening reconnects. Destructive and + irreversible (recomputes content hashes), so it's admin-only, confirm-gated, and disabled on + read-only mounts; already-current files are skipped. New TOME_ALLOW_INFILE_BAKE + (default true) is a hard off-switch. See + Admin tools. +
  • Send to KOReader (beta). Queue a book from the web straight to your e-reader — no email, no Amazon. A split Send to KOReader button (the caret diff --git a/website/src/pages/docs/configuration.astro b/website/src/pages/docs/configuration.astro index 5a45f9c..24ee8f7 100644 --- a/website/src/pages/docs/configuration.astro +++ b/website/src/pages/docs/configuration.astro @@ -42,6 +42,7 @@ import { withBase } from '../../lib/path' TOME_SMTP_USE_SSLfalseImplicit SSL (port 465) TOME_SMTP_DAILY_LIMIT50Per-user sends/day (0 = unlimited) TOME_SEND_TO_KOREADERfalseEnable the Send to KOReader inbox (beta) — queue books to the device, no email + TOME_ALLOW_INFILE_BAKEtrueAllow baking metadata into source files on disk. Set false to forbid it entirely