diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b1f227..12942f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ All notable changes to Tome are documented here. Format loosely follows ## [Unreleased] ### Added +- **Bake metadata to files (admin).** A new **Admin → Bake to File** page writes + Tome's metadata — title, author, series, description, cover and tags — directly + into the source library files on disk, rather than only into the lazy + download-time cache. This makes the embedded metadata visible to tools that read + the files outside Tome (Syncthing, a Calibre library pointed at the same folder, + direct NAS browsing). It runs as a single background job with a live, + byte-weighted progress bar, current-file display, baked/skipped/failed counters + and an ETA; closing the tab or navigating away does not stop it, and reopening + the page reconnects to the run in progress. When it finishes you get a summary + screen listing any files that were skipped or failed. The action is destructive + and irreversible (it recomputes each file's content hash), so it is admin-only, + confirm-gated, and disabled on read-only library mounts. Files already carrying + current metadata are skipped, so re-running is cheap. EPUB/CBZ get full metadata + plus cover (CBZ pages are now stored uncompressed during embedding, cutting bake + time on large comic archives); PDF gets title/author/subject. New env var + `TOME_ALLOW_INFILE_BAKE` (default `true`) is a hard off-switch for operators who + never want their files mutated. - **Send to KOReader (beta).** Queue a book from the web straight to your e-reader — no email, no Amazon Send-to-Kindle. It's the KOReader-native counterpart to email send-to-device: the original EPUB/CBZ arrives in your diff --git a/backend/api/users.py b/backend/api/users.py index 02a6b70..acab22a 100644 --- a/backend/api/users.py +++ b/backend/api/users.py @@ -830,3 +830,75 @@ def clear_covers_cache( os.remove(fp) deleted += 1 return {"deleted": deleted} + + +# ── Bake metadata to file (admin) ────────────────────────────────────────────── +# Whole-library, in-place rewrite of source files with Tome's metadata. Runs as a +# single background job; the UI polls /admin/bake/status. See services/bake_job.py. + +@router.get("/admin/bake/status") +def bake_status(current_user: User = Depends(get_current_user)) -> dict: + require_role(current_user, "admin") + from backend.services import bake_job + from backend.services.metadata_embed import library_writable + from backend.core.config import settings + return { + **bake_job.get_status(), + "library_writable": library_writable(), + "enabled": settings.allow_infile_bake, + } + + +@router.get("/admin/bake/preflight") +def bake_preflight(current_user: User = Depends(get_current_user)) -> dict: + require_role(current_user, "admin") + from backend.services import bake_job + from backend.services.metadata_embed import library_writable + from backend.core.config import settings + return { + **bake_job.preflight(), + "library_writable": library_writable(), + "enabled": settings.allow_infile_bake, + } + + +@router.post("/admin/bake/start") +def bake_start( + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +) -> dict: + require_role(current_user, "admin") + from backend.services import bake_job + from backend.services.metadata_embed import library_writable + from backend.core.config import settings + if not settings.allow_infile_bake: + raise HTTPException(403, "In-file baking is disabled (TOME_ALLOW_INFILE_BAKE=false)") + if not library_writable(): + raise HTTPException(409, "Library directory is read-only") + try: + state = bake_job.start(username=current_user.username) + except bake_job.BakeAlreadyRunning: + raise HTTPException(409, "A bake is already running") + audit( + db, + "books.metadata_baked_started", + user_id=current_user.id, + username=current_user.username, + details={"total_files": state.get("total_files"), "total_bytes": state.get("total_bytes")}, + ) + return state + + +@router.post("/admin/bake/cancel") +def bake_cancel(current_user: User = Depends(get_current_user)) -> dict: + require_role(current_user, "admin") + from backend.services import bake_job + cancelled = bake_job.request_cancel() + return {"cancelling": cancelled, **bake_job.get_status()} + + +@router.post("/admin/bake/dismiss") +def bake_dismiss(current_user: User = Depends(get_current_user)) -> dict: + require_role(current_user, "admin") + from backend.services import bake_job + return bake_job.dismiss() diff --git a/backend/core/config.py b/backend/core/config.py index e786f60..3d26b27 100644 --- a/backend/core/config.py +++ b/backend/core/config.py @@ -64,6 +64,11 @@ class Settings(BaseSettings): # switch thereafter. send_to_koreader: bool = False + # Bake-to-file: write Tome's metadata into the source library files on disk + # (vs the lazy download cache). Destructive + admin-gated. Hard off-switch for + # operators who never want their files mutated. env TOME_ALLOW_INFILE_BAKE. + allow_infile_bake: bool = True + # JWT settings jwt_algorithm: str = "HS256" jwt_expire_minutes: int = 60 * 24 * 7 # 7 days diff --git a/backend/main.py b/backend/main.py index 3c56a84..6c14475 100644 --- a/backend/main.py +++ b/backend/main.py @@ -130,6 +130,12 @@ async def lifespan(app: FastAPI): if "scope" not in at_cols: conn.execute(text("ALTER TABLE api_tokens ADD COLUMN scope VARCHAR(16) NOT NULL DEFAULT 'full'")) conn.commit() + # Bake-to-file: track when a file's on-disk bytes were last written with + # Tome's metadata. NULL for every pre-existing row (= never baked in place). + bf_cols = {r[1] for r in conn.execute(text("PRAGMA table_info(book_files)")).fetchall()} + if "metadata_synced_at" not in bf_cols: + conn.execute(text("ALTER TABLE book_files ADD COLUMN metadata_synced_at DATETIME")) + conn.commit() init_fts(engine) backfill_fts(engine) settings.ensure_dirs() diff --git a/backend/models/book.py b/backend/models/book.py index 016938f..7b0514c 100644 --- a/backend/models/book.py +++ b/backend/models/book.py @@ -66,6 +66,12 @@ class BookFile(Base): file_size: Mapped[Optional[int]] = mapped_column(Integer) content_hash: Mapped[Optional[str]] = mapped_column(String(64), index=True) added_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, nullable=False) + # Set to the owning Book.updated_at at the moment this file was baked to disk + # (in-file metadata write). When it equals Book.updated_at, the bytes on disk + # already carry the current metadata → download path can serve the raw file + # and skip the lazy bake. Goes stale (≠ updated_at) the next time metadata is + # edited, transparently re-enabling lazy baking. NULL = never baked in place. + metadata_synced_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) book: Mapped["Book"] = relationship("Book", back_populates="files") diff --git a/backend/services/bake_job.py b/backend/services/bake_job.py new file mode 100644 index 0000000..07bb717 --- /dev/null +++ b/backend/services/bake_job.py @@ -0,0 +1,262 @@ +"""Whole-library 'bake metadata to file' background job. + +A single, serial, cancellable job rewrites the source files on disk with Tome's +metadata (see ``metadata_embed.bake_to_file``). It runs in a daemon thread so it +survives the request that started it — closing the browser tab or navigating +away never cancels it. Progress lives in module-global state (guarded by a lock) +and is read back via the status endpoint, so the UI is just a poller and can +reconnect at any time. + +Single-process model: state is in-memory, so a server restart drops the job and +its progress. That is safe and cheap to recover from — ``metadata_synced_at`` +makes a re-run skip everything already baked, so the admin just starts it again +and it resumes where it left off. (Tome runs a single uvicorn worker; SQLite is +single-writer, so there is no multi-process state to reconcile.) + +Progress is weighted by *bytes*, not file count: a 300 MB CBZ costs ~150× an +EPUB, so a file-count bar/ETA would lie. We sum file sizes up front and track +bytes processed; the ETA is ``elapsed * bytes_left / bytes_done``. +""" +from __future__ import annotations + +import logging +import threading +import time +from typing import Optional + +from backend.core.database import SessionLocal +from backend.models.book import Book, BookFile +from backend.services.metadata_embed import BAKEABLE_FORMATS, bake_to_file + +log = logging.getLogger(__name__) + +# Cap on how many per-file issues (skipped/failed) we retain for the end screen. +_MAX_ISSUES = 500 + +_lock = threading.Lock() +_cancel = threading.Event() +_thread: Optional[threading.Thread] = None + +# The single job's state. ``status`` is one of: +# idle | running | done | cancelled | error +_state: dict = {"status": "idle"} + + +class BakeAlreadyRunning(Exception): + pass + + +def _needs_bake(synced, updated) -> bool: + return not (synced is not None and updated is not None and synced == updated) + + +def preflight() -> dict: + """Cheap, DB-only summary of what a run would do, using stored file_size + (no disk stat). Safe to call on page load.""" + with SessionLocal() as db: + rows = ( + db.query( + BookFile.file_size, + BookFile.format, + BookFile.metadata_synced_at, + Book.updated_at, + ) + .join(Book, BookFile.book_id == Book.id) + .all() + ) + bakeable = already = pending = 0 + pending_bytes = 0 + for size, fmt, synced, updated in rows: + if (fmt or "").lower() not in BAKEABLE_FORMATS: + continue + bakeable += 1 + if _needs_bake(synced, updated): + pending += 1 + pending_bytes += size or 0 + else: + already += 1 + return { + "bakeable_total": bakeable, + "already_current": already, + "pending": pending, + "pending_bytes": pending_bytes, + } + + +def _snapshot() -> dict: + """A JSON-able copy of state with derived elapsed/ETA filled in.""" + s = dict(_state) + if s.get("status") == "running" and s.get("started_at"): + elapsed = time.time() - s["started_at"] + elif s.get("started_at") and s.get("finished_at"): + elapsed = s["finished_at"] - s["started_at"] + else: + elapsed = 0.0 + s["elapsed_seconds"] = round(elapsed, 1) + + eta = None + if s.get("status") == "running": + done_bytes = s.get("done_bytes", 0) + total_bytes = s.get("total_bytes", 0) + if done_bytes > 0 and total_bytes > done_bytes: + eta = elapsed * (total_bytes - done_bytes) / done_bytes + s["eta_seconds"] = round(eta, 1) if eta is not None else None + return s + + +def get_status() -> dict: + with _lock: + return _snapshot() + + +def dismiss() -> dict: + """Clear a finished run's summary so the page resets to idle. No-op while + a run is in progress.""" + global _state + with _lock: + if _state.get("status") != "running": + _state = {"status": "idle"} + return _snapshot() + + +def request_cancel() -> bool: + with _lock: + if _state.get("status") == "running": + _cancel.set() + return True + return False + + +def start(*, username: Optional[str] = None) -> dict: + """Begin a whole-library bake. Raises BakeAlreadyRunning if one is active.""" + global _state, _thread + with _lock: + if _state.get("status") == "running": + raise BakeAlreadyRunning() + + # Build the work list now (file_id, book_id, size) for files that need it. + with SessionLocal() as db: + rows = ( + db.query( + BookFile.id, + BookFile.book_id, + BookFile.file_size, + BookFile.format, + BookFile.metadata_synced_at, + Book.updated_at, + ) + .join(Book, BookFile.book_id == Book.id) + .all() + ) + work: list[tuple[int, int, int]] = [] + total_bytes = 0 + for file_id, book_id, size, fmt, synced, updated in rows: + if (fmt or "").lower() not in BAKEABLE_FORMATS: + continue + if _needs_bake(synced, updated): + size = size or 0 + work.append((file_id, book_id, size)) + total_bytes += size + + _cancel.clear() + _state = { + "status": "running", + "started_at": time.time(), + "finished_at": None, + "triggered_by": username, + "total_files": len(work), + "total_bytes": total_bytes, + "done_files": 0, + "done_bytes": 0, + "baked": 0, + "skipped": 0, + "failed": 0, + "current_file": None, + "issues": [], + "error": None, + } + _thread = threading.Thread( + target=_run, args=(work,), name="bake-to-file", daemon=True + ) + _thread.start() + return _snapshot() + + +def _set_current(path: str) -> None: + with _lock: + _state["current_file"] = path + + +def _record(result, size: int) -> None: + with _lock: + _state["done_files"] += 1 + _state["done_bytes"] += size + st = result.status + if st == "baked": + _state["baked"] += 1 + elif st == "failed": + _state["failed"] += 1 + else: # skipped | readonly + _state["skipped"] += 1 + if st != "baked" and len(_state["issues"]) < _MAX_ISSUES: + _state["issues"].append( + {"path": result.file_path, "status": st, "reason": result.reason} + ) + + +def _finish(status: str, error: Optional[str] = None) -> None: + with _lock: + _state["status"] = status + _state["finished_at"] = time.time() + _state["current_file"] = None + if error: + _state["error"] = error + + +def _run(work: list[tuple[int, int, int]]) -> None: + try: + for file_id, book_id, size in work: + if _cancel.is_set(): + _finish("cancelled") + return + try: + with SessionLocal() as db: + bf = db.get(BookFile, file_id) + if bf is None: + _record(_Missing(f"file #{file_id}"), size) + continue + _set_current(bf.file_path) + book = db.get(Book, book_id) + if book is None: + _record(_Missing(bf.file_path), size) + continue + result = bake_to_file(book, bf) + if result.ok: + db.commit() + else: + db.rollback() + _record(result, size) + except Exception as e: # noqa: BLE001 — one bad file never aborts the run + log.exception("bake: unexpected error on file #%s", file_id) + _record(_Failed(str(file_id), str(e)), size) + _finish("done") + except Exception as e: # noqa: BLE001 + log.exception("bake: job crashed") + _finish("error", error=str(e)) + + +# Lightweight stand-ins so _record can treat every outcome uniformly. +class _Missing: + status = "skipped" + reason = "file missing" + + def __init__(self, path: str): + self.file_path = path + + +class _Failed: + status = "failed" + + def __init__(self, path: str, reason: str): + self.file_path = path + self.reason = reason diff --git a/backend/services/metadata_embed.py b/backend/services/metadata_embed.py index fd48893..7137a67 100644 --- a/backend/services/metadata_embed.py +++ b/backend/services/metadata_embed.py @@ -12,18 +12,36 @@ import io import logging +import os import shutil import zipfile +from dataclasses import dataclass +from datetime import datetime from pathlib import Path from typing import Optional from xml.etree import ElementTree as ET from backend.core.config import settings from backend.models.book import Book, BookFile +from backend.services.metadata import sha256_file from backend.services.xml_ns import namespaces log = logging.getLogger(__name__) +# Formats whose bytes we can rewrite with embedded metadata. Everything else is +# served/left as-is. +BAKEABLE_FORMATS = {"epub", "cbz", "pdf"} + +# Image entries are already compressed; re-deflating them is pure CPU cost for +# zero size win (and, for big CBZ archives, the bulk of the bake time). Store +# them uncompressed instead. Text-ish entries (XHTML, CSS, XML) still deflate. +_STORED_EXTS = {".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".avif", ".jxl"} + + +def _compress_for(name: str) -> int: + ext = Path(name).suffix.lower() + return zipfile.ZIP_STORED if ext in _STORED_EXTS else zipfile.ZIP_DEFLATED + OPF_NS = "http://www.idpf.org/2007/opf" DC_NS = "http://purl.org/dc/elements/1.1/" CONTAINER_NS = "urn:oasis:names:tc:opendocument:xmlns:container" @@ -75,6 +93,15 @@ def get_baked_path(book: Book, book_file: BookFile) -> Path: if not src.exists(): return src + # If the on-disk file was already baked in place for this exact metadata + # revision, its bytes are current — serve the raw file, skip embed + cache. + if ( + book_file.metadata_synced_at is not None + and book.updated_at is not None + and book_file.metadata_synced_at == book.updated_at + ): + return src + key = _cache_key(book) baked = _baked_dir() / f"{key}.{book_file.format}" if baked.exists() and baked.stat().st_size > 0: @@ -308,7 +335,7 @@ def _embed_cbz(book: Book, src: Path, cover_bytes: Optional[bytes]) -> Optional[ buf = io.BytesIO() with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zout: if cover_bytes is not None: - zout.writestr(COVER_NAME, cover_bytes) + zout.writestr(COVER_NAME, cover_bytes, compress_type=zipfile.ZIP_STORED) for n in names: # Skip the old ComicInfo — we rewrite it below if n.lower() == "comicinfo.xml": @@ -316,7 +343,8 @@ def _embed_cbz(book: Book, src: Path, cover_bytes: Optional[bytes]) -> Optional[ # Skip the slot we're writing the Tome cover into (if any) if cover_bytes is not None and n == COVER_NAME: continue - zout.writestr(n, zf.read(n)) + # Pages are already-compressed images → store, don't re-deflate. + zout.writestr(n, zf.read(n), compress_type=_compress_for(n)) zout.writestr(COMIC_INFO_NAME, comic_info) return buf.getvalue() @@ -377,3 +405,119 @@ def _embed_pdf(book: Book, src: Path) -> Optional[bytes]: finally: doc.close() return buf + + +# ── In-file bake (write metadata into the source file on disk) ───────────────── +# +# Unlike get_baked_path (which writes a disposable cache copy), bake_to_file +# mutates the real library file. It is destructive + irreversible, so the heavy +# lifting here is safety: validate the freshly-embedded bytes parse, write to a +# sibling tmp, fsync, then atomically os.replace the original. A buggy embed must +# never clobber a good source. + +@dataclass +class BakeFileResult: + """Outcome of baking one BookFile in place.""" + file_path: str + status: str # "baked" | "skipped" | "readonly" | "failed" + reason: Optional[str] = None + old_hash: Optional[str] = None + new_hash: Optional[str] = None + + @property + def ok(self) -> bool: + return self.status == "baked" + + +def library_writable() -> bool: + """Is the library root writable? Used to gate the bake UI on a `:ro` mount.""" + try: + return os.access(settings.library_dir, os.W_OK) + except OSError: + return False + + +def _validate_baked(out: bytes, fmt: str) -> bool: + """Cheap sanity check that the embedded bytes are a usable file of `fmt` + before we let them replace the original.""" + fmt = (fmt or "").lower() + if fmt in ("epub", "cbz"): + try: + with zipfile.ZipFile(io.BytesIO(out), "r") as zf: + if zf.testzip() is not None: + return False + if fmt == "epub" and "META-INF/container.xml" not in zf.namelist(): + return False + return True + except (zipfile.BadZipFile, OSError): + return False + if fmt == "pdf": + return out[:5] == b"%PDF-" + return False + + +def bake_to_file(book: Book, book_file: BookFile) -> BakeFileResult: + """Write Tome's metadata into ``book_file`` on disk, in place. + + Mutates ``book_file.content_hash`` + ``book_file.metadata_synced_at`` on the + attached ORM objects **without committing** — the caller owns the transaction. + Per-file isolated: any failure leaves the original file untouched, never + leaks a ``.tmp``, and returns a non-``baked`` status rather than raising. + """ + fmt = (book_file.format or "").lower() + src = Path(book_file.file_path) + + if fmt not in BAKEABLE_FORMATS: + return BakeFileResult(str(src), "skipped", reason="unsupported format") + if not src.exists(): + return BakeFileResult(str(src), "skipped", reason="file missing") + if not os.access(src.parent, os.W_OK): + return BakeFileResult(str(src), "readonly", reason="directory not writable") + + try: + cover_bytes = _load_cover(book) + out = _embed(book, src, fmt, cover_bytes) + except Exception as e: # noqa: BLE001 — embed must never abort a bulk run + log.warning("bake: embed failed for book_id=%s file=%s: %s", book.id, src.name, e) + return BakeFileResult(str(src), "failed", reason=f"embed error: {e}") + + if out is None: + # _embed returns None for formats it can't handle (e.g. PDF without + # PyMuPDF, or a malformed EPUB with no OPF). + return BakeFileResult(str(src), "skipped", reason="nothing to embed") + + if not _validate_baked(out, fmt): + return BakeFileResult(str(src), "failed", reason="validation failed — original kept") + + old_hash = book_file.content_hash + tmp = src.with_suffix(src.suffix + ".bake.tmp") + try: + with open(tmp, "wb") as f: + f.write(out) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, src) # atomic within the same directory + except OSError as e: + log.warning("bake: write failed for book_id=%s file=%s: %s", book.id, src.name, e) + try: + if tmp.exists(): + tmp.unlink() + except OSError: + pass + return BakeFileResult(str(src), "failed", reason=f"write error: {e}") + + new_hash = sha256_file(src) + book_file.content_hash = new_hash + book_file.metadata_synced_at = book.updated_at + try: + book_file.file_size = src.stat().st_size + except OSError: + pass + + # The lazy cache for this book is now redundant. + try: + _purge_stale(book.id) + except OSError: + pass + + return BakeFileResult(str(src), "baked", old_hash=old_hash, new_hash=new_hash) diff --git a/frontend/src/components/BakeToFile.tsx b/frontend/src/components/BakeToFile.tsx new file mode 100644 index 0000000..94f3743 --- /dev/null +++ b/frontend/src/components/BakeToFile.tsx @@ -0,0 +1,319 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import { + HardDriveDownload, AlertTriangle, Loader2, Check, X, + FileWarning, Play, Ban, RotateCcw, CircleSlash, +} from 'lucide-react' +import { cn } from '@/lib/utils' +import { + getBakeStatus, getBakePreflight, startBake, cancelBake, dismissBake, + type BakeState, type BakePreflight, +} from '@/lib/bake' + +const POLL_MS = 1500 + +function formatBytes(n: number): string { + if (!n) return '0 B' + const units = ['B', 'KB', 'MB', 'GB', 'TB'] + let i = 0 + let v = n + while (v >= 1024 && i < units.length - 1) { v /= 1024; i++ } + return `${v.toFixed(i === 0 ? 0 : 1)} ${units[i]}` +} + +function formatDuration(secs: number | null): string { + if (secs == null) return '—' + const s = Math.max(0, Math.round(secs)) + if (s < 60) return `${s}s` + const m = Math.floor(s / 60) + const rem = s % 60 + if (m < 60) return `${m}m ${rem}s` + const h = Math.floor(m / 60) + return `${h}h ${m % 60}m` +} + +const TERMINAL = new Set(['done', 'cancelled', 'error']) + +export function BakeToFileTab() { + const [preflight, setPreflight] = useState(null) + const [state, setState] = useState(null) + const [confirming, setConfirming] = useState(false) + const [busy, setBusy] = useState(false) + const [error, setError] = useState(null) + const pollRef = useRef | null>(null) + + const stopPolling = useCallback(() => { + if (pollRef.current) { clearInterval(pollRef.current); pollRef.current = null } + }, []) + + const poll = useCallback(async () => { + try { + const s = await getBakeStatus() + setState(s) + if (TERMINAL.has(s.status) || s.status === 'idle') { + stopPolling() + // refresh the pending/already-current counts after a run + getBakePreflight().then(setPreflight).catch(() => {}) + } + } catch { /* keep last state */ } + }, [stopPolling]) + + const startPolling = useCallback(() => { + stopPolling() + pollRef.current = setInterval(poll, POLL_MS) + }, [poll, stopPolling]) + + // On mount: load preflight + current status; reconnect to a running job. + useEffect(() => { + let alive = true + Promise.all([getBakePreflight(), getBakeStatus()]) + .then(([pf, s]) => { + if (!alive) return + setPreflight(pf) + setState(s) + if (s.status === 'running') startPolling() + }) + .catch(() => { if (alive) setError('Failed to load bake status') }) + return () => { alive = false; stopPolling() } + }, [startPolling, stopPolling]) + + async function handleStart() { + setBusy(true) + setError(null) + try { + const s = await startBake() + setState(s) + setConfirming(false) + startPolling() + } catch (e) { + setError(e instanceof Error ? e.message : 'Failed to start') + } finally { + setBusy(false) + } + } + + async function handleCancel() { + setBusy(true) + try { + const s = await cancelBake() + setState(s) + } catch { /* ignore */ } finally { setBusy(false) } + } + + async function handleDismiss() { + setBusy(true) + try { + const s = await dismissBake() + setState(s) + getBakePreflight().then(setPreflight).catch(() => {}) + } catch { /* ignore */ } finally { setBusy(false) } + } + + const running = state?.status === 'running' + const terminal = state ? TERMINAL.has(state.status) : false + const writable = preflight?.library_writable ?? state?.library_writable ?? true + const enabled = preflight?.enabled ?? state?.enabled ?? true + const pending = preflight?.pending ?? 0 + const pct = state && state.total_bytes > 0 + ? Math.min(100, (state.done_bytes / state.total_bytes) * 100) + : 0 + + return ( +
+
+
+ +
+
+

Bake metadata to files

+

+ Write Tome's metadata (title, author, series, description, cover, tags) directly + into the source files on disk. Useful when files are read outside Tome — Syncthing, + a Calibre library on the same folder, or direct NAS browsing. +

+
+
+ + {/* Caveat box */} +
+ +
+ This rewrites the actual library files and recomputes their content hash. + It cannot be undone. EPUB and CBZ get full + metadata + cover; PDF gets title/author/subject. Files already current are skipped. +
+
+ + {error && ( +
+ {error} + +
+ )} + + {!enabled && ( +
+ + In-file baking is disabled on this server (TOME_ALLOW_INFILE_BAKE=false). +
+ )} + {enabled && !writable && ( +
+ + The library directory is mounted read-only — files cannot be modified. +
+ )} + + {/* ── Idle: pre-flight + start ───────────────────────────────────────── */} + {!running && !terminal && ( +
+
+ + + 0} sub={preflight ? formatBytes(preflight.pending_bytes) : undefined} /> +
+
+ {confirming ? ( +
+

+ Rewrite {pending} file{pending !== 1 ? 's' : ''} on disk + {preflight ? <> ({formatBytes(preflight.pending_bytes)}) : null}? + This cannot be undone. +

+
+ + +
+
+ ) : ( + + )} +
+
+ )} + + {/* ── Running: progress ──────────────────────────────────────────────── */} + {running && state && ( +
+
+
+ + + Baking… {state.done_files} / {state.total_files} files + + {pct.toFixed(1)}% +
+
+
+
+
+ {formatBytes(state.done_bytes)} / {formatBytes(state.total_bytes)} + Elapsed {formatDuration(state.elapsed_seconds)} · ~{formatDuration(state.eta_seconds)} left +
+
+
+
+ Current:{' '} + {state.current_file?.split('/').pop() ?? '…'} +
+
+ + + 0} /> +
+ +
+
+ )} + + {/* ── Terminal: end summary ──────────────────────────────────────────── */} + {terminal && state && ( +
+
+ {state.status === 'done' && } + {state.status === 'cancelled' && } + {state.status === 'error' && } + + {state.status === 'done' && 'Bake complete'} + {state.status === 'cancelled' && 'Bake stopped'} + {state.status === 'error' && 'Bake failed'} + + Took {formatDuration(state.elapsed_seconds)} +
+
+ + + 0} /> +
+ {state.error && ( +
{state.error}
+ )} + {state.issues.length > 0 && ( +
+

+ Files not baked ({state.issues.length}) +

+
+ {state.issues.map((it, i) => ( +
+ + {it.status} + + {it.path.split('/').pop()} + {it.reason && {it.reason}} +
+ ))} +
+
+ )} +
+ + {pending > 0 && ( + + )} +
+
+ )} +
+ ) +} + +function Stat({ label, value, sub, accent, danger }: { + label: string; value: number | string; sub?: string; accent?: boolean; danger?: boolean +}) { + return ( +
+

+ {value} +

+

{label}

+ {sub &&

{sub}

} +
+ ) +} diff --git a/frontend/src/lib/bake.ts b/frontend/src/lib/bake.ts new file mode 100644 index 0000000..b8e66ed --- /dev/null +++ b/frontend/src/lib/bake.ts @@ -0,0 +1,45 @@ +import { api } from './api' + +export type BakeStatus = 'idle' | 'running' | 'done' | 'cancelled' | 'error' + +export interface BakeIssue { + path: string + status: 'skipped' | 'readonly' | 'failed' + reason: string | null +} + +export interface BakeState { + status: BakeStatus + started_at: number | null + finished_at: number | null + triggered_by: string | null + total_files: number + total_bytes: number + done_files: number + done_bytes: number + baked: number + skipped: number + failed: number + current_file: string | null + issues: BakeIssue[] + error: string | null + elapsed_seconds: number + eta_seconds: number | null + library_writable: boolean + enabled: boolean +} + +export interface BakePreflight { + bakeable_total: number + already_current: number + pending: number + pending_bytes: number + library_writable: boolean + enabled: boolean +} + +export const getBakeStatus = () => api.get('/admin/bake/status') +export const getBakePreflight = () => api.get('/admin/bake/preflight') +export const startBake = () => api.post('/admin/bake/start') +export const cancelBake = () => api.post('/admin/bake/cancel') +export const dismissBake = () => api.post('/admin/bake/dismiss') diff --git a/frontend/src/pages/AdminPage.tsx b/frontend/src/pages/AdminPage.tsx index 12a416a..9c0a79b 100644 --- a/frontend/src/pages/AdminPage.tsx +++ b/frontend/src/pages/AdminPage.tsx @@ -11,6 +11,7 @@ import { import { DOCS, docsLink } from '@/lib/docs' import { MetadataManager } from '@/components/MetadataManager' import { LibraryHealthTab } from '@/components/LibraryHealth' +import { BakeToFileTab } from '@/components/BakeToFile' import { SeriesCoverageStrip } from '@/components/SeriesCoverageStrip' import { useAuth, isAdmin } from '@/contexts/AuthContext' import { api } from '@/lib/api' @@ -2003,7 +2004,7 @@ function WishlistTab() { // ── AdminPage ───────────────────────────────────────────────────────────── -type Tab = 'users' | 'scanner' | 'server' | 'types' | 'audit' | 'metadata' | 'library' | 'sync' | 'duplicates' | 'email' | 'wishlist' +type Tab = 'users' | 'scanner' | 'server' | 'types' | 'audit' | 'metadata' | 'library' | 'bake' | 'sync' | 'duplicates' | 'email' | 'wishlist' export function AdminPage() { const { user } = useAuth() @@ -2027,6 +2028,7 @@ export function AdminPage() { { id: 'audit', label: 'Audit Log' }, { id: 'metadata', label: 'Metadata' }, { id: 'library', label: 'Library' }, + { id: 'bake', label: 'Bake to File' }, { id: 'sync', label: 'Sync Status' }, { id: 'duplicates', label: 'Duplicates' }, { id: 'email', label: 'Email' }, @@ -2070,6 +2072,7 @@ export function AdminPage() { {tab === 'audit' && } {tab === 'metadata' && } {tab === 'library' && } + {tab === 'bake' && } {tab === 'sync' && } {tab === 'duplicates' && } {tab === 'email' && } diff --git a/tests/test_bake_to_file.py b/tests/test_bake_to_file.py new file mode 100644 index 0000000..d887457 --- /dev/null +++ b/tests/test_bake_to_file.py @@ -0,0 +1,208 @@ +"""Tests for in-file metadata baking (backend/services/metadata_embed.bake_to_file). + +The embed itself is covered by test_metadata_embed.py; this focuses on the +safety machinery: validate-before-replace, atomic write, hash recompute, +metadata_synced_at semantics, the read-only guard, and per-file failure +isolation (a bad embed must never clobber a good source). +""" +import zipfile +from datetime import timedelta + +import pytest + +from backend.services import metadata_embed as me +from backend.services.metadata_embed import ( + bake_to_file, get_baked_path, _compress_for, _validate_baked, +) +from backend.services.metadata import sha256_file + + +# ── builders ─────────────────────────────────────────────────────────────────── + +def _make_epub(path): + with zipfile.ZipFile(path, "w") as z: + z.writestr("mimetype", "application/epub+zip", compress_type=zipfile.ZIP_STORED) + z.writestr( + "META-INF/container.xml", + '' + '' + '', + ) + z.writestr( + "content.opf", + '' + '' + 'Original Title' + '' + '' + '', + ) + z.writestr("t.xhtml", "hi") + + +def _make_cbz(path): + with zipfile.ZipFile(path, "w") as z: + z.writestr("001.jpg", b"\xff\xd8\xff\xe0" + b"fakejpeg" * 50) + z.writestr("002.jpg", b"\xff\xd8\xff\xe0" + b"moredata" * 50) + + +# ── unit: helpers ──────────────────────────────────────────────────────────── + +def test_compress_for_images_stored(): + assert _compress_for("001.jpg") == zipfile.ZIP_STORED + assert _compress_for("cover.PNG") == zipfile.ZIP_STORED + assert _compress_for("page.xhtml") == zipfile.ZIP_DEFLATED + assert _compress_for("ComicInfo.xml") == zipfile.ZIP_DEFLATED + + +def test_validate_baked(): + assert _validate_baked(b"%PDF-1.7 ...", "pdf") is True + assert _validate_baked(b"not a pdf", "pdf") is False + assert _validate_baked(b"not a zip", "epub") is False + assert _validate_baked(b"not a zip", "cbz") is False + + +# ── bake: EPUB happy path ────────────────────────────────────────────────────── + +def test_bake_epub_rewrites_metadata_and_hash(db, make_book, tmp_path): + epub = tmp_path / "book.epub" + _make_epub(epub) + book = make_book(title="Baked Title", author="Jane Doe", series="My Series", + series_index=3, file_path=str(epub), file_format="epub", + content_hash="oldhash") + bf = book.files[0] + + result = bake_to_file(book, bf) + db.commit() + + assert result.status == "baked" + # File is still a valid EPUB carrying Tome's title. + with zipfile.ZipFile(epub, "r") as z: + assert z.testzip() is None + opf = z.read("content.opf").decode() + assert "Baked Title" in opf + assert "calibre:series" in opf and "My Series" in opf + # Hash recomputed to match the new bytes, and the file is marked synced. + assert bf.content_hash == sha256_file(epub) + assert bf.content_hash != "oldhash" + assert bf.metadata_synced_at == book.updated_at + + +# ── bake: CBZ stores pages uncompressed ──────────────────────────────────────── + +def test_bake_cbz_embeds_comicinfo_and_stores_images(db, make_book, tmp_path): + cbz = tmp_path / "comic.cbz" + _make_cbz(cbz) + book = make_book(title="Vol 1", series="Comic Series", series_index=1, + file_path=str(cbz), file_format="cbz") + bf = book.files[0] + + result = bake_to_file(book, bf) + db.commit() + + assert result.status == "baked" + with zipfile.ZipFile(cbz, "r") as z: + assert z.testzip() is None + names = z.namelist() + assert "ComicInfo.xml" in names + assert "Comic Series" in z.read("ComicInfo.xml").decode() + # Image pages must be stored, not re-deflated. + assert z.getinfo("001.jpg").compress_type == zipfile.ZIP_STORED + + +# ── get_baked_path early-return ──────────────────────────────────────────────── + +def test_get_baked_path_serves_raw_when_synced(db, make_book, tmp_path): + epub = tmp_path / "synced.epub" + _make_epub(epub) + book = make_book(title="Synced", file_path=str(epub), file_format="epub") + bf = book.files[0] + + bake_to_file(book, bf) + db.commit() + + # Already current → raw path, no cache copy. + assert get_baked_path(book, bf) == epub + + # Simulate a later metadata edit bumping updated_at → stale → lazy-bake again. + book.updated_at = book.updated_at + timedelta(seconds=5) + db.flush() + out = get_baked_path(book, bf) + assert out != epub + assert out.exists() + + +# ── safety: read-only directory ──────────────────────────────────────────────── + +def test_bake_readonly_dir_leaves_file_untouched(db, make_book, tmp_path, monkeypatch): + epub = tmp_path / "ro.epub" + _make_epub(epub) + before = epub.read_bytes() + book = make_book(title="RO", file_path=str(epub), file_format="epub") + bf = book.files[0] + bf.content_hash = "keepme" + db.flush() + + monkeypatch.setattr(me.os, "access", lambda p, mode: False) + result = bake_to_file(book, bf) + + assert result.status == "readonly" + assert epub.read_bytes() == before + assert bf.content_hash == "keepme" + assert bf.metadata_synced_at is None + + +# ── safety: corrupt embed must not replace the original ───────────────────────── + +def test_bake_corrupt_embed_fails_safe(db, make_book, tmp_path, monkeypatch): + epub = tmp_path / "corrupt.epub" + _make_epub(epub) + before = epub.read_bytes() + book = make_book(title="Corrupt", file_path=str(epub), file_format="epub") + bf = book.files[0] + + monkeypatch.setattr(me, "_embed", lambda *a, **k: b"not a real zip") + result = bake_to_file(book, bf) + + assert result.status == "failed" + assert "validation" in (result.reason or "") + assert epub.read_bytes() == before # original intact + assert not (tmp_path / "corrupt.epub.bake.tmp").exists() # no leftover tmp + assert bf.metadata_synced_at is None + + +def test_bake_embed_raises_fails_safe(db, make_book, tmp_path, monkeypatch): + epub = tmp_path / "boom.epub" + _make_epub(epub) + before = epub.read_bytes() + book = make_book(title="Boom", file_path=str(epub), file_format="epub") + bf = book.files[0] + + def _boom(*a, **k): + raise RuntimeError("kaboom") + monkeypatch.setattr(me, "_embed", _boom) + result = bake_to_file(book, bf) + + assert result.status == "failed" + assert epub.read_bytes() == before + + +# ── skips ─────────────────────────────────────────────────────────────────── + +def test_bake_unsupported_format_skipped(db, make_book, tmp_path): + mobi = tmp_path / "x.mobi" + mobi.write_bytes(b"whatever") + book = make_book(title="Mobi", file_path=str(mobi), file_format="mobi") + result = bake_to_file(book, book.files[0]) + assert result.status == "skipped" + assert "format" in (result.reason or "") + + +def test_bake_missing_file_skipped(db, make_book, tmp_path): + book = make_book(title="Gone", file_path=str(tmp_path / "nope.epub"), + file_format="epub") + result = bake_to_file(book, book.files[0]) + assert result.status == "skipped" + assert "missing" in (result.reason or "") diff --git a/website/src/pages/docs/admin.astro b/website/src/pages/docs/admin.astro index 52d7040..125476a 100644 --- a/website/src/pages/docs/admin.astro +++ b/website/src/pages/docs/admin.astro @@ -87,6 +87,37 @@ import { withBase } from '../../lib/path' read-write first. +

Bake metadata to files

+

+ By default Tome never modifies your source files: it embeds its metadata (title, author, + series, description, cover, tags) into a disposable copy at download time, leaving the + originals in TOME_LIBRARY_DIR untouched. That copy is only seen by things that + download through Tome. The Bake to File tab writes that same metadata + directly into the source files on disk — useful when the files are read by something + outside Tome: Syncthing to another device, a Calibre library pointed at the same folder, or + plain NAS browsing. +

+

+ Clicking Write metadata to files shows how many files need baking (files that + already carry current metadata are skipped) and asks you to confirm. The run happens in the + background with a live progress bar, the file currently being written, and baked / skipped / + failed counts plus an estimated time remaining. You can leave the page or close the tab — the + job keeps running, and reopening the page reconnects to it. Stop halts cleanly + between files. When it finishes you get a summary, including any files that were skipped or + failed. +

+
    +
  • EPUB / CBZ — full metadata plus cover
    • +
  • PDF — title, author and subject
    • +
+ + + Baking replaces the bytes of each source file and recomputes its content hash. There is no undo. + It is disabled when TOME_LIBRARY_DIR is mounted read-only, and can be turned off + entirely with TOME_ALLOW_INFILE_BAKE=false. Editing a book's metadata later simply + marks its file as out of date again, and a re-run re-bakes only what changed. + +

Email (Send to Device)

The Email tab shows SMTP status (configured or not, with host/port/from), a diff --git a/website/src/pages/docs/changelog.astro b/website/src/pages/docs/changelog.astro index 1beb488..bc7677d 100644 --- a/website/src/pages/docs/changelog.astro +++ b/website/src/pages/docs/changelog.astro @@ -24,6 +24,17 @@ import { withBase } from '../../lib/path'

Unreleased

    +
  • + Bake metadata to files (admin). A new Admin → Bake to File + page writes Tome's metadata straight into your source library files on disk (not just the + lazy download cache), so it's visible to Syncthing, Calibre, or direct NAS browsing. Runs as a + background job with a live byte-weighted progress bar, current file, baked/skipped/failed + counts and an ETA; closing the tab doesn't stop it and reopening reconnects. Destructive and + irreversible (recomputes content hashes), so it's admin-only, confirm-gated, and disabled on + read-only mounts; already-current files are skipped. New TOME_ALLOW_INFILE_BAKE + (default true) is a hard off-switch. See + Admin tools. +
  • Send to KOReader (beta). Queue a book from the web straight to your e-reader — no email, no Amazon. A split Send to KOReader button (the caret diff --git a/website/src/pages/docs/configuration.astro b/website/src/pages/docs/configuration.astro index 5a45f9c..24ee8f7 100644 --- a/website/src/pages/docs/configuration.astro +++ b/website/src/pages/docs/configuration.astro @@ -42,6 +42,7 @@ import { withBase } from '../../lib/path' TOME_SMTP_USE_SSLfalseImplicit SSL (port 465) TOME_SMTP_DAILY_LIMIT50Per-user sends/day (0 = unlimited) TOME_SEND_TO_KOREADERfalseEnable the Send to KOReader inbox (beta) — queue books to the device, no email + TOME_ALLOW_INFILE_BAKEtrueAllow baking metadata into source files on disk. Set false to forbid it entirely