diff --git a/CHANGELOG.md b/CHANGELOG.md index 0238494..95ae1ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,20 @@ All notable changes to Tome are documented here. Format loosely follows ## [Unreleased] +### Added +- **Time per chapter.** Book pages now show where your reading time went + chapter by chapter: the book's table of contents is extracted at ingest into + device-independent chapter boundaries, and KOReader per-page reading data is + mapped into them — robust to font/margin changes, since every page record is + interpreted against its own pagination. Renders on the book detail page next + to the reading-intensity curve whenever both a chapter map and synced page + data exist. Existing libraries get chapter maps via the Admin → Word Counts + backfill, which now also extracts chapters (and intrinsic page counts, below) + in the same pass. +- **Intrinsic page counts for fixed-layout books.** PDFs and comic archives now + store their real page count at ingest (EPUB deliberately doesn't — reflowable + pagination is not a property of the book). Backfilled by the same admin job. + ### Fixed - **Bindery no longer invents a series from the filename.** Prose ebooks named `Title 5.epub` used to come out of review with their series set to their own diff --git a/backend/api/bindery.py b/backend/api/bindery.py index 6d60345..fc672b7 100644 --- a/backend/api/bindery.py +++ b/backend/api/bindery.py @@ -360,10 +360,15 @@ def bindery_accept( )) record_ko_hash(db, book.id, ko_partial_md5(dest), "raw") - # Word count (EPUB only) — parsed from the accepted file on disk. + # Word count + chapter map (EPUB only) — parsed from the accepted + # file on disk. Fixed-layout formats get an intrinsic page count. if suffix == "epub": - from backend.services.metadata import count_words_epub + from backend.services.metadata import count_words_epub, extract_chapters_epub + from backend.services.chapters import replace_book_chapters book.word_count = count_words_epub(dest) + replace_book_chapters(db, book.id, extract_chapters_epub(dest)) + elif suffix in ("pdf", "cbz", "cbr"): + book.page_count = meta.get("page_count") # Create tag records for tag_str in item.tags: diff --git a/backend/api/books.py b/backend/api/books.py index d512d7d..70e7329 100644 --- a/backend/api/books.py +++ b/backend/api/books.py @@ -1124,6 +1124,7 @@ def get_book_reading_stats( from backend.services.reading_stats import ( compute_book_reading_stats, compute_book_aggregate_stats, + compute_book_chapter_times, compute_book_page_intensity, ) @@ -1141,8 +1142,10 @@ def get_book_reading_stats( ) # Per-page intensity from imported KOReader page-stats (None if web-only reading) intensity = compute_book_page_intensity(db, user_id=current_user.id, book_id=book_id, tz_offset=tz_offset) + # Time per TOC chapter (None without a chapter map or page-stats) + chapters = compute_book_chapter_times(db, user_id=current_user.id, book_id=book_id) - return {"own": own, "aggregate": aggregate, "intensity": intensity} + return {"own": own, "aggregate": aggregate, "intensity": intensity, "chapters": chapters} class ManualSessionIn(PydanticBaseModel): @@ -2120,6 +2123,7 @@ def upload_book( language=meta.get("language"), year=meta.get("year"), word_count=meta.get("word_count"), + page_count=meta.get("page_count"), cover_path=meta.get("cover_path"), content_hash=content_hash, status="active", @@ -2129,6 +2133,9 @@ def upload_book( db.add(book) db.flush() + from backend.services.chapters import replace_book_chapters + replace_book_chapters(db, book.id, meta.get("_chapters")) + db.add(BookFile( book_id=book.id, file_path=str(dest.resolve()), @@ -2359,10 +2366,16 @@ def ingest_book( )) record_ko_hash(db, book.id, ko_partial_md5(dest), "raw") - # Word count (EPUB only) — parsed from the ingested file on disk. + # Word count + chapter map (EPUB only) — parsed from the ingested file on + # disk. Fixed-layout formats get an intrinsic page count instead. if suffix == "epub": - from backend.services.metadata import count_words_epub + from backend.services.metadata import count_words_epub, extract_chapters_epub + from backend.services.chapters import replace_book_chapters book.word_count = count_words_epub(dest) + replace_book_chapters(db, book.id, extract_chapters_epub(dest)) + elif suffix in ("pdf", "cbz", "cbr"): + from backend.services.metadata import count_pages_fixed_layout + book.page_count = count_pages_fixed_layout(dest) # Tags if meta.tags: diff --git a/backend/main.py b/backend/main.py index cce92d2..9c32cde 100644 --- a/backend/main.py +++ b/backend/main.py @@ -47,6 +47,7 @@ from backend.models.wish import Wish # noqa: F401 from backend.models.notification import Notification # noqa: F401 from backend.models.reading_goal import ReadingGoal # noqa: F401 +from backend.models.book import BookChapter # noqa: F401 @asynccontextmanager @@ -67,6 +68,11 @@ async def lifespan(app: FastAPI): if "word_count" not in cols: conn.execute(text("ALTER TABLE books ADD COLUMN word_count INTEGER")) conn.commit() + # Intrinsic PDF/CBZ page count (EPUB stays NULL by design) — filled at + # ingest, backfilled by the same admin job as word counts. + if "page_count" not in cols: + conn.execute(text("ALTER TABLE books ADD COLUMN page_count INTEGER")) + conn.commit() user_cols = {r[1] for r in conn.execute(text("PRAGMA table_info(users)")).fetchall()} if "role" not in user_cols: conn.execute(text("ALTER TABLE users ADD COLUMN role VARCHAR(16) NOT NULL DEFAULT 'guest'")) @@ -437,6 +443,7 @@ def _ingest_file(file_path: Path) -> tuple[Book, str, str | None] | None: language=meta.get("language"), year=meta.get("year"), word_count=meta.get("word_count"), + page_count=meta.get("page_count"), cover_path=cover_path, content_hash=content_hash, content_type=parsed.content_type, @@ -447,6 +454,9 @@ def _ingest_file(file_path: Path) -> tuple[Book, str, str | None] | None: db.add(book) db.flush() + from backend.services.chapters import replace_book_chapters + replace_book_chapters(db, book.id, meta.get("_chapters")) + db.add(BookFile( book_id=book.id, file_path=str(dest.resolve()), diff --git a/backend/models/book.py b/backend/models/book.py index 8c9fd02..ee4d2dc 100644 --- a/backend/models/book.py +++ b/backend/models/book.py @@ -28,6 +28,11 @@ class Book(Base): # ingest (or backfilled by the admin word-count job). NULL for PDF/CBZ or # not-yet-parsed books. Feeds words-read / true-WPM stats (Phase 4). word_count: Mapped[Optional[int]] = mapped_column(Integer) + # Intrinsic page count for PDF/CBZ/CBR only (fixed-layout formats). EPUB + # deliberately stays NULL — a reflowable page count is device/font-specific + # and must never be stored as a book property (KOReader page-stats carry + # their own total_pages per row for that). + page_count: Mapped[Optional[int]] = mapped_column(Integer) # Hardcover identity — book-level because a title's Hardcover book/edition is # user-independent. Matched lazily by the sync worker (ISBN first, strict # title+author search as fallback). match_method "none" + matched_at set = @@ -105,3 +110,27 @@ class BookTag(Base): source: Mapped[Optional[str]] = mapped_column(String(32)) # "google_books", "open_library", "user" book: Mapped["Book"] = relationship("Book", back_populates="tags") + + +class BookChapter(Base): + """One TOC chapter with device-independent boundaries. + + Boundaries are stored as fraction-of-book (by word offset), NOT page + numbers — pages are device/pagination-specific. Per-page dwell (KOReader + page-stats, which carry their own total_pages) maps into these buckets at + query time: fraction = page/total_pages. Extracted at ingest from the EPUB + TOC (or backfilled by the admin word-count job); fixed-layout formats have + no chapter map. + """ + __tablename__ = "book_chapters" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + book_id: Mapped[int] = mapped_column( + Integer, ForeignKey("books.id", ondelete="CASCADE"), nullable=False, index=True + ) + idx: Mapped[int] = mapped_column(Integer, nullable=False) # 0-based TOC order + title: Mapped[str] = mapped_column(String(512), nullable=False) + start_fraction: Mapped[float] = mapped_column(Float, nullable=False) # 0..1 + end_fraction: Mapped[float] = mapped_column(Float, nullable=False) # 0..1 + + book: Mapped["Book"] = relationship("Book") diff --git a/backend/services/chapters.py b/backend/services/chapters.py new file mode 100644 index 0000000..4678448 --- /dev/null +++ b/backend/services/chapters.py @@ -0,0 +1,27 @@ +"""Persist a book's chapter map (see BookChapter for the anchor design).""" +import logging + +from sqlalchemy.orm import Session + +from backend.models.book import BookChapter + +logger = logging.getLogger(__name__) + + +def replace_book_chapters(db: Session, book_id: int, chapters: list[dict] | None) -> int: + """Replace the stored chapter map for a book with the given extraction + result (as produced by metadata._extract_epub_chapters). An empty/None + extraction clears nothing — no TOC today doesn't invalidate a map extracted + from the same file yesterday. Returns the number of rows written.""" + if not chapters: + return 0 + db.query(BookChapter).filter(BookChapter.book_id == book_id).delete() + for c in chapters: + db.add(BookChapter( + book_id=book_id, + idx=c["idx"], + title=(c["title"] or "")[:512], + start_fraction=c["start_fraction"], + end_fraction=c["end_fraction"], + )) + return len(chapters) diff --git a/backend/services/metadata.py b/backend/services/metadata.py index 60dc795..cec9fed 100644 --- a/backend/services/metadata.py +++ b/backend/services/metadata.py @@ -103,6 +103,130 @@ def count_words_epub(path: Path) -> Optional[int]: return _count_words_from_zip(path) +def _extract_epub_chapters(book) -> list[dict]: + """TOC → device-independent chapter boundaries as fraction-of-book. + + Fractions come from cumulative word offsets of spine items: a chapter whose + TOC entry points into spine item k starts at words(items 0..k-1) / total. + Fragment anchors inside a file are ignored — top-level TOC entries almost + always sit at a file boundary, and per-file granularity is what the + time-per-chapter stat needs. Returns [] when there's no usable structure + (no TOC, unresolvable hrefs, or fewer than two distinct chapters). + """ + import ebooklib + + id_to_item = {item.get_id(): item for item in book.get_items_of_type(ebooklib.ITEM_DOCUMENT)} + spine_ids = [sid for sid, _linear in book.spine if sid in id_to_item] + if not spine_ids: + return [] + + counts: list[int] = [] + href_to_idx: dict[str, int] = {} + for i, sid in enumerate(spine_ids): + item = id_to_item[sid] + href_to_idx[item.get_name()] = i + try: + raw = item.get_content() + counts.append(count_words_text(_html_to_text(raw.decode("utf-8", "ignore")))) + except Exception: # noqa: BLE001 — unreadable spine item contributes no words + counts.append(0) + total = sum(counts) + if total <= 0: + return [] + cum = [0] + for c in counts: + cum.append(cum[-1] + c) + + def resolve(href: str) -> Optional[int]: + href = href.split("#", 1)[0] + if href in href_to_idx: + return href_to_idx[href] + # Tolerate OPF-dir path differences by unique basename. + base = href.rsplit("/", 1)[-1] + hits = [i for h, i in href_to_idx.items() if h.rsplit("/", 1)[-1] == base] + return hits[0] if len(hits) == 1 else None + + chapters: list[dict] = [] + for node in book.toc or []: + # Entries are Link or (Section, [children]); a Section may carry its own + # href, otherwise its first child anchors it. Only the top level counts — + # nested subsections are noise for a per-chapter time split. + head = node[0] if isinstance(node, tuple) else node + href = getattr(head, "href", None) + if not href and isinstance(node, tuple) and node[1]: + first = node[1][0] + first = first[0] if isinstance(first, tuple) else first + href = getattr(first, "href", None) + if not href: + continue + idx = resolve(href) + if idx is None: + continue + title = (getattr(head, "title", None) or "").strip() + chapters.append({ + "title": title or f"Chapter {len(chapters) + 1}", + "start_fraction": cum[idx] / total, + }) + + chapters.sort(key=lambda c: c["start_fraction"]) + out: list[dict] = [] + for c in chapters: + # Two TOC entries resolving to the same start (front-matter stubs, + # in-file subheadings) collapse to the first. + if out and c["start_fraction"] <= out[-1]["start_fraction"] + 1e-9: + continue + out.append(c) + if len(out) < 2: + return [] + # Front matter (nav doc, cover page, copyright) sits before the first TOC + # entry; fold it into chapter one so the map tiles the whole book. + out[0]["start_fraction"] = 0.0 + for i, c in enumerate(out): + c["idx"] = i + c["end_fraction"] = out[i + 1]["start_fraction"] if i + 1 < len(out) else 1.0 + return out + + +def count_pages_fixed_layout(path: Path) -> Optional[int]: + """Intrinsic page count for fixed-layout formats (PDF/CBZ/CBR), without + doing any of the cover/metadata work extract_metadata does. None for + reflowable or unreadable files.""" + fmt = get_format(path) + try: + if fmt == "pdf": + import fitz + with fitz.open(str(path)) as doc: + return len(doc) or None + if fmt == "cbz": + with zipfile.ZipFile(path) as zf: + n = sum(1 for name in zf.namelist() + if name.lower().endswith((".jpg", ".jpeg", ".png", ".webp")) + and not name.startswith("__MACOSX")) + return n or None + if fmt == "cbr": + import rarfile + with rarfile.RarFile(str(path)) as rf: + n = sum(1 for name in rf.namelist() + if name.lower().endswith((".jpg", ".jpeg", ".png", ".webp"))) + return n or None + except Exception as e: # noqa: BLE001 + logger.warning("page count error for %s: %s", path, e) + return None + + +def extract_chapters_epub(path: Path) -> list[dict]: + """Open an EPUB from disk and extract its chapter map. Used by the backfill + job; ingest reuses the already-open book via _extract_epub_chapters.""" + try: + from ebooklib import epub + + book = epub.read_epub(str(path), options={"ignore_ncx": True}) + return _extract_epub_chapters(book) + except Exception as e: # noqa: BLE001 — no chapters is always a valid outcome + logger.info("chapter extraction failed for %s: %s", path, e) + return [] + + def _opf_meta_by_name(book, name: str) -> Optional[str]: """Read an OPF2 value. @@ -258,6 +382,13 @@ def _first(items): if wc: meta["word_count"] = wc + # Chapter map (TOC → fraction boundaries) — same open book. Private + # key: creation sites persist it as BookChapter rows; API previews + # strip underscore keys. + chapters = _extract_epub_chapters(book) + if chapters: + meta["_chapters"] = chapters + # Cover extraction cover_id = None for item in book.get_metadata("OPF", "cover"): @@ -313,6 +444,11 @@ def extract_pdf(path: Path, covers_dir: Path) -> dict: if m: meta["year"] = int(m.group(1)) + # Intrinsic page count — PDFs are fixed-layout, so this is a real + # property of the book (unlike reflowable EPUB pagination). + if len(doc) > 0: + meta["page_count"] = len(doc) + # Cover: render first page as image if len(doc) > 0: page = doc[0] @@ -418,6 +554,11 @@ def extract_cbz(path: Path, covers_dir: Path) -> dict: ) if images: meta["_cover_data"] = zf.read(images[0]) + # Intrinsic page count: one image = one page. Counted images + # are ground truth; ComicInfo's PageCount is only a fallback. + meta["page_count"] = len(images) + elif meta.get("_page_count"): + meta["page_count"] = meta["_page_count"] except Exception as e: logger.warning("cbz extraction error for %s: %s", path, e) @@ -448,6 +589,9 @@ def extract_cbr(path: Path, covers_dir: Path) -> dict: ) if images: meta["_cover_data"] = rf.read(images[0]) + meta["page_count"] = len(images) + elif meta.get("_page_count"): + meta["page_count"] = meta["_page_count"] except Exception as e: logger.warning("cbr extraction error for %s: %s", path, e) diff --git a/backend/services/reading_stats.py b/backend/services/reading_stats.py index 1b9ac5a..bff0d69 100644 --- a/backend/services/reading_stats.py +++ b/backend/services/reading_stats.py @@ -6,6 +6,7 @@ """ from __future__ import annotations +from bisect import bisect_right from collections import defaultdict from datetime import datetime, timezone from typing import Optional @@ -416,6 +417,101 @@ def compute_book_reading_stats( # ── Per-book reading intensity (imported KOReader page-stats) ───────────────── +def compute_book_chapter_times( + db: Session, + *, + user_id: int, + book_id: int, +) -> Optional[list[dict]]: + """Time spent per chapter for one book: KOReader page-stat dwell mapped + into the book's TOC chapter boundaries. + + Chapters store device-independent fraction boundaries (see BookChapter); + each dwell row maps to a fraction via its own pagination + (``(page - 0.5) / total_pages``, mid-page so boundary pages land in the + chapter they mostly belong to). Returns None when the book has no chapter + map or the user has no page-stats for it — the caller hides the section. + """ + from backend.models.book import BookChapter + from backend.models.ko_stats import PageStat + + chapters = ( + db.query(BookChapter) + .filter(BookChapter.book_id == book_id) + .order_by(BookChapter.idx.asc()) + .all() + ) + if not chapters: + return None + + rows = ( + db.query(PageStat.page, PageStat.total_pages, PageStat.duration_seconds, + PageStat.start_time) + .filter( + PageStat.user_id == user_id, + PageStat.book_id == book_id, + PageStat.total_pages > 0, + ) + .all() + ) + if not rows: + return None + + starts = [c.start_fraction for c in chapters] + seconds = [0] * len(chapters) + stamps: list[list[tuple[int, int]]] = [[] for _ in chapters] # (ts, dur) + for page, total_pages, dur, start_time in rows: + if not total_pages or total_pages <= 0: + continue + frac = (page - 0.5) / total_pages + frac = min(max(frac, 0.0), 0.999999) + # bisect into the chapter whose start is the last one ≤ frac. Dwell + # before the first chapter (cover/front matter) folds into chapter 0. + i = bisect_right(starts, frac) - 1 + if i < 0: + i = 0 + d = int(dur or 0) + seconds[i] += d + ts = int(start_time or 0) + if ts: + stamps[i].append((ts, d)) + + if not any(seconds): + return None + + # WHEN a chapter was read: cluster its dwells into sittings with the same + # 30-minute-gap rule the session clustering uses. A chapter read across two + # evenings must be two sittings — a naive min–max range would render as a + # 28-hour "session". + GAP = 1800 + + def sittings_of(items: list[tuple[int, int]]) -> list[dict]: + if not items: + return [] + items.sort() + out: list[dict] = [] + start, end = items[0][0], items[0][0] + items[0][1] + for ts, d in items[1:]: + if ts - end > GAP: + out.append({"start_ts": start, "end_ts": end}) + start = ts + end = max(end, ts + d) + out.append({"start_ts": start, "end_ts": end}) + return out + + return [ + { + "idx": c.idx, + "title": c.title, + "start_fraction": c.start_fraction, + "end_fraction": c.end_fraction, + "seconds": seconds[i], + "sittings": sittings_of(stamps[i]), + } + for i, c in enumerate(chapters) + ] + + def compute_book_page_intensity( db: Session, *, diff --git a/backend/services/scanner.py b/backend/services/scanner.py index 72c7f26..73377a3 100644 --- a/backend/services/scanner.py +++ b/backend/services/scanner.py @@ -293,6 +293,7 @@ def _create_book_entry( language=meta.get("language"), year=meta.get("year"), word_count=meta.get("word_count"), + page_count=meta.get("page_count"), cover_path=meta.get("cover_path"), content_hash=content_hash, status="active", @@ -301,6 +302,9 @@ def _create_book_entry( db.add(book) db.flush() + from backend.services.chapters import replace_book_chapters + replace_book_chapters(db, book.id, meta.get("_chapters")) + if result is not None: result.added_ids.append(book.id) diff --git a/backend/services/word_count_job.py b/backend/services/word_count_job.py index 4cc4a2d..7ab9df6 100644 --- a/backend/services/word_count_job.py +++ b/backend/services/word_count_job.py @@ -32,8 +32,13 @@ from sqlalchemy import func from backend.core.database import SessionLocal -from backend.models.book import Book, BookFile -from backend.services.metadata import count_words_epub +from backend.models.book import Book, BookChapter, BookFile +from backend.services.chapters import replace_book_chapters +from backend.services.metadata import ( + count_pages_fixed_layout, + count_words_epub, + extract_chapters_epub, +) log = logging.getLogger(__name__) @@ -51,8 +56,9 @@ class WordCountAlreadyRunning(Exception): pass -def _epub_books() -> dict[int, tuple[Optional[int], int, str]]: - """One row per book that has an EPUB file: book_id → (word_count, size, path). +def _epub_books() -> dict[int, tuple[Optional[int], int, str, bool]]: + """One row per book that has an EPUB file: + book_id → (word_count, size, path, has_chapters). If a book has several EPUB files we keep the first.""" with SessionLocal() as db: rows = ( @@ -66,10 +72,34 @@ def _epub_books() -> dict[int, tuple[Optional[int], int, str]]: .filter(func.lower(BookFile.format) == "epub") .all() ) - out: dict[int, tuple[Optional[int], int, str]] = {} + chaptered = { + r[0] for r in db.query(BookChapter.book_id).distinct().all() + } + out: dict[int, tuple[Optional[int], int, str, bool]] = {} for book_id, wc, size, path in rows: if book_id not in out: - out[book_id] = (wc, size or 0, path) + out[book_id] = (wc, size or 0, path, book_id in chaptered) + return out + + +def _fixed_layout_books() -> list[tuple[int, str, int]]: + """Books with a PDF/CBZ/CBR file and no intrinsic page count yet.""" + with SessionLocal() as db: + rows = ( + db.query(Book.id, BookFile.file_path, BookFile.file_size) + .join(BookFile, BookFile.book_id == Book.id) + .filter( + Book.page_count.is_(None), + func.lower(BookFile.format).in_(("pdf", "cbz", "cbr")), + ) + .all() + ) + seen: set[int] = set() + out: list[tuple[int, str, int]] = [] + for book_id, path, size in rows: + if book_id not in seen: + seen.add(book_id) + out.append((book_id, path, size or 0)) return out @@ -78,15 +108,19 @@ def preflight() -> dict: books = _epub_books() total = len(books) pending = pending_bytes = 0 - for wc, size, _path in books.values(): - if wc is None: + for wc, size, _path, has_ch in books.values(): + if wc is None or not has_ch: pending += 1 pending_bytes += size + pages = _fixed_layout_books() return { "epub_total": total, "already_counted": total - pending, - "pending": pending, - "pending_bytes": pending_bytes, + "pending": pending + len(pages), + "pending_bytes": pending_bytes + sum(s for _, _, s in pages), + # split, for anyone curious (the UI shows the aggregate) + "pending_epubs": pending, + "pending_page_counts": len(pages), } @@ -139,12 +173,17 @@ def start(*, username: Optional[str] = None) -> dict: if _state.get("status") == "running": raise WordCountAlreadyRunning() - work: list[tuple[int, str, int]] = [] + # kind "epub": word count (if missing) + chapter map (if missing). + # kind "pages": intrinsic PDF/CBZ/CBR page count. + work: list[tuple[int, str, int, str, bool]] = [] total_bytes = 0 - for book_id, (wc, size, path) in _epub_books().items(): - if wc is None: - work.append((book_id, path, size)) + for book_id, (wc, size, path, has_ch) in _epub_books().items(): + if wc is None or not has_ch: + work.append((book_id, path, size, "epub", wc is None)) total_bytes += size + for book_id, path, size in _fixed_layout_books(): + work.append((book_id, path, size, "pages", False)) + total_bytes += size _cancel.clear() _state = { @@ -197,31 +236,52 @@ def _finish(status: str, error: Optional[str] = None) -> None: _state["error"] = error -def _run(work: list[tuple[int, str, int]]) -> None: +def _run(work: list[tuple[int, str, int, str, bool]]) -> None: try: - for book_id, path, size in work: + for book_id, path, size, kind, needs_words in work: if _cancel.is_set(): _finish("cancelled") return _set_current(path) + + if kind == "pages": + try: + pages = count_pages_fixed_layout(Path(path)) + if pages: + with SessionLocal() as db: + book = db.get(Book, book_id) + if book is not None: + book.page_count = pages + db.commit() + # Page counting is cheap and can't produce words — record it + # as counted so progress advances without skewing the totals. + _record(size=size, words=0 if pages else None, path=path) + except Exception: # noqa: BLE001 — one bad file never aborts the run + log.exception("word-count: page count error on book #%s", book_id) + _record(size=size, words=None, path=path) + continue + try: - words = count_words_epub(Path(path)) - except Exception as e: # noqa: BLE001 — one bad file never aborts the run + words = count_words_epub(Path(path)) if needs_words else None + chapters = extract_chapters_epub(Path(path)) + except Exception: # noqa: BLE001 — one bad file never aborts the run log.exception("word-count: error on book #%s", book_id) _record(size=size, words=None, path=path) continue - if words is not None: - try: - with SessionLocal() as db: - book = db.get(Book, book_id) - if book is not None: + try: + with SessionLocal() as db: + book = db.get(Book, book_id) + if book is not None: + if words is not None: book.word_count = words - db.commit() - except Exception: # noqa: BLE001 - log.exception("word-count: DB write failed for book #%s", book_id) - _record(size=size, words=None, path=path) - continue - _record(size=size, words=words, path=path) + replace_book_chapters(db, book_id, chapters) + db.commit() + except Exception: # noqa: BLE001 + log.exception("word-count: DB write failed for book #%s", book_id) + _record(size=size, words=None, path=path) + continue + # A chapters-only pass (words already stored) still counts as done. + _record(size=size, words=words if needs_words else 0, path=path) _finish("done") except Exception as e: # noqa: BLE001 log.exception("word-count: job crashed") diff --git a/frontend/src/pages/BookDetailPage.tsx b/frontend/src/pages/BookDetailPage.tsx index d9a1df6..500c47b 100644 --- a/frontend/src/pages/BookDetailPage.tsx +++ b/frontend/src/pages/BookDetailPage.tsx @@ -68,6 +68,51 @@ interface BookIntensity { reread_bins: number } +interface ChapterSitting { + start_ts: number + end_ts: number +} + +interface ChapterTime { + idx: number + title: string + start_fraction: number + end_fraction: number + seconds: number + sittings: ChapterSitting[] +} + +// When a chapter was actually read, honest about sittings: +// one sitting → "Jun 2, 17:40 – 18:54" +// two or three → "Jun 2 17:40–18:10 · Jun 3 21:00–21:34" +// more → "5 sittings · Jun 2 – Jun 8" +function chapterReadSpan(c: ChapterTime): string { + const s = c.sittings + if (!s || s.length === 0) return '' + const day = (ts: number) => + new Date(ts * 1000).toLocaleDateString(undefined, { day: 'numeric', month: 'short' }) + const clock = (ts: number) => + new Date(ts * 1000).toLocaleTimeString(undefined, { hour: '2-digit', minute: '2-digit' }) + if (s.length === 1) { + return `read ${day(s[0].start_ts)}, ${clock(s[0].start_ts)} – ${clock(s[0].end_ts)}` + } + if (s.length <= 3) { + return 'read ' + s.map(x => `${day(x.start_ts)} ${clock(x.start_ts)}–${clock(x.end_ts)}`).join(' · ') + } + return `${s.length} sittings · ${day(s[0].start_ts)} – ${day(s[s.length - 1].end_ts)}` +} + +// Short variant for the line-chart tooltip (dates only, no clock times). +function chapterReadSpanShort(c: ChapterTime): string { + const s = c.sittings + if (!s || s.length === 0) return '' + const day = (ts: number) => + new Date(ts * 1000).toLocaleDateString(undefined, { day: 'numeric', month: 'short' }) + const a = day(s[0].start_ts) + const b = day(s[s.length - 1].end_ts) + return a === b ? `read ${a}` : `read ${a} – ${b}` +} + interface ReadingStatsResponse { own: BookReadingStats aggregate: { @@ -76,6 +121,7 @@ interface ReadingStatsResponse { distinct_readers: number } | null intensity: BookIntensity | null + chapters: ChapterTime[] | null } async function downloadFile(book: BookDetail, f: BookFile) { @@ -134,7 +180,10 @@ export function BookDetailPage() { const [confirmingHighlight, setConfirmingHighlight] = useState(null) const [highlightsOpen, setHighlightsOpen] = useState(true) const [readingStats, setReadingStats] = useState(null) - const [statsOpen, setStatsOpen] = useState(true) + // Collapsed state persists across books/visits — the stats tower (hero + + // intensity + time-per-chapter) has grown tall enough that "keep it closed" + // is a real preference, not a per-page whim. + const [statsOpen, setStatsOpen] = useState(() => localStorage.getItem('tome_book_stats_open') !== '0') const [descExpanded, setDescExpanded] = useState(false) const [facets, setFacets] = useState({ authors: [], series: [], tags: [], formats: [] }) const [draftTags, setDraftTags] = useState([]) @@ -690,11 +739,13 @@ export function BookDetailPage() { api.get(`/books/${id}/reading-stats?tz_offset=${new Date().getTimezoneOffset()}`).then(setReadingStats).catch(() => {}) const hasReadingData = readingStats && (readingStats.own.sessions > 0 || !!readingStats.intensity) const statsFull = readingStats ? ( -
-
+ // mt-6 matches the Description/Details section rhythm — this block used to + // sit tight under the rating row (mt-1) before the description moved above it. +
+
) : ( // No history yet — the manual-tracking entry point (paper / un-synced device). @@ -1178,13 +1232,16 @@ export function BookDetailPage() {
)}
- {/* Right column: title → status → stats → description → details grid → highlights */} + {/* Right column: title → status → description → stats → details grid → highlights. + Description before stats: the page leads with what the book IS; + the telemetry tower (hero + intensity + chapters) comes after and + collapses as one. */}
{titleBlock} {statusProgressBlock} {ratingBlock} - {statsFull} {descriptionBlock} + {statsFull} {metadataGridFull} {highlightsBlock}
@@ -1403,6 +1460,119 @@ function IntensityBlock({ data }: { data: BookIntensity }) { ) } +// ── Time per chapter: page-stat dwell bucketed into the book's TOC boundaries ───── + +function ChapterTimesBlock({ chapters }: { chapters: ChapterTime[] }) { + const [showDetails, setShowDetails] = useState(false) + const max = Math.max(...chapters.map(c => c.seconds), 1) + const total = chapters.reduce((s, c) => s + c.seconds, 0) + return ( +
+
+ +

Time per chapter

+ +
+

{formatDuration(total)} across {chapters.length} chapters

+
+ + {showDetails && ( +
    + {chapters.map(c => ( +
  • + + {c.title} + + + 0 ? `${Math.max(Math.round((c.seconds / max) * 100), 3)}%` : '0%' }} + /> + + + {c.seconds <= 0 ? '—' : c.seconds < 60 ? '<1m' : formatDuration(c.seconds)} + +
  • + ))} +
+ )} + +
+ ) +} + +// Line/area rendering of time-per-chapter: one series, so no legend — the +// block title names it. Hover snaps to the nearest chapter and shows a styled +// tooltip (name + time) with a marker dot; both are HTML overlays positioned +// in percent, because the SVG stretches (preserveAspectRatio="none") and +// would distort anything drawn inside it. +function ChapterLineChart({ chapters }: { chapters: ChapterTime[] }) { + const W = 600, H = 80, PAD = 2 + const n = chapters.length + const max = Math.max(...chapters.map(c => c.seconds), 1) + const [hover, setHover] = useState(null) + const x = (i: number) => n === 1 ? W / 2 : PAD + (i / (n - 1)) * (W - 2 * PAD) + const y = (s: number) => H - PAD - (s / max) * (H - 2 * PAD) + const pts = chapters.map((c, i) => `${x(i).toFixed(1)},${y(c.seconds).toFixed(1)}`).join(' ') + const area = `${PAD},${H - PAD} ${pts} ${(W - PAD).toFixed(1)},${H - PAD}` + + const onMove = (e: React.MouseEvent) => { + const rect = e.currentTarget.getBoundingClientRect() + const frac = (e.clientX - rect.left) / rect.width + setHover(Math.min(Math.max(Math.round(frac * (n - 1)), 0), n - 1)) + } + const h = hover !== null ? chapters[hover] : null + const hx = hover !== null ? (x(hover) / W) * 100 : 0 + const hy = h ? (y(h.seconds) / H) * 100 : 0 + + return ( +
+
setHover(null)}> + + + + + + {h && ( + <> + +
+ {h.title} + + {' · '}{h.seconds <= 0 ? 'not read' : h.seconds < 60 ? '<1m' : formatDuration(h.seconds)} + {chapterReadSpanShort(h) && ` · ${chapterReadSpanShort(h)}`} + +
+ + )} +
+
+ ch. 1 + ch. {n} +
+
+ ) +} + // ── Hero layout: large time-read headline + activity chart, bottom-pinned date/span stats ───── // Manual reading-log entry point: log a session by hand (paper / un-synced diff --git a/tests/test_book_chapters.py b/tests/test_book_chapters.py new file mode 100644 index 0000000..cea2afa --- /dev/null +++ b/tests/test_book_chapters.py @@ -0,0 +1,215 @@ +"""Phase 3 plumbing: chapter map extraction (BookChapter), fixed-layout page +counts, and the time-per-chapter stat built on both.""" +import io +import zipfile +from pathlib import Path + +from ebooklib import epub + +from backend.models.book import BookChapter +from backend.models.ko_stats import PageStat +from backend.services.metadata import ( + count_pages_fixed_layout, + extract_chapters_epub, + extract_metadata, +) +from backend.services.reading_stats import compute_book_chapter_times + + +def _make_epub(path: Path, chapters: list[tuple[str, str]]) -> None: + """chapters: list of (title, body-text).""" + book = epub.EpubBook() + book.set_identifier("ch-test") + book.set_title("Chapter Test") + book.set_language("en") + items = [] + for i, (title, txt) in enumerate(chapters): + c = epub.EpubHtml(title=title, file_name=f"c{i}.xhtml", lang="en") + c.content = f"

{title}

{txt}

" + book.add_item(c) + items.append(c) + book.toc = tuple(items) + book.add_item(epub.EpubNcx()) + book.add_item(epub.EpubNav()) + book.spine = ["nav"] + items + epub.write_epub(str(path), book) + + +def _make_cbz(path: Path, pages: int) -> None: + # 1x1 white JPEG, tiny but real enough for a namelist count + jpeg = bytes.fromhex( + "ffd8ffe000104a46494600010100000100010000ffdb004300ffffffffffffffff" + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + "ffffffffffffffffffffffffc00b080001000101011100ffc40014000100000000" + "00000000000000000000000009ffc40014100100000000000000000000000000000" + "000ffda0008010100003f0037ffd9" + ) + with zipfile.ZipFile(path, "w") as zf: + for i in range(pages): + zf.writestr(f"{i:03d}.jpg", jpeg) + + +# ── extraction ─────────────────────────────────────────────────────────────── + +class TestChapterExtraction: + def test_fractions_follow_word_weight(self, tmp_path): + # ch1 has 100 words, ch2 has 300 → ch2 starts at 0.25 of the book. + path = tmp_path / "b.epub" + _make_epub(path, [ + ("One", " ".join(f"w{i}" for i in range(100))), + ("Two", " ".join(f"w{i}" for i in range(300))), + ]) + chapters = extract_chapters_epub(path) + assert [c["title"] for c in chapters] == ["One", "Two"] + assert chapters[0]["start_fraction"] == 0.0 + assert abs(chapters[1]["start_fraction"] - 0.25) < 0.03 # h1 text adds a little + assert chapters[0]["end_fraction"] == chapters[1]["start_fraction"] + assert chapters[1]["end_fraction"] == 1.0 + assert [c["idx"] for c in chapters] == [0, 1] + + def test_single_chapter_toc_is_no_structure(self, tmp_path): + path = tmp_path / "b.epub" + _make_epub(path, [("Only", "some words here")]) + assert extract_chapters_epub(path) == [] + + def test_no_toc_returns_empty(self, tmp_path): + path = tmp_path / "b.epub" + book = epub.EpubBook() + book.set_identifier("x") + book.set_title("No TOC") + book.set_language("en") + c = epub.EpubHtml(title="c", file_name="c.xhtml", lang="en") + c.content = "

text body words

" + book.add_item(c) + book.add_item(epub.EpubNcx()) + book.add_item(epub.EpubNav()) + book.spine = ["nav", c] + epub.write_epub(str(path), book) + assert extract_chapters_epub(path) == [] + + def test_extract_metadata_carries_private_chapters_key(self, tmp_path): + path = tmp_path / "b.epub" + _make_epub(path, [("One", "alpha " * 50), ("Two", "beta " * 50)]) + meta = extract_metadata(path, tmp_path) + assert len(meta["_chapters"]) == 2 + + +class TestFixedLayoutPageCount: + def test_cbz_counts_images(self, tmp_path): + path = tmp_path / "c.cbz" + _make_cbz(path, 7) + assert count_pages_fixed_layout(path) == 7 + meta = extract_metadata(path, tmp_path) + assert meta["page_count"] == 7 + + def test_epub_gets_no_page_count(self, tmp_path): + path = tmp_path / "b.epub" + _make_epub(path, [("One", "alpha"), ("Two", "beta")]) + assert count_pages_fixed_layout(path) is None + meta = extract_metadata(path, tmp_path) + assert "page_count" not in meta + + +# ── time-per-chapter ───────────────────────────────────────────────────────── + +def _chapter(db, book_id, idx, title, start, end): + db.add(BookChapter(book_id=book_id, idx=idx, title=title, + start_fraction=start, end_fraction=end)) + + +def _dwell(db, user_id, book_id, page, total, seconds, start_time=1_700_000_000): + db.add(PageStat(user_id=user_id, book_id=book_id, page=page, + total_pages=total, start_time=start_time + page, + duration_seconds=seconds, device="kindle")) + + +class TestChapterTimes: + def test_buckets_dwell_by_fraction(self, db, admin_user, make_book): + user, _ = admin_user + book = make_book(title="Bucketed") + _chapter(db, book.id, 0, "One", 0.0, 0.5) + _chapter(db, book.id, 1, "Two", 0.5, 1.0) + # 100-page pagination: pages 1-50 → ch One, 51-100 → ch Two + _dwell(db, user.id, book.id, page=10, total=100, seconds=60) + _dwell(db, user.id, book.id, page=40, total=100, seconds=30) + _dwell(db, user.id, book.id, page=80, total=100, seconds=45) + db.commit() + + out = compute_book_chapter_times(db, user_id=user.id, book_id=book.id) + assert [c["seconds"] for c in out] == [90, 45] + assert [c["title"] for c in out] == ["One", "Two"] + # When the chapter was read: pages 10 and 40 are 30s apart in _dwell's + # stamping (base + page), well under the 30-min gap → ONE sitting + # covering both; chapter Two has a single dwell. + assert out[0]["sittings"] == [{"start_ts": 1_700_000_010, "end_ts": 1_700_000_070}] + assert out[1]["sittings"] == [{"start_ts": 1_700_000_080, "end_ts": 1_700_000_125}] + + def test_sittings_split_on_thirty_minute_gap(self, db, admin_user, make_book): + user, _ = admin_user + book = make_book(title="TwoSittings") + _chapter(db, book.id, 0, "One", 0.0, 1.0) + _chapter(db, book.id, 1, "Two", 1.0, 1.0) + base = 1_700_000_000 + # _dwell stamps start_time + page. Two dwells ~5 minutes apart (one + # sitting), then one ~2 hours later (a second sitting). + _dwell(db, user.id, book.id, page=1, total=100, seconds=60, start_time=base - 1) # ts base + _dwell(db, user.id, book.id, page=2, total=100, seconds=60, start_time=base + 298) # ts base+300 + _dwell(db, user.id, book.id, page=3, total=100, seconds=60, start_time=base + 7197) # ts base+7200 + db.commit() + + out = compute_book_chapter_times(db, user_id=user.id, book_id=book.id) + s = out[0]["sittings"] + assert len(s) == 2 + assert s[0]["start_ts"] == base and s[0]["end_ts"] == base + 360 + assert s[1]["start_ts"] == base + 7200 and s[1]["end_ts"] == base + 7260 + + def test_mixed_paginations_map_independently(self, db, admin_user, make_book): + user, _ = admin_user + book = make_book(title="Repaged") + _chapter(db, book.id, 0, "One", 0.0, 0.5) + _chapter(db, book.id, 1, "Two", 0.5, 1.0) + # Same physical spot (~75% in) under two different paginations. + _dwell(db, user.id, book.id, page=75, total=100, seconds=10) + _dwell(db, user.id, book.id, page=300, total=400, seconds=20) + db.commit() + + out = compute_book_chapter_times(db, user_id=user.id, book_id=book.id) + assert out[1]["seconds"] == 30 + assert out[0]["seconds"] == 0 + + def test_front_matter_folds_into_first_chapter(self, db, admin_user, make_book): + user, _ = admin_user + book = make_book(title="Fronted") + # First chapter starts at 0.1 (cover/front matter before it). + _chapter(db, book.id, 0, "One", 0.1, 1.0) + _chapter(db, book.id, 1, "Two", 1.0, 1.0) + _dwell(db, user.id, book.id, page=1, total=100, seconds=15) + db.commit() + + out = compute_book_chapter_times(db, user_id=user.id, book_id=book.id) + assert out[0]["seconds"] == 15 + + def test_none_without_chapters_or_stats(self, db, admin_user, make_book): + user, _ = admin_user + no_chapters = make_book(title="NoCh") + _dwell(db, user.id, no_chapters.id, page=1, total=10, seconds=5) + no_stats = make_book(title="NoSt") + _chapter(db, no_stats.id, 0, "One", 0.0, 0.5) + _chapter(db, no_stats.id, 1, "Two", 0.5, 1.0) + db.commit() + + assert compute_book_chapter_times(db, user_id=user.id, book_id=no_chapters.id) is None + assert compute_book_chapter_times(db, user_id=user.id, book_id=no_stats.id) is None + + def test_endpoint_carries_chapters_block(self, db, client, admin_user, make_book): + user, _ = admin_user + book = make_book(title="Endpointed") + _chapter(db, book.id, 0, "One", 0.0, 0.5) + _chapter(db, book.id, 1, "Two", 0.5, 1.0) + _dwell(db, user.id, book.id, page=20, total=100, seconds=120) + db.commit() + + r = client.get(f"/api/books/{book.id}/reading-stats") + assert r.status_code == 200, r.text + chapters = r.json()["chapters"] + assert chapters[0]["seconds"] == 120 and chapters[1]["seconds"] == 0