feat(stats): time per chapter — TOC chapter maps, fixed-layout page counts, book-page layout

CHANGELOG.md

@@ -6,6 +6,20 @@ All notable changes to Tome are documented here. Format loosely follows
 
 ## [Unreleased]
 
+### Added
+- **Time per chapter.** Book pages now show where your reading time went
+  chapter by chapter: the book's table of contents is extracted at ingest into
+  device-independent chapter boundaries, and KOReader per-page reading data is
+  mapped into them — robust to font/margin changes, since every page record is
+  interpreted against its own pagination. Renders on the book detail page next
+  to the reading-intensity curve whenever both a chapter map and synced page
+  data exist. Existing libraries get chapter maps via the Admin → Word Counts
+  backfill, which now also extracts chapters (and intrinsic page counts, below)
+  in the same pass.
+- **Intrinsic page counts for fixed-layout books.** PDFs and comic archives now
+  store their real page count at ingest (EPUB deliberately doesn't — reflowable
+  pagination is not a property of the book). Backfilled by the same admin job.
+
 ### Fixed
 - **Bindery no longer invents a series from the filename.** Prose ebooks named
   `Title 5.epub` used to come out of review with their series set to their own

backend/api/bindery.py

@@ -360,10 +360,15 @@ def bindery_accept(
             ))
             record_ko_hash(db, book.id, ko_partial_md5(dest), "raw")
 
-            # Word count (EPUB only) — parsed from the accepted file on disk.
+            # Word count + chapter map (EPUB only) — parsed from the accepted
+            # file on disk. Fixed-layout formats get an intrinsic page count.
             if suffix == "epub":
-                from backend.services.metadata import count_words_epub
+                from backend.services.metadata import count_words_epub, extract_chapters_epub
+                from backend.services.chapters import replace_book_chapters
                 book.word_count = count_words_epub(dest)
+                replace_book_chapters(db, book.id, extract_chapters_epub(dest))
+            elif suffix in ("pdf", "cbz", "cbr"):
+                book.page_count = meta.get("page_count")
 
             # Create tag records
             for tag_str in item.tags:

backend/api/books.py

@@ -1124,6 +1124,7 @@ def get_book_reading_stats(
     from backend.services.reading_stats import (
         compute_book_reading_stats,
         compute_book_aggregate_stats,
+        compute_book_chapter_times,
         compute_book_page_intensity,
     )
 
@@ -1141,8 +1142,10 @@ def get_book_reading_stats(
     )
     # Per-page intensity from imported KOReader page-stats (None if web-only reading)
     intensity = compute_book_page_intensity(db, user_id=current_user.id, book_id=book_id, tz_offset=tz_offset)
+    # Time per TOC chapter (None without a chapter map or page-stats)
+    chapters = compute_book_chapter_times(db, user_id=current_user.id, book_id=book_id)
 
-    return {"own": own, "aggregate": aggregate, "intensity": intensity}
+    return {"own": own, "aggregate": aggregate, "intensity": intensity, "chapters": chapters}
 
 
 class ManualSessionIn(PydanticBaseModel):
@@ -2120,6 +2123,7 @@ def upload_book(
         language=meta.get("language"),
         year=meta.get("year"),
         word_count=meta.get("word_count"),
+        page_count=meta.get("page_count"),
         cover_path=meta.get("cover_path"),
         content_hash=content_hash,
         status="active",
@@ -2129,6 +2133,9 @@ def upload_book(
     db.add(book)
     db.flush()
 
+    from backend.services.chapters import replace_book_chapters
+    replace_book_chapters(db, book.id, meta.get("_chapters"))
+
     db.add(BookFile(
         book_id=book.id,
         file_path=str(dest.resolve()),
@@ -2359,10 +2366,16 @@ def ingest_book(
     ))
     record_ko_hash(db, book.id, ko_partial_md5(dest), "raw")
 
-    # Word count (EPUB only) — parsed from the ingested file on disk.
+    # Word count + chapter map (EPUB only) — parsed from the ingested file on
+    # disk. Fixed-layout formats get an intrinsic page count instead.
     if suffix == "epub":
-        from backend.services.metadata import count_words_epub
+        from backend.services.metadata import count_words_epub, extract_chapters_epub
+        from backend.services.chapters import replace_book_chapters
         book.word_count = count_words_epub(dest)
+        replace_book_chapters(db, book.id, extract_chapters_epub(dest))
+    elif suffix in ("pdf", "cbz", "cbr"):
+        from backend.services.metadata import count_pages_fixed_layout
+        book.page_count = count_pages_fixed_layout(dest)
 
     # Tags
     if meta.tags:

backend/main.py

@@ -47,6 +47,7 @@
 from backend.models.wish import Wish  # noqa: F401
 from backend.models.notification import Notification  # noqa: F401
 from backend.models.reading_goal import ReadingGoal  # noqa: F401
+from backend.models.book import BookChapter  # noqa: F401
 
 
 @asynccontextmanager
@@ -67,6 +68,11 @@ async def lifespan(app: FastAPI):
         if "word_count" not in cols:
             conn.execute(text("ALTER TABLE books ADD COLUMN word_count INTEGER"))
             conn.commit()
+        # Intrinsic PDF/CBZ page count (EPUB stays NULL by design) — filled at
+        # ingest, backfilled by the same admin job as word counts.
+        if "page_count" not in cols:
+            conn.execute(text("ALTER TABLE books ADD COLUMN page_count INTEGER"))
+            conn.commit()
         user_cols = {r[1] for r in conn.execute(text("PRAGMA table_info(users)")).fetchall()}
         if "role" not in user_cols:
             conn.execute(text("ALTER TABLE users ADD COLUMN role VARCHAR(16) NOT NULL DEFAULT 'guest'"))
@@ -437,6 +443,7 @@ def _ingest_file(file_path: Path) -> tuple[Book, str, str | None] | None:
                 language=meta.get("language"),
                 year=meta.get("year"),
                 word_count=meta.get("word_count"),
+                page_count=meta.get("page_count"),
                 cover_path=cover_path,
                 content_hash=content_hash,
                 content_type=parsed.content_type,
@@ -447,6 +454,9 @@ def _ingest_file(file_path: Path) -> tuple[Book, str, str | None] | None:
             db.add(book)
             db.flush()
 
+            from backend.services.chapters import replace_book_chapters
+            replace_book_chapters(db, book.id, meta.get("_chapters"))
+
             db.add(BookFile(
                 book_id=book.id,
                 file_path=str(dest.resolve()),

backend/models/book.py

@@ -28,6 +28,11 @@ class Book(Base):
     # ingest (or backfilled by the admin word-count job). NULL for PDF/CBZ or
     # not-yet-parsed books. Feeds words-read / true-WPM stats (Phase 4).
     word_count: Mapped[Optional[int]] = mapped_column(Integer)
+    # Intrinsic page count for PDF/CBZ/CBR only (fixed-layout formats). EPUB
+    # deliberately stays NULL — a reflowable page count is device/font-specific
+    # and must never be stored as a book property (KOReader page-stats carry
+    # their own total_pages per row for that).
+    page_count: Mapped[Optional[int]] = mapped_column(Integer)
     # Hardcover identity — book-level because a title's Hardcover book/edition is
     # user-independent. Matched lazily by the sync worker (ISBN first, strict
     # title+author search as fallback). match_method "none" + matched_at set =
@@ -105,3 +110,27 @@ class BookTag(Base):
     source: Mapped[Optional[str]] = mapped_column(String(32))  # "google_books", "open_library", "user"
 
     book: Mapped["Book"] = relationship("Book", back_populates="tags")
+
+
+class BookChapter(Base):
+    """One TOC chapter with device-independent boundaries.
+
+    Boundaries are stored as fraction-of-book (by word offset), NOT page
+    numbers — pages are device/pagination-specific. Per-page dwell (KOReader
+    page-stats, which carry their own total_pages) maps into these buckets at
+    query time: fraction = page/total_pages. Extracted at ingest from the EPUB
+    TOC (or backfilled by the admin word-count job); fixed-layout formats have
+    no chapter map.
+    """
+    __tablename__ = "book_chapters"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    book_id: Mapped[int] = mapped_column(
+        Integer, ForeignKey("books.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    idx: Mapped[int] = mapped_column(Integer, nullable=False)          # 0-based TOC order
+    title: Mapped[str] = mapped_column(String(512), nullable=False)
+    start_fraction: Mapped[float] = mapped_column(Float, nullable=False)  # 0..1
+    end_fraction: Mapped[float] = mapped_column(Float, nullable=False)    # 0..1
+
+    book: Mapped["Book"] = relationship("Book")

backend/services/chapters.py

@@ -0,0 +1,27 @@
+"""Persist a book's chapter map (see BookChapter for the anchor design)."""
+import logging
+
+from sqlalchemy.orm import Session
+
+from backend.models.book import BookChapter
+
+logger = logging.getLogger(__name__)
+
+
+def replace_book_chapters(db: Session, book_id: int, chapters: list[dict] | None) -> int:
+    """Replace the stored chapter map for a book with the given extraction
+    result (as produced by metadata._extract_epub_chapters). An empty/None
+    extraction clears nothing — no TOC today doesn't invalidate a map extracted
+    from the same file yesterday. Returns the number of rows written."""
+    if not chapters:
+        return 0
+    db.query(BookChapter).filter(BookChapter.book_id == book_id).delete()
+    for c in chapters:
+        db.add(BookChapter(
+            book_id=book_id,
+            idx=c["idx"],
+            title=(c["title"] or "")[:512],
+            start_fraction=c["start_fraction"],
+            end_fraction=c["end_fraction"],
+        ))
+    return len(chapters)

backend/services/metadata.py

@@ -103,6 +103,130 @@ def count_words_epub(path: Path) -> Optional[int]:
     return _count_words_from_zip(path)
 
 
+def _extract_epub_chapters(book) -> list[dict]:
+    """TOC → device-independent chapter boundaries as fraction-of-book.
+
+    Fractions come from cumulative word offsets of spine items: a chapter whose
+    TOC entry points into spine item k starts at words(items 0..k-1) / total.
+    Fragment anchors inside a file are ignored — top-level TOC entries almost
+    always sit at a file boundary, and per-file granularity is what the
+    time-per-chapter stat needs. Returns [] when there's no usable structure
+    (no TOC, unresolvable hrefs, or fewer than two distinct chapters).
+    """
+    import ebooklib
+
+    id_to_item = {item.get_id(): item for item in book.get_items_of_type(ebooklib.ITEM_DOCUMENT)}
+    spine_ids = [sid for sid, _linear in book.spine if sid in id_to_item]
+    if not spine_ids:
+        return []
+
+    counts: list[int] = []
+    href_to_idx: dict[str, int] = {}
+    for i, sid in enumerate(spine_ids):
+        item = id_to_item[sid]
+        href_to_idx[item.get_name()] = i
+        try:
+            raw = item.get_content()
+            counts.append(count_words_text(_html_to_text(raw.decode("utf-8", "ignore"))))
+        except Exception:  # noqa: BLE001 — unreadable spine item contributes no words
+            counts.append(0)
+    total = sum(counts)
+    if total <= 0:
+        return []
+    cum = [0]
+    for c in counts:
+        cum.append(cum[-1] + c)
+
+    def resolve(href: str) -> Optional[int]:
+        href = href.split("#", 1)[0]
+        if href in href_to_idx:
+            return href_to_idx[href]
+        # Tolerate OPF-dir path differences by unique basename.
+        base = href.rsplit("/", 1)[-1]
+        hits = [i for h, i in href_to_idx.items() if h.rsplit("/", 1)[-1] == base]
+        return hits[0] if len(hits) == 1 else None
+
+    chapters: list[dict] = []
+    for node in book.toc or []:
+        # Entries are Link or (Section, [children]); a Section may carry its own
+        # href, otherwise its first child anchors it. Only the top level counts —
+        # nested subsections are noise for a per-chapter time split.
+        head = node[0] if isinstance(node, tuple) else node
+        href = getattr(head, "href", None)
+        if not href and isinstance(node, tuple) and node[1]:
+            first = node[1][0]
+            first = first[0] if isinstance(first, tuple) else first
+            href = getattr(first, "href", None)
+        if not href:
+            continue
+        idx = resolve(href)
+        if idx is None:
+            continue
+        title = (getattr(head, "title", None) or "").strip()
+        chapters.append({
+            "title": title or f"Chapter {len(chapters) + 1}",
+            "start_fraction": cum[idx] / total,
+        })
+
+    chapters.sort(key=lambda c: c["start_fraction"])
+    out: list[dict] = []
+    for c in chapters:
+        # Two TOC entries resolving to the same start (front-matter stubs,
+        # in-file subheadings) collapse to the first.
+        if out and c["start_fraction"] <= out[-1]["start_fraction"] + 1e-9:
+            continue
+        out.append(c)
+    if len(out) < 2:
+        return []
+    # Front matter (nav doc, cover page, copyright) sits before the first TOC
+    # entry; fold it into chapter one so the map tiles the whole book.
+    out[0]["start_fraction"] = 0.0
+    for i, c in enumerate(out):
+        c["idx"] = i
+        c["end_fraction"] = out[i + 1]["start_fraction"] if i + 1 < len(out) else 1.0
+    return out
+
+
+def count_pages_fixed_layout(path: Path) -> Optional[int]:
+    """Intrinsic page count for fixed-layout formats (PDF/CBZ/CBR), without
+    doing any of the cover/metadata work extract_metadata does. None for
+    reflowable or unreadable files."""
+    fmt = get_format(path)
+    try:
+        if fmt == "pdf":
+            import fitz
+            with fitz.open(str(path)) as doc:
+                return len(doc) or None
+        if fmt == "cbz":
+            with zipfile.ZipFile(path) as zf:
+                n = sum(1 for name in zf.namelist()
+                        if name.lower().endswith((".jpg", ".jpeg", ".png", ".webp"))
+                        and not name.startswith("__MACOSX"))
+                return n or None
+        if fmt == "cbr":
+            import rarfile
+            with rarfile.RarFile(str(path)) as rf:
+                n = sum(1 for name in rf.namelist()
+                        if name.lower().endswith((".jpg", ".jpeg", ".png", ".webp")))
+                return n or None
+    except Exception as e:  # noqa: BLE001
+        logger.warning("page count error for %s: %s", path, e)
+    return None
+
+
+def extract_chapters_epub(path: Path) -> list[dict]:
+    """Open an EPUB from disk and extract its chapter map. Used by the backfill
+    job; ingest reuses the already-open book via _extract_epub_chapters."""
+    try:
+        from ebooklib import epub
+
+        book = epub.read_epub(str(path), options={"ignore_ncx": True})
+        return _extract_epub_chapters(book)
+    except Exception as e:  # noqa: BLE001 — no chapters is always a valid outcome
+        logger.info("chapter extraction failed for %s: %s", path, e)
+        return []
+
+
 def _opf_meta_by_name(book, name: str) -> Optional[str]:
     """Read an OPF2 <meta name="..." content="..."/> value.
 
@@ -258,6 +382,13 @@ def _first(items):
         if wc:
             meta["word_count"] = wc
 
+        # Chapter map (TOC → fraction boundaries) — same open book. Private
+        # key: creation sites persist it as BookChapter rows; API previews
+        # strip underscore keys.
+        chapters = _extract_epub_chapters(book)
+        if chapters:
+            meta["_chapters"] = chapters
+
         # Cover extraction
         cover_id = None
         for item in book.get_metadata("OPF", "cover"):
@@ -313,6 +444,11 @@ def extract_pdf(path: Path, covers_dir: Path) -> dict:
             if m:
                 meta["year"] = int(m.group(1))
 
+        # Intrinsic page count — PDFs are fixed-layout, so this is a real
+        # property of the book (unlike reflowable EPUB pagination).
+        if len(doc) > 0:
+            meta["page_count"] = len(doc)
+
         # Cover: render first page as image
         if len(doc) > 0:
             page = doc[0]
@@ -418,6 +554,11 @@ def extract_cbz(path: Path, covers_dir: Path) -> dict:
             )
             if images:
                 meta["_cover_data"] = zf.read(images[0])
+                # Intrinsic page count: one image = one page. Counted images
+                # are ground truth; ComicInfo's PageCount is only a fallback.
+                meta["page_count"] = len(images)
+            elif meta.get("_page_count"):
+                meta["page_count"] = meta["_page_count"]
     except Exception as e:
         logger.warning("cbz extraction error for %s: %s", path, e)
 
@@ -448,6 +589,9 @@ def extract_cbr(path: Path, covers_dir: Path) -> dict:
             )
             if images:
                 meta["_cover_data"] = rf.read(images[0])
+                meta["page_count"] = len(images)
+            elif meta.get("_page_count"):
+                meta["page_count"] = meta["_page_count"]
     except Exception as e:
         logger.warning("cbr extraction error for %s: %s", path, e)