diff --git a/CHANGELOG.md b/CHANGELOG.md index 388d0f8..45af953 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,28 @@ All notable changes to Tome are documented here. Format loosely follows ## [Unreleased] ### Added +- **Search your library from the device (KOReader plugin, build 33).** The + series browser gains a "Search library…" entry: submit-based free-text search + over title, author, and series, with your last searches one tap away. Results + open the same drill-down list as a series — tap to download, hold to set read + status. +- **Browse by author on the device (build 33).** A new author axis in the + series browser — the natural way into standalone books, which previously all + hid behind the single "No Series" bucket. Downloads file each book by its own + series/author identity, exactly like the web. +- **Set read status from the device browser (build 33).** Hold any book row in + a series, author, or search list to mark it unread / reading / read on the + server — no need to open the book. The lists also show each book's current + status alongside the existing "on device" marker. + +- **Position pull strategy (KOReader plugin, build 32).** What happens on book + open when the server position differs from the device is now configurable, + like stock KOSync: "Server position is ahead" and "Server position is + behind" each offer *Ask before jumping / Jump automatically / Do nothing* + (TomeSync settings). Defaults keep the historic behavior — forward jumps + happen silently, backward jumps never — and the ask-dialog is deferred a + moment after open so it can never swallow a "on book opening" profile the + way the old layout-reset bug did. - **Highlights page polish.** The deferred cluster from the original commonplace-book release: an **only-notes filter** (show just the highlights carrying your own notes — composes with search and on-this-day), a real @@ -29,6 +51,25 @@ All notable changes to Tome are documented here. Format loosely follows pagination is not a property of the book). Backfilled by the same admin job. ### Fixed +- **A fast server clock can no longer silently swallow highlights (build 32).** + Highlight edits and deletes made in the web reader are stamped with the + server's clock; on a device whose clock runs behind (UTC container vs local + device time), those stamps landed in the device's future and outranked every + later local change — a re-highlight after a web delete just vanished until + the clocks crossed. The plugin now stamps its wall-clock on every highlight + sync and the server shifts the stamps it minted itself into that device's + clock frame, both when comparing and in what it returns; the plugin + additionally refuses to store any stamp from the future. Older plugins keep + the previous behavior. +- **The TomeSync plugin no longer bloats KOReader's global settings file + (build 32).** Its data tables (book map, pending sessions and ratings, sync + baselines, repair aliases) — which grow with your library and were parsed by + KOReader at every boot — moved into a dedicated `tomesync_state.lua`, + migrated automatically and crash-safely on first launch. State for books no + longer on the device is pruned; pending queues are never pruned. +- **The plugin's series list no longer runs one query per series.** Loading the + series browser was an N+1 that scaled with the library; it's now a single + query with the same response. - **The Highlights "copy all as Markdown" export never worked.** It requested more highlights than the API's page cap allowed, got a 422, and failed without any feedback. The cap is raised, the export stays under it, and a diff --git a/backend/api/annotations.py b/backend/api/annotations.py index f6fb2ef..7633709 100644 --- a/backend/api/annotations.py +++ b/backend/api/annotations.py @@ -176,7 +176,8 @@ def create_annotation( if len(text_) > 20_000: raise HTTPException(status_code=422, detail="highlighted_text too long") - now = (payload.datetime or "").strip()[:19] or func_now_str() + client_dt = (payload.datetime or "").strip()[:19] + now = client_dt or func_now_str() row = Annotation( user_id=current_user.id, book_id=payload.book_id, @@ -188,6 +189,9 @@ def create_annotation( color=(payload.color or None), koreader_datetime=now, koreader_datetime_updated=now, + # Stamp origin: only a server-clock fallback is "server-minted" — + # plugin syncs shift such stamps into the device's clock frame. + server_minted=not client_dt, ) db.add(row) db.commit() @@ -225,14 +229,20 @@ def edit_annotation( # The edit must be STRICTLY newer than the current mtime to win LWW on # devices; guard against a server clock at/behind the device's wall-clock. new_mtime = func_now_str() + minted = True if new_mtime <= (row.effective_mtime or ""): from datetime import datetime as _dt, timedelta as _td try: base = _dt.strptime(row.effective_mtime, "%Y-%m-%d %H:%M:%S") new_mtime = (base + _td(seconds=1)).strftime("%Y-%m-%d %H:%M:%S") + # Bumped off the row's existing mtime, so the new stamp lives in + # whatever frame that stamp was in (device unless it was itself + # server-minted) — inherit, don't assume server frame. + minted = row.server_minted except ValueError: pass row.koreader_datetime_updated = new_mtime + row.server_minted = minted db.commit() db.refresh(row) return _annotation_out(row) @@ -269,7 +279,12 @@ def delete_annotation( # drop copies with mtime <= tombstone — so maxing with effective_mtime guarantees # the delete holds against the exact copy that was deleted, while a genuinely # newer re-add (strictly greater mtime) still wins and clears the tombstone. - now = max(func_now_str(), annotation.effective_mtime) + server_now = func_now_str() + now = max(server_now, annotation.effective_mtime) + # The stamp is only "server-minted" (shifted into device frames on plugin + # sync) when the server clock actually won the max — if the highlight's own + # device mtime won, the stamp is already in that device's frame. + minted = now == server_now and now != annotation.effective_mtime db.delete(annotation) @@ -286,10 +301,11 @@ def delete_annotation( # Keep the latest deletion time so a stale re-add can't slip under it. if now > (tomb.client_deleted_at or ""): tomb.client_deleted_at = now + tomb.server_minted = minted else: db.add(AnnotationTombstone( user_id=current_user.id, book_id=book_id, anchor=anchor, - client_deleted_at=now, + client_deleted_at=now, server_minted=minted, )) db.commit() diff --git a/backend/api/tome_sync.py b/backend/api/tome_sync.py index d5d7cfd..6c83af5 100644 --- a/backend/api/tome_sync.py +++ b/backend/api/tome_sync.py @@ -6,7 +6,7 @@ import io import logging import zipfile -from datetime import datetime +from datetime import datetime, timedelta from typing import Optional from pathlib import Path @@ -47,7 +47,16 @@ # BUILD 31: half-star ratings — rating_baseline entries split into # {remote, device} so a Tome half-star rounded onto the whole-star sidecar is # never pushed back as a "local edit" (old {rating=...} entries migrate on read). -TOMESYNC_PLUGIN_BUILD = 31 +# BUILD 32: hygiene batch — clock-offset guard (device_time on annotation syncs; +# server-minted stamps shifted into the device frame; future-stamp clamps), +# pull-conflict strategy settings (forward/backward × prompt/silent/never), and +# a dedicated tomesync_state.lua for the data tables (migrated out of +# G_reader_settings, pruned for books no longer on disk). +# BUILD 33: catalog batch — device search (submit-based + recent searches), +# author browse axis, read-status write-back (hold a book row); the shared +# _bookListMenu drill-down shows per-book status markers. Server side: series +# list N+1 fixed, /tome-sync/{authors,author-books,search} + PUT status. +TOMESYNC_PLUGIN_BUILD = 33 TOMESYNC_PLUGIN_SEMVER = "1.8.0" TOMESYNC_PLUGIN_VERSION = str(TOMESYNC_PLUGIN_BUILD) @@ -538,6 +547,10 @@ class DeletedAnchor(PydanticBaseModel): class SyncAnnotationsRequest(PydanticBaseModel): upserts: list[AnnotationItem] = [] deletes: list[DeletedAnchor] = [] + # Device wall-clock at request time ("%Y-%m-%d %H:%M:%S"). Lets the server + # compute this device's clock offset and shift server-minted LWW stamps into + # the device's frame — see _clock_offset_seconds. + device_time: Optional[str] = None # KOReader's Lua rapidjson encodes an empty table as a JSON object ({}), not an # array. Coerce that back to an empty list so an empty upserts/deletes is valid. @@ -547,7 +560,49 @@ def _empty_obj_to_list(cls, v): return [] if v in (None, {}) else v -def _serialize_annotation(a: Annotation) -> dict: +# ── Clock-offset guard ──────────────────────────────────────────────────────── +# Annotation LWW stamps are plain wall-clock strings compared lexicographically. +# Stamps a DEVICE minted are in that device's frame (cross-device skew is a +# documented, accepted edge). Stamps the SERVER minted (web create/edit/delete) +# are in the server's frame — and a server clock ahead of a device makes those +# stamps land in the device's *future*, silently outranking every later local +# edit until the device clock catches up. Fix: the device stamps its wall-clock +# on sync requests; the server shifts every server-minted stamp into the +# device's frame, both in comparisons and in the response it returns. + +_KO_DT_FMT = "%Y-%m-%d %H:%M:%S" +# Below this, treat the clocks as synchronized: request latency and second +# truncation produce small spurious offsets, and shifting by them would churn +# stamps for correctly-configured setups. +_CLOCK_OFFSET_TOLERANCE_S = 120 + + +def _clock_offset_seconds(device_time: Optional[str]) -> int: + """Seconds the server clock is AHEAD of the device clock (0 = in sync).""" + if not device_time: + return 0 + try: + dev = datetime.strptime(device_time.strip()[:19], _KO_DT_FMT) + except ValueError: + return 0 + offset = round((datetime.now() - dev).total_seconds()) + return offset if abs(offset) >= _CLOCK_OFFSET_TOLERANCE_S else 0 + + +def _shift_ko_dt(stamp: Optional[str], seconds: int) -> Optional[str]: + """Shift a KOReader wall-clock string by N seconds; unparseable → unchanged.""" + if not stamp or not seconds: + return stamp + try: + base = datetime.strptime(stamp.strip()[:19], _KO_DT_FMT) + except ValueError: + return stamp + return (base + timedelta(seconds=seconds)).strftime(_KO_DT_FMT) + + +def _serialize_annotation(a: Annotation, offset: int = 0) -> dict: + # Server-minted stamps travel to the device in the DEVICE's clock frame. + shift = -offset if a.server_minted else 0 return { "id": a.id, "anchor": a.anchor, @@ -556,8 +611,8 @@ def _serialize_annotation(a: Annotation) -> dict: "note": a.note, "chapter": a.chapter, "color": a.color, - "datetime": a.koreader_datetime, - "datetime_updated": a.koreader_datetime_updated, + "datetime": _shift_ko_dt(a.koreader_datetime, shift), + "datetime_updated": _shift_ko_dt(a.koreader_datetime_updated, shift), "updated_at": a.updated_at.isoformat() + "Z", } @@ -579,15 +634,20 @@ def _annotation_state(db: Session, user_id: int, book_id: int): return alive, tombs -def _annotation_response(db: Session, user_id: int, book_id: int, **extra) -> dict: +def _annotation_response(db: Session, user_id: int, book_id: int, offset: int = 0, **extra) -> dict: alive, tombs = _annotation_state(db, user_id, book_id) rows = sorted(alive.values(), key=lambda a: (a.koreader_datetime or "", a.id)) return { "book_id": book_id, - "annotations": [_serialize_annotation(a) for a in rows], + "annotations": [_serialize_annotation(a, offset) for a in rows], "tombstones": [ - {"anchor": t.anchor, "deleted_at": t.client_deleted_at} for t in tombs.values() + { + "anchor": t.anchor, + "deleted_at": _shift_ko_dt(t.client_deleted_at, -offset if t.server_minted else 0), + } + for t in tombs.values() ], + "server_time": datetime.now().strftime(_KO_DT_FMT), **extra, } @@ -612,19 +672,27 @@ def sync_annotations( alive, tombs = _annotation_state(db, user.id, book_id) created = updated = deleted = skipped = 0 + # Server clock minus device clock; server-minted stamps are compared (and + # returned) in the device's frame so a fast server clock can't make web + # actions permanently outrank the device's next local edit. + offset = _clock_offset_seconds(body.device_time) + + def in_device_frame(stamp: Optional[str], minted: bool) -> str: + return _shift_ko_dt(stamp, -offset if minted else 0) or "" + for item in body.upserts: if not item.anchor: continue tomb = tombs.get(item.anchor) # A re-add only wins over a delete if it's strictly newer than the delete. - if tomb and item.mtime <= (tomb.client_deleted_at or ""): + if tomb and item.mtime <= in_device_frame(tomb.client_deleted_at, tomb.server_minted): skipped += 1 continue if tomb: db.delete(tomb); tombs.pop(item.anchor, None) row = alive.get(item.anchor) if row: - if item.mtime >= row.effective_mtime: # newer edit wins + if item.mtime >= in_device_frame(row.effective_mtime, row.server_minted): # newer edit wins row.anchor_end = item.anchor_end or row.anchor_end row.highlighted_text = item.highlighted_text row.note = item.note @@ -632,6 +700,7 @@ def sync_annotations( row.color = item.color row.koreader_datetime = item.datetime or row.koreader_datetime row.koreader_datetime_updated = item.mtime or row.koreader_datetime_updated + row.server_minted = False # stamp is now device-authored updated += 1 else: skipped += 1 @@ -660,7 +729,7 @@ def sync_annotations( continue row = alive.get(d.anchor) # If a live edit is newer than this delete, the edit wins — keep it. - if row and row.effective_mtime > (d.datetime or ""): + if row and in_device_frame(row.effective_mtime, row.server_minted) > (d.datetime or ""): skipped += 1 continue if row: @@ -668,8 +737,9 @@ def sync_annotations( deleted += 1 tomb = tombs.get(d.anchor) if tomb: - if (d.datetime or "") > (tomb.client_deleted_at or ""): + if (d.datetime or "") > in_device_frame(tomb.client_deleted_at, tomb.server_minted): tomb.client_deleted_at = d.datetime + tomb.server_minted = False # stamp is now device-authored else: db.add(AnnotationTombstone( user_id=user.id, book_id=book_id, anchor=d.anchor, @@ -678,7 +748,7 @@ def sync_annotations( db.commit() return _annotation_response( - db, user.id, book_id, + db, user.id, book_id, offset=offset, applied={"created": created, "updated": updated, "deleted": deleted, "skipped": skipped}, ) @@ -686,16 +756,51 @@ def sync_annotations( @router.get("/tome-sync/annotations/{book_id}") def get_annotations_plugin( book_id: int, + device_time: Optional[str] = None, db: Session = Depends(get_db), user: User = Depends(_get_api_key_user), ): """Full annotation state (alive + tombstones) for this user+book — what the plugin pulls and merges on book open.""" - return _annotation_response(db, user.id, book_id) + return _annotation_response(db, user.id, book_id, offset=_clock_offset_seconds(device_time)) # ── Series endpoints (API-key-authed, for the plugin) ──────────────────────── +def _status_map(db: Session, user_id: int, book_ids: list[int]) -> dict[int, str]: + """book_id -> reading status for this user, one query.""" + if not book_ids: + return {} + rows = ( + db.query(UserBookStatus.book_id, UserBookStatus.status) + .filter(UserBookStatus.user_id == user_id, UserBookStatus.book_id.in_(book_ids)) + .all() + ) + return {bid: status for bid, status in rows if status} + + +def _book_entry(b: Book, status_map: dict[int, str]) -> dict: + """The book shape the plugin's volume-list menus consume. `series` and + `status` are additive (build 33+); older plugins ignore them.""" + entry = { + "id": b.id, + "title": b.title, + "series_index": b.series_index, + "author": b.author, + "book_type": b.book_type.slug if b.book_type else None, + "status": status_map.get(b.id, "unread"), + "files": [ + {"id": f.id, "format": f.format, "file_size": f.file_size} + for f in b.files + ], + } + # Only include series when real — JSON null crashes the KOReader menus + # (rapidjson decodes it to a truthy userdata sentinel). + if b.series: + entry["series"] = b.series + return entry + + @router.get("/tome-sync/series") def list_series( db: Session = Depends(get_db), @@ -705,59 +810,172 @@ def list_series( # Only count/expose books this user is allowed to see. book_visibility_filter # uses correlated EXISTS subqueries (no joins), so it doesn't duplicate rows # under the group_by, and returns True (no-op) for admins. + # + # One query for everything: ordered by (series, series_index, title), the + # first row seen per series IS its first book — the old per-series + # first_book query was an N+1 that scaled with the library. visibility = book_visibility_filter(db, user) rows = ( - db.query(Book.series, func.count(Book.id).label("book_count")) + db.query(Book.series, Book.id, Book.author) .filter(Book.status == "active", Book.series.isnot(None), visibility) - .group_by(Book.series) - .order_by(Book.series) + .order_by(Book.series.asc(), + Book.series_index.asc().nullslast(), Book.title.asc()) .all() ) result = [] - for series_name, book_count in rows: - first_book = ( - db.query(Book) - .filter(Book.status == "active", Book.series == series_name, visibility) - .order_by(Book.series_index.asc().nullslast(), Book.title.asc()) - .first() - ) - entry = { - "name": series_name, - "book_count": book_count, - "first_book_id": first_book.id if first_book else None, - } - # Only include author when it's a real string. Emitting JSON null here - # crashes the KOReader series browser, because rapidjson decodes null to - # a (truthy) userdata sentinel that the plugin then tries to concatenate. - if first_book and first_book.author: - entry["author"] = first_book.author - result.append(entry) + by_name: dict[str, dict] = {} + for series_name, book_id, author in rows: + entry = by_name.get(series_name) + if entry is None: + entry = {"name": series_name, "book_count": 0, "first_book_id": book_id} + # Only include author when it's a real string. Emitting JSON null here + # crashes the KOReader series browser, because rapidjson decodes null to + # a (truthy) userdata sentinel that the plugin then tries to concatenate. + if author: + entry["author"] = author + by_name[series_name] = entry + result.append(entry) + entry["book_count"] += 1 # Append the unserialized bucket last, mirroring backend/api/books.py, so the # plugin's series browser exposes a single "No Series" entry through which # standalone books can be browsed and downloaded. - unserialized_count = ( - db.query(func.count(Book.id)) + unserialized = ( + db.query(Book.id) .filter(Book.status == "active", Book.series.is_(None), visibility) - .scalar() + .order_by(Book.id) + .all() ) - if unserialized_count: - first_unserialized = ( - db.query(Book) - .filter(Book.status == "active", Book.series.is_(None), visibility) - .order_by(Book.id) - .first() - ) + if unserialized: result.append({ "name": "__unserialized__", - "book_count": unserialized_count, - "first_book_id": first_unserialized.id if first_unserialized else None, + "book_count": len(unserialized), + "first_book_id": unserialized[0][0], }) return result +@router.get("/tome-sync/authors") +def list_authors( + db: Session = Depends(get_db), + user: User = Depends(_get_api_key_user), +): + """List authors for the plugin's author browse axis — the natural way into + standalone books, which the series browser lumps into one No Series bucket.""" + visibility = book_visibility_filter(db, user) + rows = ( + db.query(Book.author, func.count(Book.id)) + .filter(Book.status == "active", Book.author.isnot(None), Book.author != "", visibility) + .group_by(Book.author) + .order_by(Book.author.asc()) + .all() + ) + result = [{"name": name, "book_count": count} for name, count in rows] + unknown = ( + db.query(func.count(Book.id)) + .filter(Book.status == "active", + (Book.author.is_(None)) | (Book.author == ""), visibility) + .scalar() + ) + if unknown: + result.append({"name": "__unknown__", "book_count": unknown}) + return result + + +@router.get("/tome-sync/author-books") +def get_author_books( + author: str, + db: Session = Depends(get_db), + user: User = Depends(_get_api_key_user), +): + """All of one author's visible books, shaped like a series volume list so + the plugin reuses the same drill-down menu. Query param (not a path + segment): author names contain slashes, dots, and everything else.""" + visibility = book_visibility_filter(db, user) + if author == "__unknown__": + author_filter = (Book.author.is_(None)) | (Book.author == "") + else: + author_filter = Book.author == author + books = ( + db.query(Book) + .options(joinedload(Book.files), joinedload(Book.book_type)) + .filter(Book.status == "active", author_filter, visibility) + .order_by(Book.series.asc().nullslast(), + Book.series_index.asc().nullslast(), Book.title.asc()) + .all() + ) + smap = _status_map(db, user.id, [b.id for b in books]) + return {"author": author, "books": [_book_entry(b, smap) for b in books]} + + +@router.get("/tome-sync/search") +def search_books( + q: str, + db: Session = Depends(get_db), + user: User = Depends(_get_api_key_user), +): + """Free-text search over title/author/series for the plugin (LIKE terms, + all must match). Same book shape as the series drill-down.""" + terms = [t for t in q.strip().split() if t] + if not terms: + return {"query": q, "total": 0, "books": []} + visibility = book_visibility_filter(db, user) + query = ( + db.query(Book) + .options(joinedload(Book.files), joinedload(Book.book_type)) + .filter(Book.status == "active", visibility) + ) + for t in terms: + like = f"%{t}%" + query = query.filter( + Book.title.ilike(like) | Book.author.ilike(like) | Book.series.ilike(like) + ) + total = query.count() + books = ( + query.order_by(Book.series.asc().nullslast(), + Book.series_index.asc().nullslast(), Book.title.asc()) + .limit(50) + .all() + ) + smap = _status_map(db, user.id, [b.id for b in books]) + return {"query": q, "total": total, "books": [_book_entry(b, smap) for b in books]} + + +class StatusUpdate(PydanticBaseModel): + status: str + + +@router.put("/tome-sync/status/{book_id}") +def put_reading_status( + book_id: int, + body: StatusUpdate, + db: Session = Depends(get_db), + user: User = Depends(_get_api_key_user), +): + """Set unread/reading/read from the device's volume list (write-back). + + A deliberate user action, so it writes status directly — unlike telemetry + (position/stats), which only ever *suggests* status via the sticky rule.""" + if body.status not in ("unread", "reading", "read"): + raise HTTPException(status_code=422, detail="status must be unread|reading|read") + book = db.get(Book, book_id) + if not book or book.status != "active" or not user_can_see_book(db, user, book): + raise HTTPException(status_code=404, detail="Book not found") + row = ( + db.query(UserBookStatus) + .filter(UserBookStatus.user_id == user.id, UserBookStatus.book_id == book_id) + .first() + ) + if row is None: + row = UserBookStatus(user_id=user.id, book_id=book_id) + db.add(row) + row.status = body.status + db.commit() + return {"ok": True, "book_id": book_id, "status": body.status} + + @router.get("/tome-sync/series/{book_id}") def get_series_books( book_id: int, @@ -793,23 +1011,11 @@ def get_series_books( # below, which newer plugins prefer when filing downloads. book_type_slug = books[0].book_type.slug if books and books[0].book_type else "book" + smap = _status_map(db, user.id, [b.id for b in books]) return { "series_name": series_name, "book_type": book_type_slug, - "books": [ - { - "id": b.id, - "title": b.title, - "series_index": b.series_index, - "author": b.author, - "book_type": b.book_type.slug if b.book_type else None, - "files": [ - {"id": f.id, "format": f.format, "file_size": f.file_size} - for f in b.files - ], - } - for b in books - ], + "books": [_book_entry(b, smap) for b in books], } @@ -1243,6 +1449,9 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: local socketutil = require("socketutil") local Dispatcher = require("dispatcher") local Event = require("ui/event") +local LuaSettings = require("luasettings") +local DataStorage = require("datastorage") +local ButtonDialog = require("ui/widget/buttondialog") -- ── Register in wrench menu (tools tab, after calibre) ────────────────────── -- Runs once per KOReader process via require() caching. @@ -1290,6 +1499,8 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: -- ReaderReady event reaches more than one live TomeSync instance for the same -- open. Cleared in onCloseDocument so an immediate reopen still inits. local last_session_init = {{ book_id = nil, at = 0 }} +-- Once-per-process guard for the state prune (init runs per reader instance). +local state_pruned = false -- ── HTTP client ────────────────────────────────────────────────────────────── @@ -1620,8 +1831,15 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: self.progress_start = nil self.last_progress = nil self.enabled = true - self.book_map = G_reader_settings:readSetting("tomesync_book_map") or {{}} - self.pending_sessions = G_reader_settings:readSetting("tomesync_pending_sessions") or {{}} + -- Dedicated state file: the plugin's data tables live in their own + -- LuaSettings file, NOT in G_reader_settings — KOReader parses the global + -- settings file at every boot, and these tables grow with the library. + -- (tomesync_update stays global: the frozen shim reads it and is never + -- replaced by self-update.) + self.state = LuaSettings:open(DataStorage:getSettingsDir() .. "/tomesync_state.lua") + self:_migrateState() + self.book_map = self.state:readSetting("tomesync_book_map") or {{}} + self.pending_sessions = self.state:readSetting("tomesync_pending_sessions") or {{}} -- Send-to-KOReader inbox (beta): enabled only if the server reports the -- feature; count drives the menu badge. Populated by the launch poll below. self.inbox_enabled = false @@ -1629,28 +1847,29 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: self.inbox_items = {{}} -- Web-adoption ledger: real_anchor -> provisional "web:" anchor, persisted so a -- failed push retries next sync (baseline alone would swallow the adoption). - self.adopt_pending = G_reader_settings:readSetting("tomesync_adopt_pending") or {{}} + self.adopt_pending = self.state:readSetting("tomesync_adopt_pending") or {{}} -- "|" -> {{ anchor, anchor_end }} of the SERVER -- identity for foreign highlights that had to be re-anchored on this copy -- (see _applyForeign). Keys are book-scoped: xPointers are only unique -- WITHIN a book, and structurally common positions (p[1]/text().0) would -- otherwise collide across books and mistranslate pushes. Persisted so -- repairs survive restarts. - self.repair_map = G_reader_settings:readSetting("tomesync_repair_map") or {{}} + self.repair_map = self.state:readSetting("tomesync_repair_map") or {{}} self._heartbeat_armed = false self._heartbeat_task = function() self:_heartbeatNow() end -- Per-book annotation sync baseline: book_id -> {{ anchor -> mtime }} as of last -- sync. Lets a diff tell "I deleted this" from "this is new from another device". - self.annot_baseline = G_reader_settings:readSetting("tomesync_annot_baseline") or {{}} + self.annot_baseline = self.state:readSetting("tomesync_annot_baseline") or {{}} -- Per-book rating sync baseline: book_id (string) -> {{ rating=, review= }} as of -- the last reconcile. Lets a diff tell which side (device or Tome) changed. - self.rating_baseline = G_reader_settings:readSetting("tomesync_rating_baseline") or {{}} + self.rating_baseline = self.state:readSetting("tomesync_rating_baseline") or {{}} -- Ratings set offline (or lost to a server error) that never reached Tome. -- Keyed by book_id (string) so re-rating the same book before a flush keeps -- only the latest value. Flushed on resume / Sync now / close like sessions: -- the per-book open/close push alone misses a book you rate and never reopen -- (e.g. a finished book), so the rating would otherwise sit unsent forever. - self.pending_ratings = G_reader_settings:readSetting("tomesync_pending_ratings") or {{}} + self.pending_ratings = self.state:readSetting("tomesync_pending_ratings") or {{}} + self:_pruneState() self:onDispatcherRegisterActions() self.ui.menu:registerToMainMenu(self) logger.info("TomeSync: init complete, menu registered,", @@ -1687,6 +1906,98 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: end end +-- The data tables that live in the dedicated state file (tomesync_update and +-- the boolean preferences stay in G_reader_settings — the frozen shim reads +-- the former, and the latter are what user-settings files are for). +local STATE_KEYS = {{ + "tomesync_book_map", "tomesync_pending_sessions", "tomesync_adopt_pending", + "tomesync_repair_map", "tomesync_annot_baseline", "tomesync_rating_baseline", + "tomesync_pending_ratings", +}} + +function TomeSync:_saveState(key, value) + self.state:saveSetting(key, value) + -- G_reader_settings flushed on app close; our file must flush itself. Save + -- sites are already at meaningful boundaries (sync done, queue changed), so + -- write-through is the right durability trade for a file this small. + self.state:flush() +end + +function TomeSync:_migrateState() + -- One-time move of the data tables out of G_reader_settings. Crash-safe + -- order: write + flush the new file FIRST, delete the old keys after — a + -- crash in between leaves a harmless duplicate, and the marker branch + -- below re-deletes leftovers on the next boot. + if self.state:readSetting("migrated_from_global") then + local leftover = false + for _, k in ipairs(STATE_KEYS) do + if G_reader_settings:has(k) then + G_reader_settings:delSetting(k) + leftover = true + end + end + if leftover then G_reader_settings:flush() end + return + end + local found = false + for _, k in ipairs(STATE_KEYS) do + local v = G_reader_settings:readSetting(k) + if v ~= nil then + self.state:saveSetting(k, v) + found = true + end + end + self.state:saveSetting("migrated_from_global", true) + self.state:flush() + for _, k in ipairs(STATE_KEYS) do G_reader_settings:delSetting(k) end + G_reader_settings:flush() + if found then + logger.info("TomeSync: migrated plugin state to tomesync_state.lua") + end +end + +function TomeSync:_pruneState() + -- Drop per-book state whose file is gone so the state file can't grow + -- unboundedly. Queues (pending_sessions/pending_ratings) are never pruned — + -- they are owed to the server regardless of the local file's fate. Baseline + -- loss is delete-safe by construction: deletes are only ever pushed FROM + -- baseline entries, so a wrongly-pruned baseline can only cause a harmless + -- re-upsert echo, never a delete. + if state_pruned then return end + state_pruned = true + local changed = false + local ids = {{}} + for path, id in pairs(self.book_map) do + if lfs.attributes(path, "mode") == "file" then + ids[tostring(id)] = true + else + self.book_map[path] = nil + changed = true + end + end + local function pruneById(tbl) + for key in pairs(tbl) do + -- repair_map keys are "|"; baselines use "" + local id = key:match("^(%d+)|") or key + if not ids[id] then + tbl[key] = nil + changed = true + end + end + end + pruneById(self.annot_baseline) + pruneById(self.rating_baseline) + pruneById(self.repair_map) + if changed then + self.state:saveSetting("tomesync_book_map", self.book_map) + self.state:saveSetting("tomesync_annot_baseline", self.annot_baseline) + self.state:saveSetting("tomesync_rating_baseline", self.rating_baseline) + self.state:saveSetting("tomesync_repair_map", self.repair_map) + self.state:flush() + logger.info("TomeSync: pruned state for books no longer on disk") + end +end + function TomeSync:onDispatcherRegisterActions() Dispatcher:registerAction("tome_open_menu", {{ category = "none", @@ -1850,7 +2161,7 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: while #self.pending_sessions > 50 do table.remove(self.pending_sessions, 1) end - G_reader_settings:saveSetting("tomesync_pending_sessions", self.pending_sessions) + self:_saveState("tomesync_pending_sessions", self.pending_sessions) logger.info("TomeSync: session queued for retry, pending:", #self.pending_sessions) end end @@ -2034,7 +2345,7 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: end self.pending_sessions = remaining - G_reader_settings:saveSetting("tomesync_pending_sessions", remaining) + self:_saveState("tomesync_pending_sessions", remaining) if #remaining == 0 then logger.info("TomeSync: all pending sessions flushed") else @@ -2107,7 +2418,7 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: if rok and result and type(rcode) == "number" and rcode == 200 and result.book_id then self.book_id = result.book_id self.book_map[doc.file] = self.book_id - G_reader_settings:saveSetting("tomesync_book_map", self.book_map) + self:_saveState("tomesync_book_map", self.book_map) logger.info("TomeSync: resolved to book_id", self.book_id) else logger.dbg("TomeSync: could not resolve", filename) @@ -2131,7 +2442,16 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: if ok and pos and code == 200 then local server_pct = pos.percentage or 0 local local_pct = self:_getCurrentPercentage() + -- Pull-conflict strategy (like stock kosync): forward and backward + -- pulls each get prompt / silent / never. Defaults keep the historic + -- behavior: forward silent, backward never. + local mode = nil if server_pct > (local_pct + 0.01) and server_pct < 0.99 then + mode = G_reader_settings:readSetting("tomesync_pull_forward") or "silent" + elseif server_pct < (local_pct - 0.01) and server_pct > 0.01 then + mode = G_reader_settings:readSetting("tomesync_pull_backward") or "never" + end + if mode == "silent" then self.progress_start = server_pct -- Must be a toast (Notification), not an InfoMessage: this shows -- right when Profiles auto-exec ("on book opening") dispatches its @@ -2145,20 +2465,26 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: ), timeout = 3, }}) - if self.ui and self.ui.rolling then - if type(pos.progress) == "string" and pos.progress:sub(1, 1) == "/" then - pcall(function() - self.ui.rolling:onGotoXPointer(pos.progress, pos.progress) - end) - else - -- Not a crengine xpointer (e.g. the web reader stores a - -- foliate epubcfi here) — onGotoXPointer with it lands on - -- page 1, so jump by percentage instead. - pcall(function() - self.ui.rolling:onGotoPercent(server_pct * 100) - end) - end - end + self:_gotoServerPosition(pos, server_pct) + elseif mode == "prompt" then + self.progress_start = local_pct + -- Deferred: a ConfirmBox is a non-toast window, and showing one at + -- open time would eat the Profiles auto-exec dispatch exactly like + -- the InfoMessage bug above. 1.5s lets the open settle first. + UIManager:scheduleIn(1.5, function() + if not self.ui or not self.ui.document then return end + UIManager:show(ConfirmBox:new{{ + text = string.format( + "TomeSync: Server position is at %.0f%% (this device: %.0f%%).\\nJump there?", + server_pct * 100, local_pct * 100 + ), + ok_text = "Jump", + ok_callback = function() + self.progress_start = server_pct + self:_gotoServerPosition(pos, server_pct) + end, + }}) + end) else self.progress_start = local_pct end @@ -2168,6 +2494,22 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: self.last_progress = self.progress_start end +function TomeSync:_gotoServerPosition(pos, server_pct) + if not (self.ui and self.ui.rolling) then return end + if type(pos.progress) == "string" and pos.progress:sub(1, 1) == "/" then + pcall(function() + self.ui.rolling:onGotoXPointer(pos.progress, pos.progress) + end) + else + -- Not a crengine xpointer (e.g. the web reader stores a + -- foliate epubcfi here) — onGotoXPointer with it lands on + -- page 1, so jump by percentage instead. + pcall(function() + self.ui.rolling:onGotoPercent(server_pct * 100) + end) + end +end + function TomeSync:_getCurrentPercentage() if not self.ui or not self.ui.document then return 0 end local ok, result = pcall(function() @@ -2279,11 +2621,11 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: end base.remote, base.device, base.review = remote_rating, device_rating, remote_review self.rating_baseline[key] = base - G_reader_settings:saveSetting("tomesync_rating_baseline", self.rating_baseline) + self:_saveState("tomesync_rating_baseline", self.rating_baseline) -- Tome's value supersedes any device rating still queued for this book. if self.pending_ratings[key] ~= nil then self.pending_ratings[key] = nil - G_reader_settings:saveSetting("tomesync_pending_ratings", self.pending_ratings) + self:_saveState("tomesync_pending_ratings", self.pending_ratings) end logger.info("TomeSync: applied Tome rating to device for book", self.book_id) elseif local_changed then @@ -2308,7 +2650,7 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: -- A device push is whole-star, so remote and device coincide. base.remote, base.device, base.review = rating, rating, review self.rating_baseline[key] = base - G_reader_settings:saveSetting("tomesync_rating_baseline", self.rating_baseline) + self:_saveState("tomesync_rating_baseline", self.rating_baseline) return true end @@ -2321,12 +2663,12 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: if self:_putRating(self.book_id, rating, review) then if self.pending_ratings[key] ~= nil then self.pending_ratings[key] = nil - G_reader_settings:saveSetting("tomesync_pending_ratings", self.pending_ratings) + self:_saveState("tomesync_pending_ratings", self.pending_ratings) end logger.info("TomeSync: pushed device rating to Tome for book", self.book_id) else self.pending_ratings[key] = {{ rating = rating, review = review }} - G_reader_settings:saveSetting("tomesync_pending_ratings", self.pending_ratings) + self:_saveState("tomesync_pending_ratings", self.pending_ratings) logger.info("TomeSync: rating queued for retry for book", self.book_id) end end @@ -2348,7 +2690,7 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: end end self.pending_ratings = remaining - G_reader_settings:saveSetting("tomesync_pending_ratings", remaining) + self:_saveState("tomesync_pending_ratings", remaining) if flushed then logger.info("TomeSync: pending ratings flushed") end end @@ -2433,12 +2775,25 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: local changed = false local localmap = self:_localAnnotationMap() or {{}} + -- Clamp incoming stamps to this device's clock: server-minted stamps + -- already arrive shifted into our frame (device_time), but stamps from a + -- third device with a fast clock can still be "in the future" here — and a + -- future stamp stored locally would outrank every later local edit. Never + -- store or compare a stamp ahead of now. + local device_now = os.date("%Y-%m-%d %H:%M:%S") + local function clampStamp(stamp) + if type(stamp) == "string" and stamp > device_now then return device_now end + return stamp + end + local pending_web = {{}} for _, s in ipairs(alive or {{}}) do if isWebAnchor(s.anchor) then -- Not a real position — never addItem it; adopt it below instead. table.insert(pending_web, s) elseif s.anchor then + s.datetime = clampStamp(s.datetime) + s.datetime_updated = clampStamp(s.datetime_updated) local L = localmap[s.anchor] local smtime = s.datetime_updated or s.datetime or "" if not L then @@ -2460,7 +2815,7 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: for _, t in ipairs(tombstones or {{}}) do local map2 = self:_localAnnotationMap() or {{}} local L = map2[t.anchor] - if L and L.mtime <= (t.deleted_at or "") then + if L and L.mtime <= clampStamp(t.deleted_at or "") then for i = #ann.annotations, 1, -1 do if ann.annotations[i] == L.item then table.remove(ann.annotations, i); changed = true; break @@ -2558,7 +2913,7 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: end if not add(p0, p1) then return false end self.repair_map[tostring(self.book_id) .. "|" .. p0] = {{ anchor = s.anchor, anchor_end = s.anchor_end }} - G_reader_settings:saveSetting("tomesync_repair_map", self.repair_map) + self:_saveState("tomesync_repair_map", self.repair_map) logger.info("TomeSync: repaired foreign highlight to", p0) return true end @@ -2603,7 +2958,7 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: end end if adopted > 0 then - G_reader_settings:saveSetting("tomesync_adopt_pending", self.adopt_pending) + self:_saveState("tomesync_adopt_pending", self.adopt_pending) end return adopted end @@ -2632,6 +2987,29 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: self.ui.doc_settings:saveSetting("tomesync_annot_bound", true) end + -- Future-watermark guard: no local stamp may sit ahead of this device's + -- clock. Future stamps arrive via server-minted datetimes applied before + -- the clock-offset guard existed (or a third device's fast clock) and would + -- outrank every later local edit until the clock catches up. Clamp the + -- annotation AND its baseline entry to now — equal values, so no spurious + -- re-push, and the user's next edit is strictly newer again. + local now = os.date("%Y-%m-%d %H:%M:%S") -- local wall-clock, matches KOReader's + for anchor, L in pairs(localmap) do + if L.mtime > now then + if L.item.datetime_updated and L.item.datetime_updated > now then + L.item.datetime_updated = now + end + if L.item.datetime and L.item.datetime > now then + L.item.datetime = now + end + L.mtime = annotMtime(L.item) + if baseline[anchor] ~= nil then baseline[anchor] = L.mtime end + end + end + for anchor, m in pairs(baseline) do + if m > now then baseline[anchor] = now end + end + local upserts, deletes = {{}}, {{}} for anchor, L in pairs(localmap) do if baseline[anchor] == nil or baseline[anchor] ~= L.mtime then @@ -2648,15 +3026,17 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: end end end - local now = os.date("%Y-%m-%d %H:%M:%S") -- local wall-clock, matches KOReader's for anchor, _ in pairs(baseline) do if localmap[anchor] == nil then table.insert(deletes, {{ anchor = anchor, datetime = now }}) end end + -- device_time lets the server shift its own (web-minted) stamps into THIS + -- device's clock frame — see the server's clock-offset guard. local resp = apiRequest("POST", "/tome-sync/annotations/" .. self.book_id .. "/sync", - {{ upserts = upserts, deletes = deletes }}) + {{ upserts = upserts, deletes = deletes, + device_time = os.date("%Y-%m-%d %H:%M:%S") }}) if not resp then return nil end -- offline/failed: keep baseline so we retry self:_applyServerState(resp.annotations, resp.tombstones) @@ -2682,14 +3062,14 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: for _, it in ipairs(adopts) do self.adopt_pending[it.anchor] = nil end end end - G_reader_settings:saveSetting("tomesync_adopt_pending", self.adopt_pending) + self:_saveState("tomesync_adopt_pending", self.adopt_pending) -- Rebuild the baseline from the post-merge local state. local newbase = {{}} local after = self:_localAnnotationMap() or {{}} for anchor, L in pairs(after) do newbase[anchor] = L.mtime end self.annot_baseline[bk] = newbase - G_reader_settings:saveSetting("tomesync_annot_baseline", self.annot_baseline) + self:_saveState("tomesync_annot_baseline", self.annot_baseline) -- Drop aliases whose local rendering is gone (a local delete already went -- out under the server identity above) so the map can't grow stale. @@ -2704,13 +3084,13 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: self.repair_map[key] = nil; pruned = true end end - if pruned then G_reader_settings:saveSetting("tomesync_repair_map", self.repair_map) end + if pruned then self:_saveState("tomesync_repair_map", self.repair_map) end return resp end function TomeSync:registerBookId(file_path, book_id) self.book_map[file_path] = book_id - G_reader_settings:saveSetting("tomesync_book_map", self.book_map) + self:_saveState("tomesync_book_map", self.book_map) logger.info("TomeSync: registered book_id", book_id, "for", file_path) end @@ -2902,7 +3282,7 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: if progress_msg then UIManager:close(progress_msg) end -- Persist book_map - G_reader_settings:saveSetting("tomesync_book_map", self.book_map) + self:_saveState("tomesync_book_map", self.book_map) if not quiet then if failed > 0 and #failed_books > 0 then @@ -2944,18 +3324,20 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: return {{ downloaded = downloaded, skipped = skipped, failed = failed }} end --- Drill-down list for one series (or the No Series bucket): a "Download all" row --- plus one row per book, so a single title can be downloaded on its own. -function TomeSync:_seriesBooksMenu(data) - local display = data.series_name - if display == "__unserialized__" then display = "No Series" end - +-- Shared drill-down list: a "Download all" row plus one row per book. Used by +-- the series browser, the author browser, and search results. Tap downloads a +-- book; hold sets its read status. `data`: +-- title menu title +-- books list of server book entries +-- series_name filing default for books without their own series (optional) +-- book_type filing fallback for older servers (optional) +-- mixed true → prefix each row with the book's own series +-- reload called after a status write-back to rebuild with fresh data +function TomeSync:_bookListMenu(data) local items = {{}} table.insert(items, {{ text = string.format("Download all (%d)", #data.books), - callback = function() - self:_downloadSeriesBooks(data.series_name, data.books, nil, data.book_type) - end, + callback = function() self:_downloadListBooks(data, nil) end, }}) -- book_id → on-device path (same signal the download queue uses to skip) local id_to_path = {{}} @@ -2971,27 +3353,275 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: else label = book.title end + if data.mixed and type(book.series) == "string" and book.series ~= "" then + label = book.series .. " · " .. label + end if id_to_path[book.id] then label = label .. " · on device" end + -- Status marker (build 33): only for the non-default states. + if book.status == "reading" or book.status == "read" then + label = label .. " · " .. book.status + end table.insert(items, {{ text = label, - callback = function() - self:_downloadSeriesBooks(data.series_name, {{ book }}, nil, data.book_type) - end, + book = book, + callback = function() self:_downloadListBooks(data, book) end, }}) end - local menu = Menu:new{{ - title = display, + local menu + menu = Menu:new{{ + title = data.title, item_table = items, width = Device.screen:getWidth() - 20, height = Device.screen:getHeight() - 20, show_parent = self.ui or UIManager, }} + -- Hold a book row to set its read status (write-back to Tome). + menu.onMenuHold = function(_, item) + if item.book then + self:_statusDialog(item.book, function() + UIManager:close(menu) + if data.reload then data.reload() end + end) + end + return true + end UIManager:show(menu) end +-- Route one book (or the whole list) into the download machinery with the +-- right filing identity: a book with its own series files under that series; +-- a standalone files like the No Series bucket (book-type/author folders). +function TomeSync:_downloadListBooks(data, book) + local function seriesOf(b) + if type(b.series) == "string" and b.series ~= "" then return b.series end + return data.series_name or "__unserialized__" + end + if book then + self:_downloadSeriesBooks(seriesOf(book), {{ book }}, nil, + book.book_type or data.book_type) + return + end + if data.series_name then + -- Homogeneous list (one series / the No Series bucket): one batch. + self:_downloadSeriesBooks(data.series_name, data.books, nil, data.book_type) + return + end + -- Mixed list (author / search): file each book by its own identity, then + -- roll the counts up into one summary. + local downloaded, skipped, failed = 0, 0, 0 + for _, b in ipairs(data.books) do + local r = self:_downloadSeriesBooks(seriesOf(b), {{ b }}, nil, b.book_type, true) + downloaded = downloaded + (r.downloaded or 0) + skipped = skipped + (r.skipped or 0) + failed = failed + (r.failed or 0) + end + UIManager:show(InfoMessage:new{{ + text = string.format("%s\\n\\nDownloaded: %d\\nSkipped: %d\\nFailed: %d", + data.title or "Download", downloaded, skipped, failed), + timeout = 6, + }}) +end + +-- Hold-a-row dialog: set unread/reading/read on the server (deliberate user +-- action — unlike telemetry, which only suggests status). +function TomeSync:_statusDialog(book, on_done) + local dialog + local function setStatus(status) + UIManager:close(dialog) + whenConnected(function() + local ok, resp, code = pcall(apiRequest, "PUT", "/tome-sync/status/" .. book.id, + {{ status = status }}) + if ok and type(code) == "number" and code < 300 then + book.status = status + UIManager:show(Notification:new{{ + text = string.format('"%s" marked %s.', book.title or "Book", status), + timeout = 2, + }}) + if on_done then on_done() end + else + UIManager:show(InfoMessage:new{{ + text = "Could not update status (" .. tostring(code) .. ").", + timeout = 4, + }}) + end + end) + end + dialog = ButtonDialog:new{{ + title = (book.title or "Book") .. "\\nSet read status", + buttons = {{ + {{ {{ text = "Unread", callback = function() setStatus("unread") end }} }}, + {{ {{ text = "Reading", callback = function() setStatus("reading") end }} }}, + {{ {{ text = "Read", callback = function() setStatus("read") end }} }}, + }}, + }} + UIManager:show(dialog) +end + +-- Adapter kept for the series flow (name/shape unchanged for its callers). +function TomeSync:_seriesBooksMenu(data) + local display = data.series_name + if display == "__unserialized__" then display = "No Series" end + self:_bookListMenu{{ + title = display, + books = data.books, + series_name = data.series_name, + book_type = data.book_type, + reload = function() + if #data.books > 0 then self:_openSeriesBooks(data.books[1].id) end + end, + }} +end + +-- ── Author browse axis (build 33) ──────────────────────────────────────────── + +function TomeSync:_authorsMenu() + whenConnected(function() self:_authorsMenuImpl() end) +end + +function TomeSync:_authorsMenuImpl() + local ok, authors, code = pcall(apiRequest, "GET", "/tome-sync/authors") + if not ok or type(authors) ~= "table" or (type(code) == "number" and code >= 300) then + UIManager:show(ConfirmBox:new{{ + text = "Failed to load authors.", + ok_text = "Retry", + cancel_text = "Close", + ok_callback = function() self:_authorsMenuImpl() end, + }}) + return + end + local items = {{}} + for _, a in ipairs(authors) do + local name = a.name + if name == "__unknown__" then name = "Unknown author" end + table.insert(items, {{ + text = name .. " (" .. a.book_count .. ")", + callback = function() self:_openAuthorBooks(a.name) end, + }}) + end + UIManager:show(Menu:new{{ + title = "Authors", + item_table = items, + width = Device.screen:getWidth() - 20, + height = Device.screen:getHeight() - 20, + show_parent = self.ui or UIManager, + }}) +end + +function TomeSync:_openAuthorBooks(author) + local ok, data, code = pcall(apiRequest, "GET", + "/tome-sync/author-books?author=" .. urlEncode(author)) + if ok and type(data) == "table" and data.books then + local display = author + if display == "__unknown__" then display = "Unknown author" end + self:_bookListMenu{{ + title = display, + books = data.books, + mixed = true, + reload = function() self:_openAuthorBooks(author) end, + }} + else + UIManager:show(ConfirmBox:new{{ + text = "Failed to load author's books.", + ok_text = "Retry", + cancel_text = "Close", + ok_callback = function() self:_openAuthorBooks(author) end, + }}) + end + local _ = code +end + +-- ── Search from the device (build 33) ──────────────────────────────────────── + +function TomeSync:_recentSearches() + return self.state:readSetting("tomesync_recent_searches") or {{}} +end + +function TomeSync:_rememberSearch(q) + local recents = self:_recentSearches() + for i = #recents, 1, -1 do + if recents[i] == q then table.remove(recents, i) end + end + table.insert(recents, 1, q) + while #recents > 8 do table.remove(recents) end + self:_saveState("tomesync_recent_searches", recents) +end + +function TomeSync:_searchMenu() + -- Submit-based input (the right call on e-ink), with recent searches one + -- tap away underneath. + local dialog + dialog = InputDialog:new{{ + title = "Search library", + input_hint = "title, author, or series", + buttons = {{{{ + {{ text = "Cancel", id = "close", + callback = function() UIManager:close(dialog) end }}, + {{ text = "Search", is_enter_default = true, + callback = function() + local q = dialog:getInputText() + if q and q:match("%S") then + UIManager:close(dialog) + self:_runSearch(q) + end + end }}, + }}}}, + }} + local recents = self:_recentSearches() + if #recents > 0 then + -- A second row of up to 3 recent queries; the full list lives in the + -- results menu title history anyway, and 3 covers the muscle-memory case. + local row = {{}} + for i = 1, math.min(3, #recents) do + local q = recents[i] + table.insert(row, {{ text = q, callback = function() + UIManager:close(dialog) + self:_runSearch(q) + end }}) + end + table.insert(dialog.buttons, 1, row) + end + UIManager:show(dialog) + dialog:onShowKeyboard() +end + +function TomeSync:_runSearch(q) + whenConnected(function() + local ok, data, code = pcall(apiRequest, "GET", + "/tome-sync/search?q=" .. urlEncode(q)) + if not ok or type(data) ~= "table" or not data.books + or (type(code) == "number" and code >= 300) then + UIManager:show(ConfirmBox:new{{ + text = "Search failed.", + ok_text = "Retry", + cancel_text = "Close", + ok_callback = function() self:_runSearch(q) end, + }}) + return + end + self:_rememberSearch(q) + if #data.books == 0 then + UIManager:show(InfoMessage:new{{ + text = 'No results for "' .. q .. '".', + timeout = 3, + }}) + return + end + local title = string.format('Search: %s (%d)', q, data.total or #data.books) + if (data.total or 0) > #data.books then + title = string.format('Search: %s (%d of %d)', q, #data.books, data.total) + end + self:_bookListMenu{{ + title = title, + books = data.books, + mixed = true, + reload = function() self:_runSearch(q) end, + }} + end) +end + function TomeSync:_browseSeriesMenu() whenConnected(function() self:_browseSeriesMenuImpl() end) end @@ -3017,6 +3647,17 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: end local items = {{}} + -- Cross-axis entry points (build 33): search and the author axis live at + -- the top of the browser, so standalones aren't stuck behind "No Series". + table.insert(items, {{ + text = "Search library…", + callback = function() self:_searchMenu() end, + }}) + table.insert(items, {{ + text = "Browse by author", + separator = true, + callback = function() self:_authorsMenu() end, + }}) for _, s in ipairs(series_list) do local name = s.name if name == "__unserialized__" then name = "No Series" end @@ -3367,6 +4008,46 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: not G_reader_settings:isTrue("tomesync_auto_connect")) end, }}) + -- Position pull strategy (like stock kosync): what to do when the server + -- position differs from this device's on book open. + local function pullModeItems(key, default) + local function current() + return G_reader_settings:readSetting(key) or default + end + local items = {{}} + for _, m in ipairs({{ + {{ "prompt", "Ask before jumping" }}, + {{ "silent", "Jump automatically" }}, + {{ "never", "Do nothing" }}, + }}) do + local value, label = m[1], m[2] + table.insert(items, {{ + text = label, + checked_func = function() return current() == value end, + callback = function() + if value == default then + G_reader_settings:delSetting(key) + else + G_reader_settings:saveSetting(key, value) + end + end, + }}) + end + return items + end + table.insert(settings_items, {{ + text = "Server position is ahead", + help_text = "What to do on book open when the server position is " + .. "further along than this device (you read elsewhere).", + sub_item_table = pullModeItems("tomesync_pull_forward", "silent"), + }}) + table.insert(settings_items, {{ + text = "Server position is behind", + help_text = "What to do on book open when the server position is " + .. "earlier than this device (e.g. re-reading a section " + .. "on another device).", + sub_item_table = pullModeItems("tomesync_pull_backward", "never"), + }}) local function currentTemplate() return G_reader_settings:readSetting("tomesync_download_template") or "" end @@ -3464,7 +4145,7 @@ def _main_impl_lua(server_url: str, api_key: str, username: str) -> str: callback = function() self.book_map = {{}} self.book_id = nil - G_reader_settings:saveSetting("tomesync_book_map", {{}}) + self:_saveState("tomesync_book_map", {{}}) UIManager:show(InfoMessage:new{{ text = "All book mappings cleared.\\nRe-open a book to re-resolve.", timeout = 3, diff --git a/backend/main.py b/backend/main.py index 9c32cde..babd25e 100644 --- a/backend/main.py +++ b/backend/main.py @@ -211,6 +211,16 @@ async def lifespan(app: FastAPI): if an_cols and "cfi" not in an_cols: conn.execute(text("ALTER TABLE annotations ADD COLUMN cfi TEXT")) conn.commit() + # Clock-offset guard: marks LWW stamps minted by the server's clock (web + # create/edit/delete) so plugin syncs can shift them into the device frame. + # Pre-existing rows default 0 (device-minted) — the safe assumption. + if an_cols and "server_minted" not in an_cols: + conn.execute(text("ALTER TABLE annotations ADD COLUMN server_minted BOOLEAN NOT NULL DEFAULT 0")) + conn.commit() + at_tomb_cols = {r[1] for r in conn.execute(text("PRAGMA table_info(annotation_tombstones)")).fetchall()} + if at_tomb_cols and "server_minted" not in at_tomb_cols: + conn.execute(text("ALTER TABLE annotation_tombstones ADD COLUMN server_minted BOOLEAN NOT NULL DEFAULT 0")) + conn.commit() # Release detection stores the tracker's latest volume title + date on # follows (wishes table pre-exists on upgraded installs). wi_cols = {r[1] for r in conn.execute(text("PRAGMA table_info(wishes)")).fetchall()} diff --git a/backend/models/tome_sync.py b/backend/models/tome_sync.py index c45e49f..b403081 100644 --- a/backend/models/tome_sync.py +++ b/backend/models/tome_sync.py @@ -3,7 +3,7 @@ from datetime import datetime from typing import Optional -from sqlalchemy import DateTime, Float, ForeignKey, Integer, String, Text, UniqueConstraint +from sqlalchemy import Boolean, DateTime, Float, ForeignKey, Integer, String, Text, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column, relationship from backend.core.database import Base @@ -128,6 +128,12 @@ class Annotation(Base): koreader_datetime: Mapped[Optional[str]] = mapped_column(String(32), nullable=True) # KOReader's last-modification time — the last-write-wins key for edit conflicts. koreader_datetime_updated: Mapped[Optional[str]] = mapped_column(String(32), nullable=True) + # True when the LWW stamp above was minted by the SERVER's clock (web create/edit) + # rather than a device's. Server-minted stamps are shifted into the requesting + # device's clock frame during plugin sync, so a server clock ahead of a device + # can't make web actions permanently outrank the user's next local edit. + # Cleared as soon as a device upsert wins the row (the stamp is then device time). + server_minted: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, nullable=False) updated_at: Mapped[datetime] = mapped_column( DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False @@ -164,4 +170,7 @@ class AnnotationTombstone(Base): ) anchor: Mapped[str] = mapped_column(String(512), nullable=False) client_deleted_at: Mapped[Optional[str]] = mapped_column(String(32), nullable=True) + # See Annotation.server_minted — True when client_deleted_at came from the + # server's clock (web delete), so plugin syncs shift it into the device frame. + server_minted: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, nullable=False) diff --git a/tests/test_tomesync_catalog.py b/tests/test_tomesync_catalog.py new file mode 100644 index 0000000..f3e515c --- /dev/null +++ b/tests/test_tomesync_catalog.py @@ -0,0 +1,175 @@ +"""Plugin catalog batch (build 33) — server endpoints. + +Author browse axis, device search, read-status write-back, and the series-list +N+1 rewrite (single query, response shape unchanged). +""" +from backend.models.tome_sync import ApiKey +from backend.models.user import User +from backend.models.user_book_status import UserBookStatus +from backend.core.security import hash_password + + +def _api_key_for(db, user_id: int) -> str: + plaintext = ApiKey.generate() + db.add(ApiKey(user_id=user_id, key_hash=ApiKey.hash_key(plaintext), + key_prefix=plaintext[:11], label="test")) + db.flush() + return plaintext + + +def _hdr(db, user): + return {"Authorization": f"Bearer {_api_key_for(db, user.id)}"} + + +# ── series list (N+1 rewrite parity) ───────────────────────────────────────── + +def test_series_list_shape_and_first_book(db, client, admin_user, make_book): + user, _ = admin_user + make_book(title="B two", series="Alpha", series_index=2, author="A. Author") + make_book(title="A one", series="Alpha", series_index=1, author="A. Author") + make_book(title="Standalone", author="S. Alone") + r = client.get("/api/tome-sync/series", headers=_hdr(db, user)) + assert r.status_code == 200 + data = r.json() + alpha = [s for s in data if s["name"] == "Alpha"][0] + assert alpha["book_count"] == 2 + assert alpha["author"] == "A. Author" + # first_book_id must be the lowest series_index, not insertion order + books = client.get(f"/api/tome-sync/series/{alpha['first_book_id']}", + headers=_hdr(db, user)).json()["books"] + assert books[0]["title"] == "A one" + # unserialized bucket appended last + assert data[-1]["name"] == "__unserialized__" + assert data[-1]["book_count"] == 1 + + +def test_series_index_null_sorts_last_for_first_book(db, client, admin_user, make_book): + user, _ = admin_user + noidx = make_book(title="No index", series="Beta", author="X") + make_book(title="Indexed", series="Beta", series_index=1, author="X") + r = client.get("/api/tome-sync/series", headers=_hdr(db, user)).json() + beta = [s for s in r if s["name"] == "Beta"][0] + assert beta["first_book_id"] != noidx.id + + +# ── authors axis ───────────────────────────────────────────────────────────── + +def test_authors_list_counts_and_unknown_bucket(db, client, admin_user, make_book): + user, _ = admin_user + make_book(title="One", author="Zed Writer") + make_book(title="Two", author="Zed Writer", series="S", series_index=1) + make_book(title="Anon", author=None) + r = client.get("/api/tome-sync/authors", headers=_hdr(db, user)) + assert r.status_code == 200 + data = r.json() + zed = [a for a in data if a["name"] == "Zed Writer"][0] + assert zed["book_count"] == 2 + assert data[-1] == {"name": "__unknown__", "book_count": 1} + + +def test_author_books_shape_and_unknown(db, client, admin_user, make_book): + user, _ = admin_user + make_book(title="Vol 2", author="Zed Writer", series="S", series_index=2) + make_book(title="Vol 1", author="Zed Writer", series="S", series_index=1) + make_book(title="Alone", author="Zed Writer") + anon = make_book(title="Anon", author=None) + r = client.get("/api/tome-sync/author-books", params={"author": "Zed Writer"}, + headers=_hdr(db, user)) + assert r.status_code == 200 + data = r.json() + assert [b["title"] for b in data["books"]] == ["Vol 1", "Vol 2", "Alone"] + assert data["books"][0]["series"] == "S" + assert "series" not in data["books"][2] # no JSON null for the plugin + assert data["books"][0]["status"] == "unread" # default + assert isinstance(data["books"][0]["files"], list) + + r2 = client.get("/api/tome-sync/author-books", params={"author": "__unknown__"}, + headers=_hdr(db, user)).json() + assert [b["id"] for b in r2["books"]] == [anon.id] + + +# ── search ─────────────────────────────────────────────────────────────────── + +def test_search_matches_title_author_series_all_terms(db, client, admin_user, make_book): + user, _ = admin_user + make_book(title="The Iron Duke", author="Meljean Brook", series="Iron Seas", series_index=1) + make_book(title="Something Else", author="Other Person") + hdr = _hdr(db, user) + + r = client.get("/api/tome-sync/search", params={"q": "iron"}, headers=hdr).json() + assert r["total"] == 1 and r["books"][0]["title"] == "The Iron Duke" + # multi-term: all must match (title+author across fields) + r = client.get("/api/tome-sync/search", params={"q": "iron brook"}, headers=hdr).json() + assert r["total"] == 1 + r = client.get("/api/tome-sync/search", params={"q": "iron nobody"}, headers=hdr).json() + assert r["total"] == 0 + # blank query is a no-op, not a full dump + r = client.get("/api/tome-sync/search", params={"q": " "}, headers=hdr).json() + assert r["total"] == 0 and r["books"] == [] + + +def test_search_caps_at_50_but_reports_total(db, client, admin_user, make_book): + user, _ = admin_user + for i in range(55): + make_book(title=f"Common Word {i}", author="Bulk") + r = client.get("/api/tome-sync/search", params={"q": "common word"}, + headers=_hdr(db, user)).json() + assert r["total"] == 55 + assert len(r["books"]) == 50 + + +# ── read-status write-back ─────────────────────────────────────────────────── + +def test_status_writeback_upserts_and_reads_back(db, client, admin_user, make_book): + user, _ = admin_user + book = make_book(title="Statusable", author="X", series="St", series_index=1) + hdr = _hdr(db, user) + + r = client.put(f"/api/tome-sync/status/{book.id}", json={"status": "reading"}, headers=hdr) + assert r.status_code == 200 and r.json()["status"] == "reading" + row = db.query(UserBookStatus).filter_by(user_id=user.id, book_id=book.id).first() + assert row is not None and row.status == "reading" + + # update, not duplicate + client.put(f"/api/tome-sync/status/{book.id}", json={"status": "read"}, headers=hdr) + assert db.query(UserBookStatus).filter_by(user_id=user.id, book_id=book.id).count() == 1 + + # and the volume list reflects it + books = client.get(f"/api/tome-sync/series/{book.id}", headers=hdr).json()["books"] + assert books[0]["status"] == "read" + + +def test_status_writeback_validates_and_404s(db, client, admin_user, make_book): + user, _ = admin_user + book = make_book(title="V", author="X") + hdr = _hdr(db, user) + assert client.put(f"/api/tome-sync/status/{book.id}", + json={"status": "devoured"}, headers=hdr).status_code == 422 + assert client.put("/api/tome-sync/status/999999", + json={"status": "read"}, headers=hdr).status_code == 404 + + +# ── visibility ─────────────────────────────────────────────────────────────── + +def test_catalog_endpoints_respect_visibility(db, client, admin_user, make_book): + """A member must not see (or mutate status of) another member's private + unfiled upload via authors/search/status.""" + admin, _ = admin_user + owner = User(username="owner", email="o@example.com", + hashed_password=hash_password("pw"), is_active=True, role="member") + peeker = User(username="peeker", email="p@example.com", + hashed_password=hash_password("pw"), is_active=True, role="member") + db.add_all([owner, peeker]) + db.flush() + private = make_book(title="Private Thing", author="Hidden Person") + private.added_by = owner.id + db.flush() + peek_hdr = _hdr(db, peeker) + + authors = client.get("/api/tome-sync/authors", headers=peek_hdr).json() + assert not any(a["name"] == "Hidden Person" for a in authors) + search = client.get("/api/tome-sync/search", params={"q": "private thing"}, + headers=peek_hdr).json() + assert search["total"] == 0 + assert client.put(f"/api/tome-sync/status/{private.id}", + json={"status": "read"}, headers=peek_hdr).status_code == 404 diff --git a/tests/test_tomesync_clock_offset.py b/tests/test_tomesync_clock_offset.py new file mode 100644 index 0000000..9d24fa3 --- /dev/null +++ b/tests/test_tomesync_clock_offset.py @@ -0,0 +1,204 @@ +"""Clock-offset guard for annotation sync (plugin build 32). + +LWW stamps are wall-clock strings. Stamps a device minted stay in that device's +frame (accepted cross-device skew); stamps the SERVER minted (web create/edit/ +delete) are shifted into the requesting device's frame — computed from the +`device_time` the plugin stamps on each sync — so a server clock hours ahead of +a device can't make a web delete/edit permanently outrank the user's next local +change (the silent highlight-swallowing class). +""" +from datetime import datetime, timedelta + +from backend.api.tome_sync import _clock_offset_seconds, _shift_ko_dt, _KO_DT_FMT +from backend.models.tome_sync import ApiKey, Annotation, AnnotationTombstone + + +def _api_key_for(db, user_id: int) -> str: + plaintext = ApiKey.generate() + db.add(ApiKey(user_id=user_id, key_hash=ApiKey.hash_key(plaintext), + key_prefix=plaintext[:11], label="test")) + db.flush() + return plaintext + + +def _hl(anchor, text="t", note=None, dt="2026-06-03 10:00:00", dtu=None): + return {"anchor": anchor, "highlighted_text": text, "note": note, + "chapter": "C1", "color": "yellow", "datetime": dt, "datetime_updated": dtu} + + +def _sync(client, hdr, book_id, upserts=(), deletes=(), device_time=None): + body = {"upserts": list(upserts), "deletes": list(deletes)} + if device_time is not None: + body["device_time"] = device_time + return client.post(f"/api/tome-sync/annotations/{book_id}/sync", headers=hdr, json=body) + + +def _dev_now(behind_hours: float = 0) -> str: + """A device wall-clock string N hours behind the server's clock.""" + return (datetime.now() - timedelta(hours=behind_hours)).strftime(_KO_DT_FMT) + + +A1 = "/body/DocFragment[2]/p[1]/text().0" + + +# ── unit: offset + shift helpers ───────────────────────────────────────────── + +def test_offset_zero_without_device_time(): + assert _clock_offset_seconds(None) == 0 + assert _clock_offset_seconds("") == 0 + assert _clock_offset_seconds("not a datetime") == 0 + + +def test_offset_small_skew_is_noise(): + # 30s behind: below tolerance, treated as synchronized + assert _clock_offset_seconds(_dev_now(30 / 3600)) == 0 + + +def test_offset_detects_slow_device_clock(): + off = _clock_offset_seconds(_dev_now(3)) + assert 3 * 3600 - 5 <= off <= 3 * 3600 + 5 + + +def test_shift_ko_dt(): + assert _shift_ko_dt("2026-06-03 10:00:00", -3600) == "2026-06-03 09:00:00" + assert _shift_ko_dt("2026-06-03 10:00:00", 0) == "2026-06-03 10:00:00" + assert _shift_ko_dt(None, -3600) is None + assert _shift_ko_dt("garbage", -3600) == "garbage" + + +# ── the swallow scenario: web delete vs device re-add, device clock behind ─── + +def _web_delete(client, db, book_id, anchor): + """Delete via the web endpoint (server-minted tombstone). The client + fixture already carries the admin JWT in its default headers.""" + row = (db.query(Annotation) + .filter(Annotation.book_id == book_id, Annotation.anchor == anchor).first()) + assert row is not None + r = client.delete(f"/api/annotations/{row.id}") + assert r.status_code == 204, r.text + + +def test_readd_after_web_delete_wins_with_device_time(client, db, admin_user, make_book): + """Device 3h behind server. Web delete mints a tombstone in the device's + future; the user re-highlights the same passage a moment later (device + frame). With device_time the server shifts its stamp into the device frame + and the re-add wins instead of being silently swallowed for 3 hours.""" + user, _ = admin_user + book = make_book() + hdr = {"Authorization": f"Bearer {_api_key_for(db, user.id)}"} + dev_behind = 3 # hours + + # Device creates the highlight, stamped in its own (slow) frame. + created = _dev_now(dev_behind) + _sync(client, hdr, book.id, upserts=[_hl(A1, "keep me", dt=created, dtu=created)], + device_time=_dev_now(dev_behind)) + + # Web delete: tombstone stamped with the server clock (device future). + _web_delete(client, db, book.id, A1) + tomb = db.query(AnnotationTombstone).filter_by(book_id=book.id, anchor=A1).first() + assert tomb is not None and tomb.server_minted is True + assert tomb.client_deleted_at > _dev_now(dev_behind) # in the device's future + + # User re-highlights one device-minute later. Without the guard this is + # mtime <= tombstone and gets skipped. + readd = (datetime.now() - timedelta(hours=dev_behind) + timedelta(minutes=1)).strftime(_KO_DT_FMT) + r = _sync(client, hdr, book.id, upserts=[_hl(A1, "keep me", dt=readd, dtu=readd)], + device_time=_dev_now(dev_behind)) + assert r.status_code == 200, r.text + assert r.json()["applied"]["created"] == 1, r.json()["applied"] + assert db.query(AnnotationTombstone).filter_by(book_id=book.id, anchor=A1).count() == 0 + + +def test_readd_after_web_delete_swallowed_without_device_time(client, db, admin_user, make_book): + """Documents the legacy behavior an old plugin (no device_time) still gets: + the server-frame tombstone outranks the slow device's re-add.""" + user, _ = admin_user + book = make_book() + hdr = {"Authorization": f"Bearer {_api_key_for(db, user.id)}"} + dev_behind = 3 + + created = _dev_now(dev_behind) + _sync(client, hdr, book.id, upserts=[_hl(A1, "gone", dt=created, dtu=created)]) + _web_delete(client, db, book.id, A1) + + readd = (datetime.now() - timedelta(hours=dev_behind) + timedelta(minutes=1)).strftime(_KO_DT_FMT) + r = _sync(client, hdr, book.id, upserts=[_hl(A1, "gone", dt=readd, dtu=readd)]) + assert r.json()["applied"]["skipped"] == 1 + + +def test_device_edit_beats_web_edit_when_actually_later(client, db, admin_user, make_book): + """Web edit bumps the mtime with the server clock; a device 3h behind edits + afterwards (later in real time). With device_time the device edit wins.""" + user, _ = admin_user + book = make_book() + hdr = {"Authorization": f"Bearer {_api_key_for(db, user.id)}"} + dev_behind = 3 + + created = _dev_now(dev_behind) + _sync(client, hdr, book.id, upserts=[_hl(A1, "text", note="device v1", dt=created, dtu=created)], + device_time=_dev_now(dev_behind)) + + row = db.query(Annotation).filter_by(book_id=book.id, anchor=A1).first() + r = client.put(f"/api/annotations/{row.id}", json={"note": "web edit"}) + assert r.status_code == 200 + db.refresh(row) + assert row.server_minted is True + + # Device edits a device-minute later (genuinely after the web edit). + edited = (datetime.now() - timedelta(hours=dev_behind) + timedelta(minutes=1)).strftime(_KO_DT_FMT) + r = _sync(client, hdr, book.id, + upserts=[_hl(A1, "text", note="device v2", dt=created, dtu=edited)], + device_time=_dev_now(dev_behind)) + assert r.json()["applied"]["updated"] == 1, r.json()["applied"] + db.refresh(row) + assert row.note == "device v2" + assert row.server_minted is False # stamp is device-authored now + + +def test_response_shifts_server_minted_stamps_into_device_frame(client, db, admin_user, make_book): + """A web-created highlight travels to a slow device with its stamps shifted + into that device's frame — never in the device's future.""" + user, _ = admin_user + book = make_book() + hdr = {"Authorization": f"Bearer {_api_key_for(db, user.id)}"} + dev_behind = 3 + + r = client.post("/api/annotations", json={ + "book_id": book.id, "highlighted_text": "from the web", "cfi": "epubcfi(/6/4!/4/2/1:0)", + }) + assert r.status_code in (200, 201), r.text + + device_now = _dev_now(dev_behind) + g = _sync(client, hdr, book.id, device_time=device_now).json() + web = [a for a in g["annotations"] if a["anchor"].startswith("web:")] + assert len(web) == 1 + assert web[0]["datetime"] <= device_now # shifted, not in the device future + + # Same pull without device_time: raw server frame (device future). + g2 = _sync(client, hdr, book.id).json() + web2 = [a for a in g2["annotations"] if a["anchor"].startswith("web:")] + assert web2[0]["datetime"] > device_now + + +def test_device_minted_stamps_are_never_shifted(client, db, admin_user, make_book): + """Cross-device skew stays accepted: another device's stamps pass through + verbatim even when the requesting device reports a big offset.""" + user, _ = admin_user + book = make_book() + hdr = {"Authorization": f"Bearer {_api_key_for(db, user.id)}"} + + stamp = "2026-06-03 10:00:00" + _sync(client, hdr, book.id, upserts=[_hl(A1, "device made", dt=stamp, dtu=stamp)]) + + g = _sync(client, hdr, book.id, device_time=_dev_now(3)).json() + a = [x for x in g["annotations"] if x["anchor"] == A1][0] + assert a["datetime"] == stamp + assert a["datetime_updated"] == stamp + + +def test_sync_response_carries_server_time(client, db, admin_user, make_book): + user, _ = admin_user + book = make_book() + hdr = {"Authorization": f"Bearer {_api_key_for(db, user.id)}"} + g = _sync(client, hdr, book.id).json() + datetime.strptime(g["server_time"], _KO_DT_FMT) # parseable, present diff --git a/tests/test_tomesync_hygiene.py b/tests/test_tomesync_hygiene.py new file mode 100644 index 0000000..d072dad --- /dev/null +++ b/tests/test_tomesync_hygiene.py @@ -0,0 +1,138 @@ +"""Plugin build 32 hygiene batch — generated-impl contract tests. + +Three features: the dedicated state file (data tables out of G_reader_settings), +pull-conflict strategy settings, and the device-side half of the clock-offset +guard. Server-side clock-offset behavior is covered in +test_tomesync_clock_offset.py. +""" +import shutil +import subprocess +import tempfile +from pathlib import Path + +import pytest + +from backend.api.tome_sync import _main_impl_lua + + +def _impl() -> str: + return _main_impl_lua("https://tome.example.org", "tk_testkey", "tester") + + +def _body(lua: str, func: str) -> str: + # Line-anchored: "function TomeSync:init" also appears inside validateImpl's + # string literals, so a bare find() would land there. + start = lua.find(f"\nfunction TomeSync:{func}") + assert start != -1, f"missing function {func}" + return lua[start:lua.find("\nfunction ", start + 1)] + + +def test_impl_compiles_under_luajit(): + luajit = shutil.which("luajit") + if luajit is None: + pytest.skip("luajit not installed") + with tempfile.NamedTemporaryFile(suffix=".lua", delete=False, mode="w") as f: + f.write(_impl()) + path = f.name + try: + r = subprocess.run([luajit, "-bl", path], capture_output=True, text=True) + assert r.returncode == 0, r.stderr + finally: + Path(path).unlink(missing_ok=True) + + +# ── dedicated state file ───────────────────────────────────────────────────── + +DATA_KEYS = [ + "tomesync_book_map", "tomesync_pending_sessions", "tomesync_adopt_pending", + "tomesync_repair_map", "tomesync_annot_baseline", "tomesync_rating_baseline", + "tomesync_pending_ratings", +] + + +def test_data_tables_live_in_state_file_not_global(): + lua = _impl() + assert 'LuaSettings:open(DataStorage:getSettingsDir() .. "/tomesync_state.lua")' in lua + for key in DATA_KEYS: + assert f'G_reader_settings:readSetting("{key}")' not in _body(lua, "init"), key + assert f'self.state:readSetting("{key}")' in _body(lua, "init"), key + # No save site may write a data table back to the global settings file. + assert f'G_reader_settings:saveSetting("{key}"' not in lua, key + + +def test_update_state_stays_in_global_settings(): + # The frozen shim reads tomesync_update from G_reader_settings and is never + # replaced by self-update — the impl must keep writing it there. + lua = _impl() + assert 'G_reader_settings:readSetting("tomesync_update")' in lua + assert 'G_reader_settings:saveSetting("tomesync_update"' in lua + + +def test_migration_is_crash_safe_ordered(): + # New file must be flushed BEFORE the old keys are deleted, and the marker + # branch re-deletes leftovers from a crash between the two flushes. + body = _body(_impl(), "_migrateState") + assert 'readSetting("migrated_from_global")' in body + flush_new = body.find("self.state:flush()") + del_old = body.rfind("G_reader_settings:delSetting(k)") + assert flush_new != -1 and del_old != -1 and flush_new < del_old + + +def test_save_state_writes_through(): + body = _body(_impl(), "_saveState") + assert "self.state:saveSetting(key, value)" in body + assert "self.state:flush()" in body + + +def test_prune_never_touches_pending_queues(): + body = _body(_impl(), "_pruneState") + assert "pending_sessions" not in body.replace( + "Queues (pending_sessions/pending_ratings) are never pruned", "") + assert "tomesync_annot_baseline" in body + assert "tomesync_repair_map" in body + + +# ── pull-conflict strategy ──────────────────────────────────────────────────── + +def test_pull_modes_cover_both_directions_with_compatible_defaults(): + body = _body(_impl(), "_initSession") + assert 'readSetting("tomesync_pull_forward") or "silent"' in body + assert 'readSetting("tomesync_pull_backward") or "never"' in body + # Backward pull exists and is bounded away from 0%. + assert "server_pct < (local_pct - 0.01)" in body + + +def test_prompt_is_deferred_off_the_open_path(): + # A ConfirmBox at open time would eat the Profiles auto-exec dispatch + # exactly like the InfoMessage layout-reset bug — it must be scheduled. + body = _body(_impl(), "_initSession") + prompt_at = body.find('mode == "prompt"') + sched_at = body.find("UIManager:scheduleIn", prompt_at) + confirm_at = body.find("ConfirmBox:new", prompt_at) + assert prompt_at != -1 and sched_at != -1 and confirm_at != -1 + assert sched_at < confirm_at, "ConfirmBox must be inside the deferred callback" + + +def test_pull_settings_menu_items_exist(): + lua = _impl() + assert "Server position is ahead" in lua + assert "Server position is behind" in lua + + +# ── clock-offset guard, device half ────────────────────────────────────────── + +def test_sync_request_carries_device_time(): + body = _body(_impl(), "_syncAnnotations") + assert "device_time = os.date" in body + + +def test_future_stamps_are_scrubbed_before_diffing(): + body = _body(_impl(), "_syncAnnotations") + assert "L.mtime > now" in body + assert "baseline[anchor] = now" in body + + +def test_incoming_stamps_are_clamped_to_device_clock(): + body = _body(_impl(), "_applyServerState") + assert "clampStamp" in body + assert "device_now" in body diff --git a/tests/test_tomesync_initsession.py b/tests/test_tomesync_initsession.py index 2325bcb..16f05ad 100644 --- a/tests/test_tomesync_initsession.py +++ b/tests/test_tomesync_initsession.py @@ -47,13 +47,20 @@ def test_goto_guards_against_non_crengine_xpointers(): # The web reader stores a foliate epubcfi in TomeSyncPosition.progress; # crengine xpointers always start with "/". Anything else must fall back # to a percentage jump instead of onGotoXPointer (which lands on page 1). - body = _init_session_body(_impl()) + # The jump lives in the shared _gotoServerPosition helper (build 32: both + # the silent pull and the prompt callback route through it). + lua = _impl() + start = lua.find("function TomeSync:_gotoServerPosition") + assert start != -1, "missing _gotoServerPosition helper" + body = lua[start:lua.find("\nfunction ", start + 1)] guard = body.find('pos.progress:sub(1, 1) == "/"') goto_xp = body.find("onGotoXPointer") goto_pct = body.find("onGotoPercent") assert guard != -1, "missing xpointer-shape guard" assert goto_xp != -1 and guard < goto_xp, "goto must be behind the guard" assert goto_pct != -1, "missing percentage fallback for web positions" + # And _initSession must actually route through the helper. + assert "self:_gotoServerPosition(pos, server_pct)" in _init_session_body(lua) def test_session_init_is_deduped_per_open(): diff --git a/tests/test_tomesync_rating_sync.py b/tests/test_tomesync_rating_sync.py index c151e80..9a3a623 100644 --- a/tests/test_tomesync_rating_sync.py +++ b/tests/test_tomesync_rating_sync.py @@ -85,7 +85,8 @@ def test_failed_rating_push_is_queued_and_flushed(): # lost — the per-book close trigger never fires again for a book you rate and # never reopen (e.g. a finished book). It rides a pending queue, like sessions. lua = _impl() - assert 'G_reader_settings:readSetting("tomesync_pending_ratings")' in lua + # Build 32: data tables live in the dedicated state file, not G_reader_settings. + assert 'self.state:readSetting("tomesync_pending_ratings")' in lua # On a failed push, _pushRating stows the value for retry. push = _body(lua, "_pushRating") assert "self.pending_ratings[key] = { rating = rating, review = review }" in push @@ -104,8 +105,9 @@ def test_json_null_is_normalized_to_nil_on_read(): def test_baseline_is_persisted(): lua = _impl() - assert 'G_reader_settings:readSetting("tomesync_rating_baseline")' in lua - assert 'G_reader_settings:saveSetting("tomesync_rating_baseline"' in lua + # Build 32: data tables live in the dedicated state file, not G_reader_settings. + assert 'self.state:readSetting("tomesync_rating_baseline")' in lua + assert 'self:_saveState("tomesync_rating_baseline"' in lua def test_open_and_leave_hooks_are_wired():