feat: Import Readest highlights as clippings

app/src/Settings.jsx

@@ -51,7 +51,7 @@ const HINTS = {
   jellyfin: 'Syncs reading progress with your Jellyfin book library (Jellyfin 12.0 or newer). Sign in with your Jellyfin username and password.',
   bookorbit: 'Syncs progress both ways, adds clippings as highlights, and exchanges reading sessions for matched books. Imports BookOrbit history and exports sessions recorded by newer CrossInk firmware. Includes other BookOrbit reader history alongside your device stats. Repeated sessions are counted once.',
   seekquel: 'Logs your reading in your Seekquel library: where you are in each book, the books you finish, and your highlights. If you read further in Seekquel, that place comes back to your reader. You approve the request in the Seekquel app.',
-  readest: 'Syncs your reading progress with Readest and brings your place back when you read further in a Readest app. Clippings become Readest highlights for books in your Readest cloud library. Use this instead of signing in to the Readest plugin\'s progress sync, which moves your reader off CrossPoint Sync and stops clipping sync. You approve the request on readest.com.',
+  readest: 'Syncs your reading progress with Readest and brings your place back when you read further in a Readest app. Clippings become Readest highlights, and Readest highlights become clippings, for books in your Readest cloud library. Use this instead of signing in to the Readest plugin\'s progress sync, which moves your reader off CrossPoint Sync and stops clipping sync. You approve the request on readest.com.',
   spotify: 'Moves your progress forward when you listen ahead in the Spotify audiobook, and resumes it there. Audiobooks are in the US, UK, Canada, Ireland, Australia and New Zealand.',
   kindle: 'Linking needs the CrossPoint Kindle Link browser extension.',
 }

src/connectors/fanin.ts

@@ -103,7 +103,7 @@ export async function pollConnector(
           const known = clippingTexts(db, userId, match.document);
           const highlights = await conn.pullHighlights(credential, {
             externalId: match.external_id, externalEdition: match.external_edition, confidence: match.confidence,
-          }, http, (t) => known.has(squash(t)));
+          }, http, (t) => known.has(squash(t)), { db, userId, document: match.document });
           options.signal?.throwIfAborted();
           // The slower EPUB/highlight request can outlive an unlink or rematch.
           const accountAfter = getAccount(db, userId, connectorId);

src/connectors/readest.ts

@@ -1,7 +1,7 @@
 import crypto from 'node:crypto';
 import { documentMeta, getMatchSnapshot, setMatchSnapshot } from './store.js';
 import { ConnectorOperationError, SAVE_CREDENTIAL, type SavableCredential } from './types.js';
-import { cachedEpub, epubQuoteRangeCfi, epubRangeCfi, epubXPath, normalizeEpubQuote, type Epub } from './bookfusion-epub.js';
+import { cachedEpub, epubQuoteRangeCfi, epubRangeCfi, epubRangeOffsets, epubXPath, normalizeEpubQuote, type Epub } from './bookfusion-epub.js';
 import type {
   Connector,
   ConnectorContext,
@@ -11,6 +11,7 @@ import type {
   DocumentMeta,
   HttpTransport,
   InboundChange,
+  InboundHighlight,
   Match,
   OutboundEvent,
   PushResult,
@@ -48,7 +49,9 @@ import type {
  * A clipping is placed in Readest's own copy of the book, so the book must be
  * in the reader's Readest cloud library: its chapter offsets become a range CFI
  * plus start and end XPointers there, checked against the clipping text. A
- * re-sent or edited clipping keeps its note id and updates in place.
+ * re-sent or edited clipping keeps its note id and updates in place. Readest
+ * highlights come back the same way: their range CFI in that copy becomes
+ * chapter offsets, and a highlight whose text isn't there is left out.
  */
 
 const DEFAULT_BASE = 'https://web.readest.com';
@@ -58,6 +61,7 @@ const MAX_TEXT = 500;
 const MS_PER_SECOND = 1000;
 const LINK_PAGE = 'https://web.readest.com/link';
 const CONNECTOR_ID = 'readest';
+const NOT_IN_LIBRARY = 'book is not in your Readest library (upload the EPUB to Readest Cloud)';
 // Readest's Supabase project and its public anon key, as its KOReader plugin ships them.
 const DEFAULT_SUPABASE_URL = 'https://vbsyxfusjjqdxkjqlysc.supabase.co';
 const DEFAULT_SUPABASE_ANON_KEY =
@@ -281,6 +285,7 @@ function epub(http: HttpTransport, c: ReadestCred, bookHash: string) {
   return cachedEpub(['readest', c.server, bookHash], async () => {
     const link = await http(`${c.server}/books/${encodeURIComponent(bookHash)}`, { method: 'GET', headers: headers(c) });
     if (link.status === 401) throw new ConnectorOperationError('unauthorized', false, true);
+    if (link.status === 404) throw new ConnectorOperationError(NOT_IN_LIBRARY, false);
     const url = link.status === 200 ? (await readJson(link)).downloadUrl : null;
     if (typeof url !== 'string') {
       throw new ConnectorOperationError(`Readest EPUB link failed (${link.status})`, link.status === 429 || link.status >= 500);
@@ -316,8 +321,8 @@ async function pushHighlight(
 ): Promise<PushResult> {
   const h = ev.highlight;
   if (!h || !h.text.trim()) return { ok: false, retryable: false, error: 'empty highlight' };
-  const book = await readestBook(c, cred, m, http, ctx);
-  if (!book) return { ok: false, retryable: false, error: 'book is not in your Readest library (upload the EPUB to Readest Cloud)' };
+  // A file Readest holds may have no library entry yet; its own hash still finds it.
+  const book = (await readestBook(c, cred, m, http, ctx)) ?? { bookHash: m.externalId, metaHash: null };
   const bytes = await epub(http, c, book.bookHash);
   const positioned = h.spine != null && h.startOffset != null && h.endOffset != null;
   const { cfi, text } = positioned
@@ -347,6 +352,74 @@ async function pushHighlight(
   return { ok: true };
 }
 
+// Books a pull found no Readest copy for, so each progress read doesn't list the whole library again.
+// ponytail: in-process only; a restart searches again.
+const noCopyUntil = new Map<string, number>();
+const SEARCH_AGAIN_MS = 60 * 60 * 1000;
+// Highlights whose text isn't at their CFI, so their book isn't downloaded again on every read.
+// Keyed with the CFI and edit time: an edited highlight is tried again.
+const unplaced = new Set<string>();
+export function clearReadestSearches(): void { noCopyUntil.clear(); unplaced.clear(); }
+
+/** Unix seconds from Readest's ISO timestamp (or epoch ms), 0 when missing. */
+function seconds(value: unknown): number {
+  const ms = typeof value === 'number' ? value : typeof value === 'string' ? Date.parse(value) : NaN;
+  return Number.isFinite(ms) ? Math.floor(ms / MS_PER_SECOND) : 0;
+}
+
+async function pullHighlights(
+  cred: Credential, m: Match, http: HttpTransport, skip: (text: string) => boolean, ctx?: ConnectorContext
+): Promise<InboundHighlight[]> {
+  const c = parseCred(cred);
+  if (!c) return [];
+  // Live highlights in one copy; its CFIs mean nothing in another copy's EPUB.
+  const highlightsIn = async (bookHash: string) => {
+    const res = await syncApi(http, c, cred, 'GET', `?since=0&type=notes&book=${encodeURIComponent(bookHash)}`);
+    return (Array.isArray(res.notes) ? res.notes : []).filter((n: any) =>
+      n?.type === 'annotation' && !n.deleted_at && n.book_hash === bookHash &&
+      typeof n.cfi === 'string' && typeof n.text === 'string' && n.text.trim());
+  };
+  // Readest keys notes by the file's own hash (which may have no library entry)
+  // and may also hold another copy of the book in its library.
+  const copies = [m.externalId];
+  const search = JSON.stringify([c.server, c.username ?? '', m.externalId]);
+  if ((noCopyUntil.get(search) ?? 0) <= Date.now()) {
+    const book = await readestBook(c, cred, m, http, ctx);
+    if (!book) noCopyUntil.set(search, Date.now() + SEARCH_AGAIN_MS);
+    else if (book.bookHash !== m.externalId) copies.push(book.bookHash);
+  }
+  const out: InboundHighlight[] = [];
+  for (const bookHash of copies) {
+    const key = (n: any) => JSON.stringify([c.server, c.username ?? '', bookHash, n.id, n.cfi, n.updated_at]);
+    const notes = (await highlightsIn(bookHash)).filter((n: any) => !skip(n.text) && !unplaced.has(key(n)));
+    if (!notes.length) continue;
+    let bytes: Epub;
+    try {
+      bytes = await epub(http, c, bookHash);
+    } catch (err) {
+      // A copy whose file Readest doesn't hold (notes without an upload) mustn't hide the other copy.
+      if (err instanceof ConnectorOperationError && !err.retryable && !err.needsReauth) continue;
+      throw err;
+    }
+    for (const n of notes) {
+      const at = await epubRangeOffsets(bytes, n.cfi).catch(() => null);
+      if (!at || normalizeEpubQuote(at.text) !== normalizeEpubQuote(n.text)) {
+        unplaced.add(key(n));
+        continue;
+      }
+      // Readest selections can take in the space around a word; whitespace is one codepoint each.
+      const text = at.text.trim();
+      const lead = at.text.length - at.text.trimStart().length;
+      out.push({
+        text, note: typeof n.note === 'string' && n.note.trim() ? n.note : null, chapter: null,
+        spine: at.spine, startOffset: at.start + lead, endOffset: at.start + lead + Array.from(text).length,
+        createdAt: seconds(n.created_at),
+      });
+    }
+  }
+  return out;
+}
+
 async function push(
   cred: Credential,
   m: Match,
@@ -536,6 +609,7 @@ export const readestConnector: Connector = {
   match,
   push,
   pullProgress,
+  pullHighlights,
   beginLink,
   pollLink,
   revoke,

src/connectors/types.ts

@@ -320,7 +320,7 @@ export interface Connector {
    * known ones (which can mean downloading the EPUB). Optional.
    */
   pullHighlights?(
-    cred: Credential, match: Match, http: HttpTransport, skip: (text: string) => boolean
+    cred: Credential, match: Match, http: HttpTransport, skip: (text: string) => boolean, ctx?: ConnectorContext
   ): Promise<InboundHighlight[]>;
 
   /** Begin an interactive device-code link (OAuth device grant). Optional. */

src/routes/landing.ts

@@ -47,7 +47,7 @@ const SERVICES: [string, string, string, 'ready' | 'beta'][] = [
   ['jellyfin', 'Jellyfin', 'Keep your place in sync with books on your Jellyfin server.', 'beta'],
   ['bookorbit', 'BookOrbit', 'Two-way progress with your own library, clippings as highlights.', 'beta'],
   ['seekquel', 'Seekquel', 'Log your progress, finished books and highlights in your Seekquel library.', 'beta'],
-  ['readest', 'Readest', 'Two-way progress with the Readest app, clippings as highlights.', 'beta'],
+  ['readest', 'Readest', 'Two-way progress and highlights with the Readest app.', 'beta'],
 ];
 
 const STYLE = `