From eafd58cf17c8f27ca67923c45d77afea95a73d78 Mon Sep 17 00:00:00 2001 From: Eszter Schuffert Date: Tue, 10 Nov 2026 21:37:56 +0100 Subject: [PATCH 1/3] feat(readest): import Readest highlights as clippings Readest highlights now come back as clippings for books in the Readest cloud library. Each is placed by its CFI in Readest's copy of the EPUB and kept only when its text is there. Readest stores notes under the file's own hash, which may have no library entry. Pull reads that hash first, and push now falls back to it too instead of failing with "not in your Readest library". --- app/src/Settings.jsx | 2 +- src/connectors/fanin.ts | 2 +- src/connectors/readest.ts | 69 ++++++++++++++++++-- src/connectors/types.ts | 2 +- src/routes/landing.ts | 2 +- test/readest.test.ts | 134 +++++++++++++++++++++++++++++++++++++- 6 files changed, 201 insertions(+), 10 deletions(-) diff --git a/app/src/Settings.jsx b/app/src/Settings.jsx index 4a5e3fe..a7298ea 100644 --- a/app/src/Settings.jsx +++ b/app/src/Settings.jsx @@ -51,7 +51,7 @@ const HINTS = { jellyfin: 'Syncs reading progress with your Jellyfin book library (Jellyfin 12.0 or newer). Sign in with your Jellyfin username and password.', bookorbit: 'Syncs progress both ways, adds clippings as highlights, and exchanges reading sessions for matched books. Imports BookOrbit history and exports sessions recorded by newer CrossInk firmware. Includes other BookOrbit reader history alongside your device stats. Repeated sessions are counted once.', seekquel: 'Logs your reading in your Seekquel library: where you are in each book, the books you finish, and your highlights. If you read further in Seekquel, that place comes back to your reader. You approve the request in the Seekquel app.', - readest: 'Syncs your reading progress with Readest and brings your place back when you read further in a Readest app. Clippings become Readest highlights for books in your Readest cloud library. Use this instead of signing in to the Readest plugin\'s progress sync, which moves your reader off CrossPoint Sync and stops clipping sync. You approve the request on readest.com.', + readest: 'Syncs your reading progress with Readest and brings your place back when you read further in a Readest app. Clippings become Readest highlights, and Readest highlights become clippings, for books in your Readest cloud library. Use this instead of signing in to the Readest plugin\'s progress sync, which moves your reader off CrossPoint Sync and stops clipping sync. You approve the request on readest.com.', spotify: 'Moves your progress forward when you listen ahead in the Spotify audiobook, and resumes it there. Audiobooks are in the US, UK, Canada, Ireland, Australia and New Zealand.', kindle: 'Linking needs the CrossPoint Kindle Link browser extension.', } diff --git a/src/connectors/fanin.ts b/src/connectors/fanin.ts index 17cd531..ee8c0e9 100644 --- a/src/connectors/fanin.ts +++ b/src/connectors/fanin.ts @@ -103,7 +103,7 @@ export async function pollConnector( const known = clippingTexts(db, userId, match.document); const highlights = await conn.pullHighlights(credential, { externalId: match.external_id, externalEdition: match.external_edition, confidence: match.confidence, - }, http, (t) => known.has(squash(t))); + }, http, (t) => known.has(squash(t)), { db, userId, document: match.document }); options.signal?.throwIfAborted(); // The slower EPUB/highlight request can outlive an unlink or rematch. const accountAfter = getAccount(db, userId, connectorId); diff --git a/src/connectors/readest.ts b/src/connectors/readest.ts index 7be9f08..386437e 100644 --- a/src/connectors/readest.ts +++ b/src/connectors/readest.ts @@ -1,7 +1,7 @@ import crypto from 'node:crypto'; import { documentMeta, getMatchSnapshot, setMatchSnapshot } from './store.js'; import { ConnectorOperationError, SAVE_CREDENTIAL, type SavableCredential } from './types.js'; -import { cachedEpub, epubQuoteRangeCfi, epubRangeCfi, epubXPath, normalizeEpubQuote, type Epub } from './bookfusion-epub.js'; +import { cachedEpub, epubQuoteRangeCfi, epubRangeCfi, epubRangeOffsets, epubXPath, normalizeEpubQuote, type Epub } from './bookfusion-epub.js'; import type { Connector, ConnectorContext, @@ -11,6 +11,7 @@ import type { DocumentMeta, HttpTransport, InboundChange, + InboundHighlight, Match, OutboundEvent, PushResult, @@ -48,7 +49,9 @@ import type { * A clipping is placed in Readest's own copy of the book, so the book must be * in the reader's Readest cloud library: its chapter offsets become a range CFI * plus start and end XPointers there, checked against the clipping text. A - * re-sent or edited clipping keeps its note id and updates in place. + * re-sent or edited clipping keeps its note id and updates in place. Readest + * highlights come back the same way: their range CFI in that copy becomes + * chapter offsets, and a highlight whose text isn't there is left out. */ const DEFAULT_BASE = 'https://web.readest.com'; @@ -58,6 +61,7 @@ const MAX_TEXT = 500; const MS_PER_SECOND = 1000; const LINK_PAGE = 'https://web.readest.com/link'; const CONNECTOR_ID = 'readest'; +const NOT_IN_LIBRARY = 'book is not in your Readest library (upload the EPUB to Readest Cloud)'; // Readest's Supabase project and its public anon key, as its KOReader plugin ships them. const DEFAULT_SUPABASE_URL = 'https://vbsyxfusjjqdxkjqlysc.supabase.co'; const DEFAULT_SUPABASE_ANON_KEY = @@ -281,6 +285,7 @@ function epub(http: HttpTransport, c: ReadestCred, bookHash: string) { return cachedEpub(['readest', c.server, bookHash], async () => { const link = await http(`${c.server}/books/${encodeURIComponent(bookHash)}`, { method: 'GET', headers: headers(c) }); if (link.status === 401) throw new ConnectorOperationError('unauthorized', false, true); + if (link.status === 404) throw new ConnectorOperationError(NOT_IN_LIBRARY, false); const url = link.status === 200 ? (await readJson(link)).downloadUrl : null; if (typeof url !== 'string') { throw new ConnectorOperationError(`Readest EPUB link failed (${link.status})`, link.status === 429 || link.status >= 500); @@ -316,8 +321,8 @@ async function pushHighlight( ): Promise { const h = ev.highlight; if (!h || !h.text.trim()) return { ok: false, retryable: false, error: 'empty highlight' }; - const book = await readestBook(c, cred, m, http, ctx); - if (!book) return { ok: false, retryable: false, error: 'book is not in your Readest library (upload the EPUB to Readest Cloud)' }; + // A file Readest holds may have no library entry yet; its own hash still finds it. + const book = (await readestBook(c, cred, m, http, ctx)) ?? { bookHash: m.externalId, metaHash: null }; const bytes = await epub(http, c, book.bookHash); const positioned = h.spine != null && h.startOffset != null && h.endOffset != null; const { cfi, text } = positioned @@ -347,6 +352,61 @@ async function pushHighlight( return { ok: true }; } +// Books a pull found no Readest copy for, so each progress read doesn't list the whole library again. +// ponytail: in-process only; a restart searches again. +const noCopyUntil = new Map(); +const SEARCH_AGAIN_MS = 60 * 60 * 1000; +export function clearReadestSearches(): void { noCopyUntil.clear(); } + +/** Unix seconds from Readest's ISO timestamp (or epoch ms), 0 when missing. */ +function seconds(value: unknown): number { + const ms = typeof value === 'number' ? value : typeof value === 'string' ? Date.parse(value) : NaN; + return Number.isFinite(ms) ? Math.floor(ms / MS_PER_SECOND) : 0; +} + +async function pullHighlights( + cred: Credential, m: Match, http: HttpTransport, skip: (text: string) => boolean, ctx?: ConnectorContext +): Promise { + const c = parseCred(cred); + if (!c) return []; + // Only live highlights in one copy: its CFIs mean nothing in another copy's EPUB. + const highlightsIn = async (bookHash: string) => { + const res = await syncApi(http, c, cred, 'GET', `?since=0&type=notes&book=${encodeURIComponent(bookHash)}`); + return (Array.isArray(res.notes) ? res.notes : []).filter((n: any) => + n?.type === 'annotation' && !n.deleted_at && n.book_hash === bookHash && + typeof n.cfi === 'string' && typeof n.text === 'string' && n.text.trim()); + }; + // Readest keys notes by the file's own hash, which a library entry may not exist for yet. + let bookHash = m.externalId; + let notes = await highlightsIn(bookHash); + if (!notes.length) { + const key = JSON.stringify([c.server, c.username ?? '', m.externalId]); + if ((noCopyUntil.get(key) ?? 0) > Date.now()) return []; + const book = await readestBook(c, cred, m, http, ctx); + if (!book) noCopyUntil.set(key, Date.now() + SEARCH_AGAIN_MS); + if (!book || book.bookHash === bookHash) return []; + bookHash = book.bookHash; + notes = await highlightsIn(bookHash); + } + notes = notes.filter((n: any) => !skip(n.text)); + if (!notes.length) return []; + const bytes = await epub(http, c, bookHash); + const out: InboundHighlight[] = []; + for (const n of notes) { + const at = await epubRangeOffsets(bytes, n.cfi).catch(() => null); + if (!at || !normalizeEpubQuote(at.text).startsWith(normalizeEpubQuote(n.text).slice(0, 40))) continue; + // Readest selections can take in the space around a word; whitespace is one codepoint each. + const text = at.text.trim(); + const lead = at.text.length - at.text.trimStart().length; + out.push({ + text, note: typeof n.note === 'string' && n.note.trim() ? n.note : null, chapter: null, + spine: at.spine, startOffset: at.start + lead, endOffset: at.start + lead + Array.from(text).length, + createdAt: seconds(n.created_at), + }); + } + return out; +} + async function push( cred: Credential, m: Match, @@ -536,6 +596,7 @@ export const readestConnector: Connector = { match, push, pullProgress, + pullHighlights, beginLink, pollLink, revoke, diff --git a/src/connectors/types.ts b/src/connectors/types.ts index 5549a3a..237f2b7 100644 --- a/src/connectors/types.ts +++ b/src/connectors/types.ts @@ -320,7 +320,7 @@ export interface Connector { * known ones (which can mean downloading the EPUB). Optional. */ pullHighlights?( - cred: Credential, match: Match, http: HttpTransport, skip: (text: string) => boolean + cred: Credential, match: Match, http: HttpTransport, skip: (text: string) => boolean, ctx?: ConnectorContext ): Promise; /** Begin an interactive device-code link (OAuth device grant). Optional. */ diff --git a/src/routes/landing.ts b/src/routes/landing.ts index b84f34c..d1c220d 100644 --- a/src/routes/landing.ts +++ b/src/routes/landing.ts @@ -47,7 +47,7 @@ const SERVICES: [string, string, string, 'ready' | 'beta'][] = [ ['jellyfin', 'Jellyfin', 'Keep your place in sync with books on your Jellyfin server.', 'beta'], ['bookorbit', 'BookOrbit', 'Two-way progress with your own library, clippings as highlights.', 'beta'], ['seekquel', 'Seekquel', 'Log your progress, finished books and highlights in your Seekquel library.', 'beta'], - ['readest', 'Readest', 'Two-way progress with the Readest app, clippings as highlights.', 'beta'], + ['readest', 'Readest', 'Two-way progress and highlights with the Readest app.', 'beta'], ]; const STYLE = ` diff --git a/test/readest.test.ts b/test/readest.test.ts index 7437cf4..66393f4 100644 --- a/test/readest.test.ts +++ b/test/readest.test.ts @@ -5,7 +5,7 @@ import { DOC, makeTestApp, registerUser } from './helpers.js'; import { resetEncryptionKeyCache } from '../src/crypto/secrets.js'; import type { HttpTransport, OutboundEvent } from '../src/connectors/types.js'; import { drainQueue } from '../src/connectors/runner.js'; -import { readestBase, readestConnector } from '../src/connectors/readest.js'; +import { clearReadestSearches, readestBase, readestConnector } from '../src/connectors/readest.js'; import { SAVE_CREDENTIAL } from '../src/connectors/types.js'; import { clearBookFusionEpubCache } from '../src/connectors/bookfusion-epub.js'; @@ -100,7 +100,7 @@ const progressEvent = (over: Partial = {}): OutboundEvent => ({ const body = (call: { body?: string }) => JSON.parse(call.body ?? '{}'); const KEY = { TOKEN_ENC_KEY: 'a'.repeat(64) }; -beforeEach(() => { Object.assign(process.env, KEY); resetEncryptionKeyCache(); clearBookFusionEpubCache(); }); +beforeEach(() => { Object.assign(process.env, KEY); resetEncryptionKeyCache(); clearBookFusionEpubCache(); clearReadestSearches(); }); afterEach(() => { vi.useRealTimers(); delete process.env.TOKEN_ENC_KEY; delete process.env.READEST_URL; resetEncryptionKeyCache(); }); describe('readest connector (unit)', () => { @@ -219,6 +219,7 @@ describe('readest connector highlights', () => { it('gives up on a book that is not in the Readest library, and on text that is not at its position', async () => { const fake = fakeTransport(); fake.on('/api/sync', 200, library([])); + fake.on(`/api/crosspoint/books/${DOC}`, 404, { error: 'Book not found' }); expect(await readestConnector.push(CRED, MATCH, highlightEvent(), fake.transport)).toMatchObject({ ok: false, retryable: false, error: expect.stringContaining('not in your Readest library'), }); @@ -228,6 +229,17 @@ describe('readest connector highlights', () => { expect(fake.calls.some((c) => c.method === 'POST')).toBe(false); }); + it('places a clipping in a file Readest holds before its library entry exists', async () => { + const fake = fakeTransport(); + fake.on('/api/sync', 200, library([])); + fake.on(`/api/crosspoint/books/${DOC}`, 200, { downloadUrl: 'https://files.readest.test/book.epub?sig=1' }); + fake.on('https://files.readest.test/', 200, EPUB); + expect(await readestConnector.push(CRED, MATCH, highlightEvent(), fake.transport)).toEqual({ ok: true }); + const note = body(fake.calls.find((c) => c.method === 'POST')!).notes[0]; + expect(note).toMatchObject({ bookHash: DOC, text: 'bold tail', cfi: 'epubcfi(/6/4!/4/4,/2/1:0,/3:5)' }); + expect(note).not.toHaveProperty('metaHash'); + }); + it('refreshes an expired session once and saves the rotated tokens', async () => { const fake = fakeTransport(); bookFiles(fake); @@ -253,6 +265,94 @@ describe('readest connector highlights', () => { }); }); +// Readest's own note rows, as GET /api/sync returns them. +const readestNote = (over: Record = {}) => ({ + book_hash: BOOK, meta_hash: 'meta-1', id: 'n1', type: 'annotation', style: 'highlight', color: 'yellow', + cfi: 'epubcfi(/6/4!/4/4,/2/1:0,/3:5)', text: 'bold tail', note: 'from Readest', + created_at: '2025-10-09T08:53:20.000Z', updated_at: '2025-10-09T08:53:20.000Z', deleted_at: null, ...over, +}); +function readestLibrary(fake: ReturnType, notes: unknown[]) { + bookFiles(fake); + fake.on('/api/sync', 200, library([libraryBook()])); + fake.on('type=notes', 200, { books: [], configs: [], notes }); +} + +describe('readest connector highlight pull', () => { + it('brings back a Readest highlight as chapter offsets in its copy of the book', async () => { + const fake = fakeTransport(); + readestLibrary(fake, [readestNote()]); + expect(await readestConnector.pullHighlights!(CRED, MATCH, fake.transport, () => false)).toEqual([{ + text: 'bold tail', note: 'from Readest', chapter: null, spine: 1, startOffset: 8, endOffset: 17, createdAt: 1_760_000_000, + }]); + // Nothing under our own hash, so the library copy with the same title is asked next. + expect(fake.calls.filter((c) => c.url.includes('type=notes')).map((c) => c.url)).toEqual([ + `https://web.readest.com/api/sync?since=0&type=notes&book=${DOC}`, + `https://web.readest.com/api/sync?since=0&type=notes&book=${BOOK}`, + ]); + expect(fake.calls.find((c) => c.url.includes('type=notes'))!.headers?.authorization).toBe('Bearer at-1'); + }); + + it('reads highlights stored under the file\'s own hash without a library entry', async () => { + const fake = fakeTransport(); + fake.on(`/api/crosspoint/books/${DOC}`, 200, { downloadUrl: 'https://files.readest.test/book.epub?sig=1' }); + fake.on('https://files.readest.test/', 200, EPUB); + fake.on('/api/sync', 200, library([])); + fake.on('type=notes', 200, { books: [], configs: [], notes: [readestNote({ book_hash: DOC })] }); + expect(await readestConnector.pullHighlights!(CRED, MATCH, fake.transport, () => false)).toEqual([ + expect.objectContaining({ text: 'bold tail', spine: 1, startOffset: 8, endOffset: 17 }), + ]); + expect(fake.calls.some((c) => c.url.includes('type=books'))).toBe(false); + }); + + it('leaves out known, deleted, other-copy and misplaced highlights, and bookmarks', async () => { + const fake = fakeTransport(); + readestLibrary(fake, [ + readestNote({ id: 'known', text: 'Pre' , cfi: 'epubcfi(/6/4!/4/4,/1:0,/1:3)' }), + readestNote({ id: 'gone', deleted_at: '2026-01-01T00:00:00Z' }), + readestNote({ id: 'other', book_hash: 'c'.repeat(32) }), + readestNote({ id: 'moved', text: 'something else' }), + readestNote({ id: 'mark', type: 'bookmark' }), + readestNote({ id: 'broken', cfi: 'epubcfi(/6/99!/4)' }), + ]); + expect(await readestConnector.pullHighlights!(CRED, MATCH, fake.transport, (t) => t === 'Pre')).toEqual([]); + // Nothing left to place, so the EPUB is never fetched for the skipped ones alone. + const none = fakeTransport(); + readestLibrary(none, [readestNote({ deleted_at: '2026-01-01T00:00:00Z' })]); + expect(await readestConnector.pullHighlights!(CRED, MATCH, none.transport, () => false)).toEqual([]); + expect(none.calls.some((c) => c.url.startsWith('https://files.readest.test/'))).toBe(false); + }); + + it('leaves the spaces a Readest selection took in out of the clipping', async () => { + const fake = fakeTransport(); + // " tail" is 12-17 in chapter 2; Readest took in the space before it. + readestLibrary(fake, [readestNote({ cfi: 'epubcfi(/6/4!/4/4,/3:0,/3:5)', text: ' tail' })]); + expect(await readestConnector.pullHighlights!(CRED, MATCH, fake.transport, () => false)).toEqual([ + expect.objectContaining({ text: 'tail', startOffset: 13, endOffset: 17 }), + ]); + }); + + it('lists the Readest library at most once an hour for a book Readest does not have', async () => { + const fake = fakeTransport(); + fake.on('/api/sync', 200, library([])); + const listings = () => fake.calls.filter((c) => c.url.endsWith('?since=0&type=books')).length; + const cred = { ...CRED, expires_at: Math.floor(Date.now() / 1000) + 24 * 3600 }; + await readestConnector.pullHighlights!(cred, MATCH, fake.transport, () => false); + await readestConnector.pullHighlights!(cred, MATCH, fake.transport, () => false); + expect(listings()).toBe(1); + vi.useFakeTimers({ toFake: ['Date'], now: Date.now() + 61 * 60_000 }); + await readestConnector.pullHighlights!(cred, MATCH, fake.transport, () => false); + expect(listings()).toBe(2); + }); + + it('brings back nothing for a book that is not in the Readest library', async () => { + const fake = fakeTransport(); + fake.on('/api/sync', 200, library([])); + expect(await readestConnector.pullHighlights!(CRED, MATCH, fake.transport, () => false)).toEqual([]); + expect(fake.calls.filter((c) => c.url.includes('type=notes'))).toHaveLength(1); + expect(fake.calls.some((c) => c.url.startsWith('https://files.readest.test/'))).toBe(false); + }); +}); + describe('readest connector (end to end)', () => { it('links by device code, then forwards a device sync', async () => { const fake = fakeTransport(); @@ -312,6 +412,36 @@ describe('readest connector (end to end)', () => { expect(body(post!).notes[0]).toMatchObject({ bookHash: BOOK, text: 'bold tail', note: 'nice', cfi: 'epubcfi(/6/4!/4/4,/2/1:0,/3:5)' }); }); + it('imports a Readest highlight as a clipping when the reader syncs', async () => { + const fake = fakeTransport(); + const { app } = makeTestApp({}, { connectorTransport: fake.transport }); + const { headers } = await registerUser(app); + signIn(fake); + const { device_code } = await (await app.request('/api/v1/connectors/readest/link/begin', { method: 'POST', headers })).json(); + await app.request('/api/v1/connectors/readest/link/poll', { method: 'POST', headers, body: JSON.stringify({ device_code }) }); + readestLibrary(fake, [readestNote()]); + await app.request('/syncs/progress', { + method: 'PUT', headers, + body: JSON.stringify({ document: DOC, progress: XPOINTER, percentage: 0.5, device_id: 'd1', metadata: { title: 'Piranesi', authors: 'Susanna Clarke' } }), + }); + // A progress read refreshes the book from Readest, highlights included. + await app.request(`/syncs/progress/${DOC}`, { headers }); + const clips = await (await app.request(`/api/v1/clippings/${DOC}`, { headers })).json(); + expect(clips.items).toEqual([expect.objectContaining({ text: 'bold tail', note: 'from Readest', spine: 1, start_offset: 8, end_offset: 17 })]); + // A second read finds it known and adds nothing, and nothing is sent back to Readest. + await app.request(`/syncs/progress/${DOC}`, { headers }); + expect((await (await app.request(`/api/v1/clippings/${DOC}`, { headers })).json()).items).toHaveLength(1); + expect(fake.calls.some((c) => c.method === 'POST' && c.url.endsWith('/api/sync'))).toBe(false); + + // Deleted on the reader: Readest still has it, but it stays deleted here. + const id = clips.items[0].id; + await app.request(`/api/v1/clippings/${DOC}`, { method: 'PUT', headers, body: JSON.stringify({ items: [{ id, deleted: 1 }] }) }); + await app.request(`/syncs/progress/${DOC}`, { headers }); + expect((await (await app.request(`/api/v1/clippings/${DOC}`, { headers })).json()).items).toEqual([ + expect.objectContaining({ id, deleted: 1 }), + ]); + }); + it('revokes the reader key and signs the session out on unlink, and the old ones on a new link', async () => { const fake = fakeTransport(); const { app } = makeTestApp({}, { connectorTransport: fake.transport }); From 6dc24bd3490ba551b9749ae1d837b3833d73abd8 Mon Sep 17 00:00:00 2001 From: Eszter Schuffert Date: Tue, 10 Nov 2026 21:55:06 +0100 Subject: [PATCH 2/3] fix(readest): read both copies and match full highlight text - Import highlights from the library copy too, not only the file's own copy. - Require the whole highlight text to match at its position. - Remember highlights that can't be placed, so their book isn't downloaded again on every sync. An edited highlight is tried again. --- src/connectors/readest.ts | 58 +++++++++++++++++++++------------------ test/readest.test.ts | 31 ++++++++++++++++++++- 2 files changed, 62 insertions(+), 27 deletions(-) diff --git a/src/connectors/readest.ts b/src/connectors/readest.ts index 386437e..24359f8 100644 --- a/src/connectors/readest.ts +++ b/src/connectors/readest.ts @@ -356,7 +356,10 @@ async function pushHighlight( // ponytail: in-process only; a restart searches again. const noCopyUntil = new Map(); const SEARCH_AGAIN_MS = 60 * 60 * 1000; -export function clearReadestSearches(): void { noCopyUntil.clear(); } +// Highlights whose text isn't at their CFI, so their book isn't downloaded again on every read. +// Keyed with the CFI and edit time: an edited highlight is tried again. +const unplaced = new Set(); +export function clearReadestSearches(): void { noCopyUntil.clear(); unplaced.clear(); } /** Unix seconds from Readest's ISO timestamp (or epoch ms), 0 when missing. */ function seconds(value: unknown): number { @@ -369,40 +372,43 @@ async function pullHighlights( ): Promise { const c = parseCred(cred); if (!c) return []; - // Only live highlights in one copy: its CFIs mean nothing in another copy's EPUB. + // Live highlights in one copy; its CFIs mean nothing in another copy's EPUB. const highlightsIn = async (bookHash: string) => { const res = await syncApi(http, c, cred, 'GET', `?since=0&type=notes&book=${encodeURIComponent(bookHash)}`); return (Array.isArray(res.notes) ? res.notes : []).filter((n: any) => n?.type === 'annotation' && !n.deleted_at && n.book_hash === bookHash && typeof n.cfi === 'string' && typeof n.text === 'string' && n.text.trim()); }; - // Readest keys notes by the file's own hash, which a library entry may not exist for yet. - let bookHash = m.externalId; - let notes = await highlightsIn(bookHash); - if (!notes.length) { - const key = JSON.stringify([c.server, c.username ?? '', m.externalId]); - if ((noCopyUntil.get(key) ?? 0) > Date.now()) return []; + // Readest keys notes by the file's own hash (which may have no library entry) + // and may also hold another copy of the book in its library. + const copies = [m.externalId]; + const search = JSON.stringify([c.server, c.username ?? '', m.externalId]); + if ((noCopyUntil.get(search) ?? 0) <= Date.now()) { const book = await readestBook(c, cred, m, http, ctx); - if (!book) noCopyUntil.set(key, Date.now() + SEARCH_AGAIN_MS); - if (!book || book.bookHash === bookHash) return []; - bookHash = book.bookHash; - notes = await highlightsIn(bookHash); + if (!book) noCopyUntil.set(search, Date.now() + SEARCH_AGAIN_MS); + else if (book.bookHash !== m.externalId) copies.push(book.bookHash); } - notes = notes.filter((n: any) => !skip(n.text)); - if (!notes.length) return []; - const bytes = await epub(http, c, bookHash); const out: InboundHighlight[] = []; - for (const n of notes) { - const at = await epubRangeOffsets(bytes, n.cfi).catch(() => null); - if (!at || !normalizeEpubQuote(at.text).startsWith(normalizeEpubQuote(n.text).slice(0, 40))) continue; - // Readest selections can take in the space around a word; whitespace is one codepoint each. - const text = at.text.trim(); - const lead = at.text.length - at.text.trimStart().length; - out.push({ - text, note: typeof n.note === 'string' && n.note.trim() ? n.note : null, chapter: null, - spine: at.spine, startOffset: at.start + lead, endOffset: at.start + lead + Array.from(text).length, - createdAt: seconds(n.created_at), - }); + for (const bookHash of copies) { + const key = (n: any) => JSON.stringify([c.server, c.username ?? '', bookHash, n.id, n.cfi, n.updated_at]); + const notes = (await highlightsIn(bookHash)).filter((n: any) => !skip(n.text) && !unplaced.has(key(n))); + if (!notes.length) continue; + const bytes = await epub(http, c, bookHash); + for (const n of notes) { + const at = await epubRangeOffsets(bytes, n.cfi).catch(() => null); + if (!at || normalizeEpubQuote(at.text) !== normalizeEpubQuote(n.text)) { + unplaced.add(key(n)); + continue; + } + // Readest selections can take in the space around a word; whitespace is one codepoint each. + const text = at.text.trim(); + const lead = at.text.length - at.text.trimStart().length; + out.push({ + text, note: typeof n.note === 'string' && n.note.trim() ? n.note : null, chapter: null, + spine: at.spine, startOffset: at.start + lead, endOffset: at.start + lead + Array.from(text).length, + createdAt: seconds(n.created_at), + }); + } } return out; } diff --git a/test/readest.test.ts b/test/readest.test.ts index 66393f4..cc9efed 100644 --- a/test/readest.test.ts +++ b/test/readest.test.ts @@ -301,7 +301,36 @@ describe('readest connector highlight pull', () => { expect(await readestConnector.pullHighlights!(CRED, MATCH, fake.transport, () => false)).toEqual([ expect.objectContaining({ text: 'bold tail', spine: 1, startOffset: 8, endOffset: 17 }), ]); - expect(fake.calls.some((c) => c.url.includes('type=books'))).toBe(false); + }); + + it('imports highlights from both the file\'s own copy and the library copy', async () => { + const fake = fakeTransport(); + readestLibrary(fake, []); + fake.on(`/api/crosspoint/books/${DOC}`, 200, { downloadUrl: 'https://files.readest.test/book.epub?sig=1' }); + fake.on(`type=notes&book=${DOC}`, 200, { books: [], configs: [], notes: [readestNote({ book_hash: DOC })] }); + fake.on(`type=notes&book=${BOOK}`, 200, { books: [], configs: [], notes: [readestNote({ id: 'n2', cfi: 'epubcfi(/6/4!/4/4,/1:0,/1:3)', text: 'Pre' })] }); + const got = await readestConnector.pullHighlights!(CRED, MATCH, fake.transport, () => false); + expect(got.map((h) => h.text).sort()).toEqual(['Pre', 'bold tail']); + }); + + it('skips a highlight whose position covers more text than the highlight', async () => { + const fake = fakeTransport(); + // The range covers "Pre bold tail"; the note says only "Pre". + readestLibrary(fake, [readestNote({ cfi: 'epubcfi(/6/4!/4/4,/1:0,/3:5)', text: 'Pre' })]); + expect(await readestConnector.pullHighlights!(CRED, MATCH, fake.transport, () => false)).toEqual([]); + }); + + it('does not download the book again for a highlight it could not place, until it is edited', async () => { + const fake = fakeTransport(); + readestLibrary(fake, [readestNote({ text: 'something else' })]); + const downloads = () => fake.calls.filter((c) => c.url.startsWith('https://files.readest.test/')).length; + await readestConnector.pullHighlights!(CRED, MATCH, fake.transport, () => false); + clearBookFusionEpubCache(); + await readestConnector.pullHighlights!(CRED, MATCH, fake.transport, () => false); + expect(downloads()).toBe(1); + fake.on('type=notes', 200, { books: [], configs: [], notes: [readestNote({ updated_at: '2026-01-02T00:00:00Z' })] }); + expect(await readestConnector.pullHighlights!(CRED, MATCH, fake.transport, () => false)).toHaveLength(1); + expect(downloads()).toBe(2); }); it('leaves out known, deleted, other-copy and misplaced highlights, and bookmarks', async () => { From 19a57cebbaeba3c2d79b5defd31930c6d27c37f8 Mon Sep 17 00:00:00 2001 From: Eszter Schuffert Date: Tue, 10 Nov 2026 22:09:55 +0100 Subject: [PATCH 3/3] fix(readest): skip a copy whose EPUB Readest doesn't have A 404 for one copy no longer stops highlights from the other copy. Server and sign-in errors still fail the pull so they are retried. --- src/connectors/readest.ts | 9 ++++++++- test/readest.test.ts | 12 ++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/connectors/readest.ts b/src/connectors/readest.ts index 24359f8..e92d7a3 100644 --- a/src/connectors/readest.ts +++ b/src/connectors/readest.ts @@ -393,7 +393,14 @@ async function pullHighlights( const key = (n: any) => JSON.stringify([c.server, c.username ?? '', bookHash, n.id, n.cfi, n.updated_at]); const notes = (await highlightsIn(bookHash)).filter((n: any) => !skip(n.text) && !unplaced.has(key(n))); if (!notes.length) continue; - const bytes = await epub(http, c, bookHash); + let bytes: Epub; + try { + bytes = await epub(http, c, bookHash); + } catch (err) { + // A copy whose file Readest doesn't hold (notes without an upload) mustn't hide the other copy. + if (err instanceof ConnectorOperationError && !err.retryable && !err.needsReauth) continue; + throw err; + } for (const n of notes) { const at = await epubRangeOffsets(bytes, n.cfi).catch(() => null); if (!at || normalizeEpubQuote(at.text) !== normalizeEpubQuote(n.text)) { diff --git a/test/readest.test.ts b/test/readest.test.ts index cc9efed..cd3c728 100644 --- a/test/readest.test.ts +++ b/test/readest.test.ts @@ -313,6 +313,18 @@ describe('readest connector highlight pull', () => { expect(got.map((h) => h.text).sort()).toEqual(['Pre', 'bold tail']); }); + it('still imports the library copy when the file\'s own copy cannot be downloaded', async () => { + const fake = fakeTransport(); + readestLibrary(fake, []); + fake.on(`/api/crosspoint/books/${DOC}`, 404, { error: 'Book not found' }); + fake.on(`type=notes&book=${DOC}`, 200, { books: [], configs: [], notes: [readestNote({ book_hash: DOC, text: 'Pre', cfi: 'epubcfi(/6/4!/4/4,/1:0,/1:3)' })] }); + fake.on(`type=notes&book=${BOOK}`, 200, { books: [], configs: [], notes: [readestNote()] }); + expect((await readestConnector.pullHighlights!(CRED, MATCH, fake.transport, () => false)).map((h) => h.text)).toEqual(['bold tail']); + // A server error still fails the pull, so it is retried. + fake.on(`/api/crosspoint/books/${DOC}`, 503, {}); + await expect(readestConnector.pullHighlights!(CRED, MATCH, fake.transport, () => false)).rejects.toThrow('503'); + }); + it('skips a highlight whose position covers more text than the highlight', async () => { const fake = fakeTransport(); // The range covers "Pre bold tail"; the note says only "Pre".