From ae57cc8eec15a300c731f77ab92c24bf004789c0 Mon Sep 17 00:00:00 2001 From: Logan Abell Date: Thu, 15 Oct 2026 01:13:30 -0400 Subject: [PATCH 1/4] Fix catalog-backed release names and EPUB font-obfuscation review --- docs/AUTOMATIC-BOOK-MATCHING.md | 8 +- docs/issues/2026-10-15-release-matching.md | 41 ++++++ services/app/domain/indexer_labels.py | 16 ++- services/app/domain/release_profiles.py | 19 ++- services/app/domain/series_identity.py | 52 +++++-- services/app/importing/inspection.py | 67 ++++++++- services/app/importing/matching.py | 2 +- tests/integration/test_automatic_imports.py | 16 ++- tests/integration/test_quick_add.py | 32 +++++ tests/integration/test_sabnzbd.py | 34 +++-- tests/integration/test_series_file_import.py | 1 + tests/integration/test_series_file_review.py | 1 + .../test_single_file_acquisition.py | 3 + tests/media_fixtures.py | 20 ++- tests/unit/test_automatic_eligibility.py | 40 ++++++ tests/unit/test_epub_encryption.py | 109 +++++++++++++++ tests/unit/test_release_name_identity.py | 127 ++++++++++++++++++ 17 files changed, 550 insertions(+), 38 deletions(-) create mode 100644 docs/issues/2026-10-15-release-matching.md create mode 100644 tests/unit/test_epub_encryption.py create mode 100644 tests/unit/test_release_name_identity.py diff --git a/docs/AUTOMATIC-BOOK-MATCHING.md b/docs/AUTOMATIC-BOOK-MATCHING.md index 82c28d7e..166d7185 100644 --- a/docs/AUTOMATIC-BOOK-MATCHING.md +++ b/docs/AUTOMATIC-BOOK-MATCHING.md @@ -19,7 +19,9 @@ Minor differences between two long descriptive subtitles require a shared editio Matching revisions include the new rules so cached failures can be reconsidered. Library matches are reconsidered on the next eligible sync; held imports can use **Check again automatically**. -For a single completed download, the saved request and agreeing release metadata may identify otherwise untagged files. Prowlarr releases may supply an exact author/title pair in their name. Series-only credits still need independent embedded evidence. Conflicting file metadata and requests for a specific edition retain stricter review. +For a single completed download, the saved request and agreeing release metadata may identify otherwise untagged files. Prowlarr releases may supply an exact author/title pair in their name. A series-only embedded title also needs agreeing authors, a complete single-book download, and fresh, unique catalog series membership at the same position. Conflicting file metadata and requests for a specific edition retain stricter review. + +Release names may place a known series and its position before or after the complete book title. Positions can use digits or canonical Roman numerals, but must agree with the catalog. For example, `Dark Tower II: The Drawing of the Three - Stephen King` can identify that book at position 2. Dotted year, retail, ebook-format and release-group suffixes can follow an exact author/title pair. These labels do not replace inspection of the downloaded files or establish their medium or completeness. One flat numbered audio sequence (`01.mp3` through `12.mp3`, for example) supplies missing track order and totals when the downloaded manifest is complete. Gaps, duplicate numbers, conflicting tags, incomplete disc sets and partial-release labels still block automatic import. Filename order does not establish book identity on its own. Older inspections can acquire a new proposed order and review revision without rewriting their original evidence or an explicit grouping review. @@ -32,3 +34,7 @@ Missing embedded titles or authors do not invalidate a unique edition identifier Successful automatic imports require no review. A held download can continue automatically with its existing association. When completeness cannot be established, the reviewer confirms the files contain the complete book and adds it to the library; no book or edition picker is shown. Incorrect or ambiguous downloads link to the existing source search for that same book. Correcting catalog metadata uses the existing book metadata page and its Hardcover search. Downloaded filenames are optional in a small dialog. Naming templates, hashes, grouping identifiers, inspection dumps and the generic catalog picker are absent from linked-download review. The standalone local-folder import tool remains separate. Import retries preserve their frozen plan and request key, and library confirmation is still required before a request becomes satisfied. + +EPUBs using standard IDPF font obfuscation can be inspected and imported with their original bytes intact. Other encryption methods, encrypted book content, and invalid font references remain held. The presence of `encryption.xml` alone no longer causes a hold; see the [EPUB font-obfuscation specification](https://www.w3.org/TR/epub-33/#sec-obfuscation-specifying). + +After upgrading, open an EPUB held by the old encryption check and choose **Check files again** to rebuild its inspection from the saved download. A source-name hold can be reassessed by searching for the same book and choosing the release again. A continuing hold still needs its stated identity, content, or permission problem resolved; opening the file in another reader does not establish all of those facts. diff --git a/docs/issues/2026-10-15-release-matching.md b/docs/issues/2026-10-15-release-matching.md new file mode 100644 index 00000000..5fedfa71 --- /dev/null +++ b/docs/issues/2026-10-15-release-matching.md @@ -0,0 +1,41 @@ +# Release matching and review investigation + +Scope: [Dewarr: Release matching and review](https://linear.app/northernlogic/project/dewarr-release-matching-and-review-7dee1b88da14), based on `371ed7c` (2026-10-15). The earlier NOR-87 fix is already merged in #69. This work extends its shared series-label contract and checks its review/import safeguards. + +## Confirmed defects and fixes + +| Issue | Cause | Result | +| --- | --- | --- | +| [NOR-95 / #62](https://github.com/logabell/dewarr/issues/62) | Series notes removed Arabic digits but left Roman numerals as unmatched text. | Canonical Roman positions match an equal catalog position after recognizing the series name. Wrong positions, malformed numerals and ranges remain rejected. | +| [NOR-97 / #64](https://github.com/logabell/dewarr/issues/64) | The parser recognized series suffixes, but not a series/position prefix before the full book title. The first hyphen could also belong to that prefix. | Both reported title layouts retain the full catalog title and exact author credit, with a known series and matching position required. | +| [NOR-99 / #66](https://github.com/logabell/dewarr/issues/66) | Trailing-label parsing omitted `ebook` and supported only uppercase release groups after audio extensions. | The exact reported dotted retail EPUB release identifies its author/title. Content warnings and actual-file format checks remain separate. | +| [NOR-89 / #56](https://github.com/logabell/dewarr/issues/56), supported EPUB case | Inspection rejected any `META-INF/encryption.xml` before inspecting its algorithm or resources. | Only IDPF font obfuscation of local, declared, non-spine font resources is permitted. Unknown encryption, encrypted chapters, path escapes and ambiguous declarations remain held. | + +The reported release names failed a regression check before the parser changes. A synthetic EPUB with standard font-obfuscation metadata likewise failed inspection before the EPUB fix. The synthetic font bytes are a parser fixture, not a test of font rendering or deobfuscation; Dewarr neither renders nor rewrites them. + +## Unresolved report details + +NOR-89 remains open. Its actual EPUB algorithm/resource list is unavailable, so the supported-font fix is not confirmation of the reporter's file. Adobe/other font-obfuscation algorithms remain unsupported. + +The controlled administrator flow withdraws a request, requests it again, completes selection, and records exactly one new download. Existing source and download review screens expose the expected actions in the tested states. This does not reproduce the reporter's missing actions or establish their user role, request history, or deployment version. No blanket approval action or request-state workaround was added. Logan authorized continuing without newer reproduction details. + +## Validation + +All checks use the repository's serial, bounded `python3 scripts/check.py` runner and synthetic media. API/worker journeys use a disposable PostgreSQL database; external indexer/downloader/library calls use fixtures. No live reporter deployment or external download was exercised. + +- 316 focused unit/boundary checks: release names, existing release profiles, series-download identity, automatic eligibility, EPUB encryption, and download inspection. +- 31 API/worker journeys: clicked Prowlarr/SABnzbd release names, withdraw/re-request, original-byte-preserving font-obfuscated EPUB import, series file review/confirmation/replay, and automatic series imports. +- 16 browser checks with fixture API responses: source selection, collection review, download matching, and unreadable-file reinspection. Production frontend build passed. +- 23 further API/worker checks passed after advancing the matching-rule revision: the 21 series-review cases above plus two saved-inspection retry cases. A prior font-encryption hold is re-inspected without submitting another download (33 distinct API/worker cases in total). +- Ruff lint/format and diff whitespace checks pass. + +Focused commands: + +```sh +python3 scripts/check.py unit tests/unit/test_release_name_identity.py tests/unit/test_release_profiles.py tests/unit/test_series_download_identity.py tests/unit/test_automatic_eligibility.py tests/unit/test_epub_encryption.py tests/unit/test_download_inspection.py +python3 scripts/check.py backend tests/integration/test_sabnzbd.py::test_clicked_usenet_release_with_catalog_labels_reaches_downloader tests/integration/test_quick_add.py::test_withdraw_then_request_again_starts_one_new_download 'tests/integration/test_single_file_acquisition.py::test_single_epub_download_to_confirmed_library_keeps_neighbor_private[automatic-obfuscated-font-]' tests/integration/test_series_file_review.py tests/integration/test_series_file_import.py +python3 scripts/check.py backend tests/integration/test_automatic_imports.py::test_retry_reinspects_unreadable_book_without_redownloading tests/integration/test_series_file_review.py +python3 scripts/check.py ui quick-add-sources.spec.ts download-matching.spec.ts collection-review.spec.ts download-inspection-errors.spec.ts +``` + +See [automatic book matching](../AUTOMATIC-BOOK-MATCHING.md) for user-facing retry instructions. This change does not mark NOR-89 resolved or repeat NOR-87's already-shipped implementation. diff --git a/services/app/domain/indexer_labels.py b/services/app/domain/indexer_labels.py index bfc4e536..59ef872e 100644 --- a/services/app/domain/indexer_labels.py +++ b/services/app/domain/indexer_labels.py @@ -7,7 +7,7 @@ # Full names avoid mistaking short language codes for initials or title words. LANGUAGES = {name: code for name, code in LANGUAGE_ALIASES.items() if len(name) > 3} LANGUAGES["deutsch"] = "de" -MEDIA = frozenset({"audiobook", "abook", "hoerbuch", "hörbuch", "hoerspiel", "hörspiel"}) +MEDIA = frozenset({"audiobook", "abook", "hoerbuch", "hörbuch", "hoerspiel", "hörspiel", "ebook"}) FORMATS = frozenset({"m4b", "mp3", "epub", "pdf", "flac", "aac", "ogg", "opus", "azw3", "mobi"}) LABELS = MEDIA | FORMATS | LANGUAGES.keys() | {"unabridged", "ungekuerzt", "ungekürzt", "retail"} _LABEL = rf"(?:{'|'.join(sorted(LABELS))}|(?:19|20)\d{{2}})" @@ -17,7 +17,10 @@ rf"[.\s_]+(?P{_LABEL}))\s*$", re.I, ) -_GROUP = re.compile(r"(?i:(?:\.|\s)(?:m4b|mp3|flac|aac|ogg|opus))-(?P[A-Z0-9]{2,16})$") +_GROUP = re.compile( + rf"(?:\.|\s)(?:{'|'.join(sorted(FORMATS))})(?:[.\s]+ebook)?-(?P[a-z0-9]{{2,16}})$", + re.I, +) def indexer_suffixes(value): @@ -29,9 +32,14 @@ def indexer_suffixes(value): labels = [] yield value, labels if group := _GROUP.search(value): - # A scene group after an audio extension is metadata. Partial/preview + # A scene group after a media extension is metadata. Partial/preview # markers are content warnings even if they use the same spelling. - if not re.fullmatch(r"SAMPLE|PREVIEW|SUMMARY|PART\d*|CD\d*|DISC\d*", group["group"]): + if not re.fullmatch( + r"sample|preview|summary|excerpt|incomplete|truncated|collection|pack|omnibus|" + r"anthology|boxset|part\d*|cd\d*|disc\d*", + group["group"], + re.I, + ): value = value[: group.start("group") - 1] while match := _SUFFIX.search(value): prefix = value[: match.start()].rstrip() diff --git a/services/app/domain/release_profiles.py b/services/app/domain/release_profiles.py index 69ad93e9..bd01356b 100644 --- a/services/app/domain/release_profiles.py +++ b/services/app/domain/release_profiles.py @@ -16,7 +16,11 @@ from app.domain.indexer_labels import indexer_language, indexer_suffixes, label_language from app.domain.narrators import NarratorNames from app.domain.request_scope import ScopePreferences -from app.domain.series_identity import position_key, title_outside_series_note +from app.domain.series_identity import ( + position_key, + title_outside_series_note, + title_outside_series_prefix, +) from app.domain.title_matching import compatible_title from app.importing.naming import fingerprint @@ -447,15 +451,20 @@ def indexer_title_authors(release, work): parts = re.split(r"\s+[-–—]\s*|[-–—]\s+", title, maxsplit=1) if len(parts) == 2 and (credit := normalized(parts[0])) in authors: source_title = re.sub(r"\s+[-–—]\s*|[-–—]\s+", ": ", parts[1]) - if compatible_title(source_title, work["title"], allow_extra_subtitle=True): + if compatible_title( + source_title, work["title"], allow_extra_subtitle=True + ) or title_outside_series_prefix(parts[1], work): matched.add(authors[credit]) # AudiobookBay names, including Prowlarr's copy of them, are # "Title - Author, Narrator". The catalog author is the first credit. if not matched: - head, separator, tail = title.partition(" - ") - if separator: + # The first separator can belong to "Series 2 - Title"; the last + # can belong to a narrator credit. Check both with the exact author. + for head, separator, tail in (title.partition(" - "), title.rpartition(" - ")): + if not separator: + continue credit = normalized(tail.split(",")[0]) - bare = title_outside_series_note(head, work) + bare = title_outside_series_note(head, work) or title_outside_series_prefix(head, work) full_title = normalized(parse_title_labels(work["title"]).title) if credit in authors and ( normalized(head) in expected_titles diff --git a/services/app/domain/series_identity.py b/services/app/domain/series_identity.py index 87c0bc2d..5693013c 100644 --- a/services/app/domain/series_identity.py +++ b/services/app/domain/series_identity.py @@ -19,6 +19,24 @@ def series_name(value): return re.sub(r"\s+(?:saga|series)$", "", value).strip() +def note_position(value): + """A numeric position or canonical Roman numeral, only inside a known series note.""" + position = position_key(value) + if position is not None: + return position + if not value or not re.fullmatch( + r"m{0,3}(?:cm|cd|d?c{0,3})(?:xc|xl|l?x{0,3})(?:ix|iv|v?i{0,3})", value + ): + return None + numbers = {"i": 1, "v": 5, "x": 10, "l": 50, "c": 100, "d": 500, "m": 1000} + total, previous = 0, 0 + for letter in reversed(value): + number = numbers[letter] + total += -number if number < previous else number + previous = number + return Decimal(total) + + def series_note_agrees(note, series, *, require_position=False): # Preserve punctuation until numbers have been checked: 5.5 is not 5, # and 1-3 is a collection, not a single position. @@ -31,22 +49,15 @@ def series_note_agrees(note, series, *, require_position=False): if not match: continue remainder = note[: match.start()] + " " + note[match.end() :] - numbers = re.findall(r"\d+(?:\.\d+)?", remainder) - if require_position and len(numbers) != 1: - continue - if numbers and ( - len(numbers) != 1 - or position_key(numbers[0]) is None - or position_key(numbers[0]) != position_key(entry.get("position")) - ): - continue - remainder = re.sub(r"\d+(?:\.\d+)?", " ", remainder) remainder = re.sub( - r"\b(?:a|an|the|trilogy|duology|quartet|saga|series|cycle|book|volume|vol)\b\.?", + r"\b(?:a|an|the|trilogy|duology|quartet|saga|series|cycle|book|bk|volume|vol)\b\.?", " ", remainder, - ) - if not remainder.strip(" ,:;#"): + ).strip(" ,:;#") + if not remainder and not require_position: + return True + position = note_position(remainder) + if position is not None and position == position_key(entry.get("position")): return True return False @@ -73,3 +84,18 @@ def title_outside_series_note(value, work): if series_note_agrees(note, series): return title return None + + +def title_outside_series_prefix(value, work): + """Keep the complete title; remove only a catalog-backed series/position prefix.""" + title, value = display_title(work["title"]), display_title(value) + if not title: + return None + series = [entry for entry in work.get("series", []) if isinstance(entry, dict)] + for boundary in re.finditer(r"\s*[:–—-]\s*|\s+", value): + if exact_title_key(value[boundary.end() :]) != exact_title_key(title): + continue + note = value[: boundary.start()].strip() + if series_note_agrees(note, series, require_position=True): + return title + return None diff --git a/services/app/importing/inspection.py b/services/app/importing/inspection.py index cb374bf4..2943471f 100644 --- a/services/app/importing/inspection.py +++ b/services/app/importing/inspection.py @@ -143,8 +143,6 @@ def inspect_epub(fd): names = {entry.filename for entry in entries} if len(names) != len(entries): raise InspectionError("EPUB contains conflicting duplicate entries") - if "META-INF/encryption.xml" in names: - raise InspectionError("EPUB declares encrypted resources; review is required") def read(name): relative_parts(name) @@ -167,6 +165,7 @@ def read(name): if metadata is None or manifest is None or spine is None or len(spine) == 0: raise InspectionError("EPUB has no readable metadata and book spine") items = {item.attrib.get("id"): item for item in manifest} + spine_paths = set() for ref in spine: item = items.get(ref.attrib.get("idref")) if item is None: @@ -175,11 +174,21 @@ def read(name): if href.scheme or href.netloc: raise InspectionError("EPUB spine references external content") path = epub_resource_path(package_path, href.path) + spine_paths.add(path) if path not in names or archive.getinfo(path).file_size == 0: raise InspectionError("EPUB spine content is missing or empty") if archive.getinfo(path).flag_bits & 1: raise InspectionError("EPUB spine content is encrypted") + if "META-INF/encryption.xml" in names: + check_epub_encryption( + ElementTree.fromstring(read("META-INF/encryption.xml")), + package_path, + manifest, + spine_paths, + archive, + ) + def values(name): return [ node.text.strip()[:600] @@ -205,6 +214,60 @@ def values(name): } +def check_epub_encryption(document, package_path, manifest, spine_paths, archive): + """Allow only IDPF font obfuscation; no content is decrypted or rewritten. + + https://www.w3.org/TR/epub-33/#sec-obfuscation-specifying + """ + ocf = "{urn:oasis:names:tc:opendocument:xmlns:container}" + enc = "{http://www.w3.org/2001/04/xmlenc#}" + font_types = { + "font/ttf", + "font/otf", + "font/woff", + "font/woff2", + "application/font-sfnt", + "application/vnd.ms-opentype", + "application/font-woff", + } + reason = "EPUB declares unsupported encrypted resources; review is required" + if document.tag != f"{ocf}encryption" or not len(document): + raise InspectionError(reason) + fonts, non_fonts = set(), set(spine_paths) + names = set(archive.namelist()) + for item in manifest: + href = urlsplit(item.attrib.get("href", "")) + if href.scheme or href.netloc or href.query or href.fragment: + continue + path = epub_resource_path(package_path, href.path) + (fonts if item.attrib.get("media-type") in font_types else non_fonts).add(path) + for data in document: + if data.tag != f"{enc}EncryptedData" or sorted(node.tag for node in data) != sorted( + [f"{enc}EncryptionMethod", f"{enc}CipherData"] + ): + raise InspectionError(reason) + method = data.find(f"{enc}EncryptionMethod") + cipher = data.find(f"{enc}CipherData") + if ( + method.attrib.get("Algorithm") != "http://www.idpf.org/2008/embedding" + or len(method) + or len(cipher) != 1 + or cipher[0].tag != f"{enc}CipherReference" + or len(cipher[0]) + ): + raise InspectionError(reason) + href = urlsplit(cipher[0].attrib.get("URI", "")) + if href.scheme or href.netloc or href.query or href.fragment: + raise InspectionError(reason) + # CipherReference is relative to the container root, not encryption.xml or the OPF. + path = epub_resource_path("root", href.path) + if path not in fonts or path in non_fonts or path not in names: + raise InspectionError(reason) + info = archive.getinfo(path) + if info.file_size == 0 or info.flag_bits & 1: + raise InspectionError(reason) + + def epub_resource_path(package_path, href): """Resolve OPF-relative URLs inside the ZIP, never against the filesystem.""" relative = unquote(href) diff --git a/services/app/importing/matching.py b/services/app/importing/matching.py index 254b8fb4..4795fa38 100644 --- a/services/app/importing/matching.py +++ b/services/app/importing/matching.py @@ -29,7 +29,7 @@ from app.importing.naming import StrictModel, fingerprint from app.importing.versioning import version_revision -MATCHER_VERSION = 7 +MATCHER_VERSION = 8 MAX_CANDIDATES = 50 diff --git a/tests/integration/test_automatic_imports.py b/tests/integration/test_automatic_imports.py index 611f3604..40ee0fcc 100644 --- a/tests/integration/test_automatic_imports.py +++ b/tests/integration/test_automatic_imports.py @@ -343,8 +343,9 @@ async def test_linked_review_retries_automatic_matching_without_new_inspection( assert downloader.calls.count("submit") == 1 +@pytest.mark.parametrize("obfuscated_font", [False, True]) async def test_retry_reinspects_unreadable_book_without_redownloading( - client, database, automatic_job, downloader + client, database, automatic_job, downloader, obfuscated_font ): from app.importing.workflow import run_inspection @@ -358,7 +359,10 @@ async def test_retry_reinspects_unreadable_book_without_redownloading( from tests.media_fixtures import epub - epub(Path(inspection.source_path) / inspection.relative_path / "book.epub") + epub( + Path(inspection.source_path) / inspection.relative_path / "book.epub", + obfuscated_font=obfuscated_font, + ) await run_inspection(operation_id) async with database() as db, db.begin(): row = await db.get(AutomaticImport, automatic_job) @@ -369,7 +373,13 @@ async def test_retry_reinspects_unreadable_book_without_redownloading( **original, "groups": [], "files": [ - {**file, "state": "held", "reason": "Invalid media metadata"} + { + **file, + "state": "held", + "reason": "EPUB declares encrypted resources; review is required" + if obfuscated_font + else "Invalid media metadata", + } for file in original["files"] ], } diff --git a/tests/integration/test_quick_add.py b/tests/integration/test_quick_add.py index e943a33f..edf0addc 100644 --- a/tests/integration/test_quick_add.py +++ b/tests/integration/test_quick_add.py @@ -426,6 +426,38 @@ async def test_cancelled_request_during_search_does_not_download( assert not await db.scalar(select(DownloadAttempt.id)) +async def test_withdraw_then_request_again_starts_one_new_download( + client, database, authorized, catalog +): + await defaults(client, authorized) + first = (await add(client, catalog["work"])).json() + request_id = first["request_id"] + request = (await client.get(f"/api/requests/{request_id}")).json() + for reason in request["reasons"]: + withdrawn = await client.delete(f"/api/requests/{request_id}/reasons/{reason['id']}") + assert withdrawn.status_code == 200, withdrawn.text + await complete_search(database, UUID(first["id"]), authorized) + await quick_add.run(UUID(first["id"])) + response = await add(client, catalog["work"]) + assert response.status_code == 202, response.text + second = response.json() + assert second["id"] != first["id"] + assert second["status"] == "queued", second + identifier = UUID(second["id"]) + await complete_search(database, identifier, authorized) + await quick_add.run(identifier) + async with database() as db: + operation = await db.get(Operation, identifier) + child_id = UUID(operation.payload["slots"]["audio"]["operation_id"]) + await automatic_selection.run(child_id) + await quick_add.run(identifier) + await quick_add.run(identifier) + async with database() as db: + operation = await db.get(Operation, identifier) + assert operation.status == "completed", operation.message + assert await db.scalar(select(func.count()).select_from(DownloadAttempt)) == 1 + + @pytest.mark.parametrize("usenet_primary", [False, True]) @pytest.mark.parametrize("blocked", [False, True]) @pytest.mark.parametrize("automatic_folders", [False, True]) diff --git a/tests/integration/test_sabnzbd.py b/tests/integration/test_sabnzbd.py index c02b0978..ab91df40 100644 --- a/tests/integration/test_sabnzbd.py +++ b/tests/integration/test_sabnzbd.py @@ -4,12 +4,20 @@ import httpx import pytest +from sqlalchemy import select from app.adapters.contracts import SubmissionReceipt from app.adapters.prowlarr import ProwlarrRelease from app.adapters.sabnzbd import SabClient from app.config import get_settings -from app.db.models import DownloadAttempt, DownloadInspection, Integration, Operation, SourceResult +from app.db.models import ( + DownloadAttempt, + DownloadInspection, + Integration, + Operation, + SourceResult, + WorkMetadataSource, +) from app.domain import automatic_selection from app.domain import download_attempts as downloads from app.domain import downloaders as downloader_settings @@ -167,16 +175,26 @@ async def submit(self, content, *, attempt_tag, save_path, category): return SubmissionReceipt(external_ids=["SABnzbd_nzo_test"]) -async def test_clicked_usenet_release_with_parenthesized_format_reaches_downloader( - client, admin, database, source, authorized, monkeypatch, prowlarr_http +@pytest.mark.parametrize( + "title", + [ + "Writer - Harbor (retail) (m4b)", + "Harbor: Sea Saga II - Writer (m4b)", + "Sea Saga II: Harbor - Writer (m4b)", + "Writer - Sea Saga, Bk 2 Harbor m4b", + ], +) +async def test_clicked_usenet_release_with_catalog_labels_reaches_downloader( + client, admin, database, source, authorized, monkeypatch, prowlarr_http, catalog, title ): await configure(client) + async with database() as db, db.begin(): + metadata = await db.scalar( + select(WorkMetadataSource).where(WorkMetadataSource.work_id == catalog["work"]) + ) + metadata.snapshot = {**metadata.snapshot, "series": [{"name": "Sea Saga", "position": "2"}]} prowlarr_http.update( - releases=[ - release( - title="Writer - Harbor (retail) (m4b)", protocol="usenet", categories=[{"id": 3030}] - ) - ], + releases=[release(title=title, protocol="usenet", categories=[{"id": 3030}])], bytes=nzb_bytes(name="Harbor", filename="Harbor.m4b"), ) found = (await search(client)).json()["items"][0] diff --git a/tests/integration/test_series_file_import.py b/tests/integration/test_series_file_import.py index 28a3217f..d3007941 100644 --- a/tests/integration/test_series_file_import.py +++ b/tests/integration/test_series_file_import.py @@ -24,6 +24,7 @@ [ ("First Harbor (Harbor #1) (Unabridged)", False), ("Harbor, Book 1", False), + ("Harbor, Book I", False), ("First Harbor (Harbor #1)", True), ], ) diff --git a/tests/integration/test_series_file_review.py b/tests/integration/test_series_file_review.py index 82e84568..cefa682d 100644 --- a/tests/integration/test_series_file_review.py +++ b/tests/integration/test_series_file_review.py @@ -213,6 +213,7 @@ async def review(client, download): "series_download", [ {}, + {"title": "Harbor, Book I"}, {"identified": True}, {"title": "First and Last Harbor (Harbor #1)", "connector": True, "identified": True}, ], diff --git a/tests/integration/test_single_file_acquisition.py b/tests/integration/test_single_file_acquisition.py index 1683350e..f134e3cc 100644 --- a/tests/integration/test_single_file_acquisition.py +++ b/tests/integration/test_single_file_acquisition.py @@ -96,6 +96,7 @@ "automatic-identified-missing-tags-audio", "automatic-subtitle-audio", "automatic-linked-ebook", + "automatic-obfuscated-font", "automatic-multiple-isbns", "automatic-multiple-isbns-unlisted", "automatic-identified-subtitle", @@ -139,6 +140,7 @@ async def test_single_epub_download_to_confirmed_library_keeps_neighbor_private( "automatic-identified-missing-tags-audio", "automatic-subtitle-audio", "automatic-linked-ebook", + "automatic-obfuscated-font", "automatic-multiple-isbns", "automatic-multiple-isbns-unlisted", "automatic-identified-subtitle", @@ -223,6 +225,7 @@ async def test_single_epub_download_to_confirmed_library_keeps_neighbor_private( language="en-US" if handoff in {"automatic-linked-ebook", "automatic-language-alias"} else "en", + obfuscated_font=handoff == "automatic-obfuscated-font", ) neighbor = source.parent.parent if numbered_audio else source.parent epub(neighbor / "unrelated.epub", title="Not part of this torrent") diff --git a/tests/media_fixtures.py b/tests/media_fixtures.py index 2140189a..a4c0ee07 100644 --- a/tests/media_fixtures.py +++ b/tests/media_fixtures.py @@ -26,6 +26,7 @@ def epub( metadata=None, isbn=None, language="en", + obfuscated_font=False, ): path.parent.mkdir(parents=True, exist_ok=True) with zipfile.ZipFile(path, "w") as book: @@ -44,10 +45,27 @@ def epub( f"{escape(language)}" f"{escape(isbn or 'synthetic-fixture')}" '' + 'media-type="application/xhtml+xml"/>' + + ( + '' + if obfuscated_font + else "" + ) + + "" '' ), ) + if obfuscated_font: + book.writestr("OEBPS/font.woff", b"synthetic obfuscated font bytes") + book.writestr( + "META-INF/encryption.xml", + '' + "' + '' + "", + ) if chapter: book.writestr( "OEBPS/chapter.xhtml", diff --git a/tests/unit/test_automatic_eligibility.py b/tests/unit/test_automatic_eligibility.py index 3b1a139d..d133b866 100644 --- a/tests/unit/test_automatic_eligibility.py +++ b/tests/unit/test_automatic_eligibility.py @@ -84,6 +84,7 @@ def test_unknown_or_conflicting_source_facts_do_not_become_eligible(change, reas ), ("Cory Doctorow - Enshittification (2025) [EPUB]", [], True), ("Cory Doctorow - Enshittification.RETAIL.EPUB", [], True), + ("Cory.Doctorow.Enshittification.2025.RETAiL.ePub.eBook-LiBRiCiDE", [], True), ("Cory Doctorow - Enshittification (retail) (epub)", [], True), ("Cory Doctorow - Enshittification (summary) (epub)", [], False), ("Cory Doctorow - Enshittification (sample) (epub)", [], False), @@ -122,6 +123,45 @@ def test_indexer_ebook_labels_preserve_complete_author_title_evidence(title, aut assert (not reasons) is allowed, reasons +@pytest.mark.parametrize( + "paths,allowed", + [ + (["Harbor.epub"], True), + (["Harbor.mp3"], False), + (["Harbor.epub", "Harbor.zip"], False), + (["Harbor.epub", "Other.epub"], False), + ], +) +def test_dotted_ebook_label_does_not_replace_inspected_content(paths, allowed): + from app.adapters.prowlarr import ProwlarrRelease + + title = "Writer.Harbor.2025.RETAiL.ePub.eBook-LiBRiCiDE" + value = ProwlarrRelease( + source_id="fixture", + title=title, + raw_title=title, + authors=[], + medium="ebook", + language="en", + protocol="torrent", + seeders=5, + formats=["epub"], + indexer_name="Fixture", + categories=[7020], + observed_at=datetime.now(UTC), + acquisition_supported=True, + ) + reasons = eligibility( + value, + WORK, + {**RULE, "medium": "ebook"}, + ReleasePreferences(), + descriptor=descriptor(paths), + unattended=True, + ) + assert (not reasons) is allowed, reasons + + @pytest.mark.parametrize( ("title", "allowed"), [ diff --git a/tests/unit/test_epub_encryption.py b/tests/unit/test_epub_encryption.py new file mode 100644 index 00000000..ca7982a8 --- /dev/null +++ b/tests/unit/test_epub_encryption.py @@ -0,0 +1,109 @@ +"""Font obfuscation is readable EPUB packaging, not permission to import DRM content.""" + +import zipfile + +import pytest + +from app.importing.inspection import inspect_download +from tests.media_fixtures import epub + + +def test_standard_font_obfuscation_keeps_readable_metadata_and_original_bytes(tmp_path): + path = tmp_path / "book.epub" + epub(path, obfuscated_font=True) + original = path.read_bytes() + snapshot = inspect_download(tmp_path, path.name) + [file] = snapshot["files"] + assert file["state"] == "inspected", file + assert file["metadata"]["title"] == "First Harbor" + assert file["metadata"]["authors"] == ["Alex Morgan"] + assert len(snapshot["groups"]) == 1 + assert path.read_bytes() == original + + +@pytest.mark.parametrize( + "change", + [ + "unknown-algorithm", + "encrypted-chapter", + "undeclared-font", + "missing-font", + "external-reference", + "escaped-reference", + "fragment-reference", + "cipher-value", + "transforms", + "empty-document", + "duplicate-method", + "mixed-encryption", + "font-in-spine", + "non-font-alias", + "entity", + ], +) +def test_other_encryption_stays_held(tmp_path, change): + path = tmp_path / "book.epub" + epub(path, obfuscated_font=True) + with zipfile.ZipFile(path) as archive: + entries = {name: archive.read(name) for name in archive.namelist()} + encryption = entries["META-INF/encryption.xml"].decode() + package = entries["OEBPS/book.opf"].decode() + if change == "unknown-algorithm": + encryption = encryption.replace("http://www.idpf.org/2008/embedding", "urn:unknown") + elif change == "encrypted-chapter": + encryption = encryption.replace("OEBPS/font.woff", "OEBPS/chapter.xhtml") + elif change == "undeclared-font": + package = package.replace('media-type="font/woff"', 'media-type="application/octet-stream"') + elif change == "missing-font": + del entries["OEBPS/font.woff"] + elif change in {"external-reference", "escaped-reference", "fragment-reference"}: + uri = { + "external-reference": "https://example.test/font.woff", + "escaped-reference": "%2e%2e/OEBPS/font.woff", + "fragment-reference": "OEBPS/font.woff#data", + }[change] + encryption = encryption.replace("OEBPS/font.woff", uri) + elif change == "cipher-value": + encryption = encryption.replace( + '', + "AAAA", + ) + elif change == "transforms": + encryption = encryption.replace( + '', + '', + ) + elif change == "empty-document": + encryption = '' + elif change == "duplicate-method": + encryption = encryption.replace( + "", '' + ) + elif change == "mixed-encryption": + extra = encryption.split("")[1].split("")[0] + encryption = encryption.replace( + "", + "" + + extra.replace("OEBPS/font.woff", "OEBPS/chapter.xhtml") + + "", + ) + elif change == "font-in-spine": + package = package.replace("", '') + elif change == "non-font-alias": + package = package.replace( + "", + '', + ) + elif change == "entity": + encryption = ']>' + ( + encryption.replace("", "&bad;") + ) + entries["META-INF/encryption.xml"] = encryption.encode() + entries["OEBPS/book.opf"] = package.encode() + with zipfile.ZipFile(path, "w") as archive: + for name, data in entries.items(): + archive.writestr(name, data) + snapshot = inspect_download(tmp_path, path.name) + assert snapshot["files"][0]["state"] == "held" + assert not snapshot["groups"] + assert "root:" not in str(snapshot) diff --git a/tests/unit/test_release_name_identity.py b/tests/unit/test_release_name_identity.py new file mode 100644 index 00000000..70469446 --- /dev/null +++ b/tests/unit/test_release_name_identity.py @@ -0,0 +1,127 @@ +"""Reported indexer layouts must preserve the full catalog-backed identity.""" + +from types import SimpleNamespace + +import pytest + +from app.domain.release_profiles import indexer_title_authors +from app.domain.series_identity import series_note_agrees + + +def authors(title, book="The Drawing of the Three", position="2", **changes): + release = SimpleNamespace(source="prowlarr", title=title, raw_title=title, authors=[]) + release.__dict__.update(changes) + work = { + "title": book, + "authors": ["Stephen King"], + "series": [{"name": "The Dark Tower", "position": position}], + } + return indexer_title_authors(release, work) + + +def test_reported_release_names_corroborate_exact_books(): + cases = [ + ("Wolves of the Calla: Dark Tower V - Stephen King", "Wolves of the Calla", "5"), + ("Song of Susannah: The Dark Tower VI - Stephen King", "Song of Susannah", "6"), + ("Dark Tower II: The Drawing of the Three - Stephen King", "The Drawing of the Three", "2"), + ("Dark Tower III: The Waste Lands - Stephen King", "The Waste Lands", "3"), + ( + "Stephen King - The Dark Tower, Bk 2 The Drawing Of The Three mp3", + "The Drawing of the Three", + "2", + ), + ("Stephen King - The Dark Tower, Bk 3 The Waste Lands mp3", "The Waste Lands", "3"), + ( + "Stephen.King.The.Wind.Through.The.Keyhole.2012.RETAiL.ePub.eBook-LiBRiCiDE", + "The Wind through the Keyhole", + None, + ), + ] + actual = [(title, authors(title, book, position)) for title, book, position in cases] + assert actual == [(title, ["Stephen King"]) for title, _, _ in cases] + + +@pytest.mark.parametrize( + "note,position,accepted", + [ + ("Dark Tower V", "5", True), + ("The Dark Tower, Book vi", "6", True), + ("Dark Tower, Bk. IV", "4", True), + ("Dark Tower #2.5", "2.5", True), + ("Dark Tower V", "6", False), + ("Dark Tower V", "5.5", False), + ("Dark Tower V-VI", "5", False), + ("Dark Tower V & VI", "5", False), + ("Dark Tower V/VI", "5", False), + ("Dark Tower V 5", "5", False), + ("Dark Tower IIV", "3", False), + ("Dark Tower IIII", "4", False), + ("Dark Tower VX", "5", False), + ("Dark Tower V.5", "5.5", False), + ("Dark Tower V excerpt", "5", False), + ("Dark Tower", "5", False), + ("Other Tower V", "5", False), + ], +) +def test_series_position_notes_are_exact_and_unambiguous(note, position, accepted): + assert ( + series_note_agrees( + note, [{"name": "The Dark Tower", "position": position}], require_position=True + ) + is accepted + ) + + +@pytest.mark.parametrize("source", ["audiobookbay", "prowlarr"]) +@pytest.mark.parametrize( + "title,accepted", + [ + ("Dark Tower II: The Drawing of the Three - Stephen King", True), + ("Dark Tower 2 - The Drawing of the Three - Stephen King", True), + ("Stephen King - The Dark Tower, Bk 2 The Drawing Of The Three mp3", True), + ("Dark Tower III: The Drawing of the Three - Stephen King", False), + ("Dark Tower II-III: The Drawing of the Three - Stephen King", False), + ("Dark Tower II & III: The Drawing of the Three - Stephen King", False), + ("Dark Tower 2.5: The Drawing of the Three - Stephen King", False), + ("Other Tower II: The Drawing of the Three - Stephen King", False), + ("Dark Tower II: The Drawing of the Three - Other Writer", False), + ("Dark Tower II: The Drawing of the Three: A Study Guide - Stephen King", False), + ("Dark Tower II: The Drawing of the Three and The Waste Lands - Stephen King", False), + ("Stephen King - The Dark Tower, Bk 3 The Drawing Of The Three mp3", False), + ("Stephen King - The Dark Tower, Bk 2 Collection The Drawing Of The Three mp3", False), + ("Stephen King - The Dark Tower, Bk 2 The Drawing mp3", False), + ], +) +def test_series_prefixed_releases_preserve_catalog_constraints(source, title, accepted): + assert authors(title, source=source) == (["Stephen King"] if accepted else []) + + +@pytest.mark.parametrize( + "suffix,accepted", + [ + (".2012.RETAiL.ePub.eBook-LiBRiCiDE", True), + (".2012.retail.EPUB-LiBRiCiDE", True), + (".ePub.eBook-LIBRICIDE", True), + (".mobi.eBook-Group", True), + (".ePub.eBook-sample", False), + (".ePub.eBook-PART01", False), + (".ePub.eBook-preview", False), + (".ePub.eBook-Collection", False), + (".ePub.eBook-Summary", False), + (".ePub.eBook-Group.part01.rar", False), + (".ePub.eBook-Unknown-words", False), + (".and.The.Gunslinger.ePub.eBook-Group", False), + ], +) +def test_ebook_packaging_does_not_erase_content_warnings(suffix, accepted): + title = "Stephen.King.The.Wind.Through.The.Keyhole" + suffix + assert authors(title, "The Wind through the Keyhole", None) == ( + ["Stephen King"] if accepted else [] + ) + + +def test_supported_layout_still_requires_structured_author_agreement_and_catalog_series(): + title = "Dark Tower II: The Drawing of the Three - Stephen King" + assert authors(title, authors=["Other Writer"]) == [] + assert authors(title, position=None) == [] + assert authors(title) == ["Stephen King"] From ea9c974e375ed79a9a3ef0c201c2f2e0ce5e92a7 Mon Sep 17 00:00:00 2001 From: Logan Abell Date: Thu, 15 Oct 2026 01:32:39 -0400 Subject: [PATCH 2/4] Preserve numbered content warnings and refresh matcher API schema --- apps/web/src/api/schema.d.ts | 2 +- docs/openapi.json | 2 +- services/app/domain/automatic_eligibility.py | 2 +- services/app/domain/indexer_labels.py | 6 +-- services/app/importing/automatic.py | 2 +- tests/integration/test_automatic_selection.py | 10 ++++- tests/unit/test_automatic_content.py | 16 ++++++++ tests/unit/test_automatic_eligibility.py | 41 +++++++++++++++++++ tests/unit/test_release_name_identity.py | 26 ++++++++++++ 9 files changed, 99 insertions(+), 8 deletions(-) diff --git a/apps/web/src/api/schema.d.ts b/apps/web/src/api/schema.d.ts index 7e3938a2..265cdec3 100644 --- a/apps/web/src/api/schema.d.ts +++ b/apps/web/src/api/schema.d.ts @@ -10336,7 +10336,7 @@ export interface components { limit: number; /** * Matcher Version - * @default 7 + * @default 8 */ matcher_version: number; }; diff --git a/docs/openapi.json b/docs/openapi.json index b21726c6..83a71fd8 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -30848,7 +30848,7 @@ "matcher_version": { "type": "integer", "title": "Matcher Version", - "default": 7 + "default": 8 } }, "additionalProperties": false, diff --git a/services/app/domain/automatic_eligibility.py b/services/app/domain/automatic_eligibility.py index 3ea84640..46e7bc99 100644 --- a/services/app/domain/automatic_eligibility.py +++ b/services/app/domain/automatic_eligibility.py @@ -14,7 +14,7 @@ EBOOKS = {"epub", "pdf", "cbz"} AUDIO = {"m4b", "mp3", "flac", "aac", "ogg", "opus"} SIDECARS = {"jpg", "jpeg", "png", "webp", "opf", "nfo", "txt", "cue", "m3u"} -PARTIAL = re.compile(r"\b(sample|excerpt|preview|incomplete|truncated)\b", re.I) +PARTIAL = re.compile(r"\b(?:sample|excerpt|preview|incomplete|truncated)\d*\b", re.I) DISC_FOLDER = re.compile(r"(?:^|[\s_(\[])(?:cd|disc|disk|part)\s*\d+\b", re.I) PACK = re.compile( r"\b(collection|pack|omnibus|box[ -]?set|anthology|complete series|" diff --git a/services/app/domain/indexer_labels.py b/services/app/domain/indexer_labels.py index 59ef872e..71b9c188 100644 --- a/services/app/domain/indexer_labels.py +++ b/services/app/domain/indexer_labels.py @@ -33,10 +33,10 @@ def indexer_suffixes(value): yield value, labels if group := _GROUP.search(value): # A scene group after a media extension is metadata. Partial/preview - # markers are content warnings even if they use the same spelling. + # markers, including numbered variants, are content warnings. if not re.fullmatch( - r"sample|preview|summary|excerpt|incomplete|truncated|collection|pack|omnibus|" - r"anthology|boxset|part\d*|cd\d*|disc\d*", + r"(?:sample|preview|summary|excerpt|incomplete|truncated|collection|pack|omnibus|" + r"anthology|boxset|part|cd|disc)\d*", group["group"], re.I, ): diff --git a/services/app/importing/automatic.py b/services/app/importing/automatic.py index be462207..5ade18a6 100644 --- a/services/app/importing/automatic.py +++ b/services/app/importing/automatic.py @@ -41,7 +41,7 @@ from app.jobs.queue import enqueue # This is a negative signal, never a positive proof of a book's identity. -PARTIAL = re.compile(r"\b(sample|excerpt|preview|incomplete|truncated)\b", re.I) +PARTIAL = re.compile(r"\b(?:sample|excerpt|preview|incomplete|truncated)\d*\b", re.I) async def check_policy(db, row, *, lock=False): diff --git a/tests/integration/test_automatic_selection.py b/tests/integration/test_automatic_selection.py index 7fefcfee..8baabd06 100644 --- a/tests/integration/test_automatic_selection.py +++ b/tests/integration/test_automatic_selection.py @@ -353,7 +353,9 @@ async def test_cancelled_preparation_does_not_keep_release_selected( assert (await detail(client, retry["id"]))["status"] == "completed" -@pytest.mark.parametrize("change", ["author", "unknown-seeds", "language", "pack", "recording"]) +@pytest.mark.parametrize( + "change", ["author", "unknown-seeds", "language", "pack", "recording", "numbered-partial"] +) async def test_ineligible_candidates_remain_reviewable_without_fetching_torrents( client, database, source, catalog, change ): @@ -368,6 +370,7 @@ async def test_ineligible_candidates_remain_reviewable_without_fetching_torrents "unknown-seeds": {"seeders": None}, "language": {"language": "fr"}, "pack": {"raw_title": "Harbor Books 1-3"}, + "numbered-partial": {"raw_title": "Writer.Harbor.M4B-Sample01"}, } if change == "recording": intent = await db.get(AcquisitionIntent, UUID(source["body"]["intent_id"])) @@ -387,6 +390,11 @@ async def test_ineligible_candidates_remain_reviewable_without_fetching_torrents value = await detail(client, operation["id"]) assert value["status"] == "held" and value["decisions"][0]["reasons"] assert source["resolver"].calls == [] + if change == "numbered-partial": + assert any("partial content" in reason for reason in value["decisions"][0]["reasons"]) + async with database() as db: + assert await db.scalar(select(func.count()).select_from(AcquisitionSelection)) == 0 + assert await db.scalar(select(func.count()).select_from(DownloadAttempt)) == 0 async def test_manifest_pack_evidence_is_retained_when_no_candidate_qualifies( diff --git a/tests/unit/test_automatic_content.py b/tests/unit/test_automatic_content.py index 9cf6e498..828e6291 100644 --- a/tests/unit/test_automatic_content.py +++ b/tests/unit/test_automatic_content.py @@ -50,6 +50,22 @@ def test_partial_labels_and_noncertified_ebook_containers_need_review(extension, ) +@pytest.mark.parametrize( + "warning", ["Sample01", "Excerpt1", "Preview02", "Incomplete3", "Truncated4"] +) +@pytest.mark.parametrize("evidence", ["release", "file"]) +def test_numbered_partial_markers_hold_automatic_import(warning, evidence): + name = f"book-{warning}.epub" if evidence == "file" else "book.epub" + title = f"Book-{warning}" if evidence == "release" else "Book" + group = SimpleNamespace(medium="ebook", files=[SimpleNamespace(path=name)]) + reason = content_reason( + group, + {name: {"path": name, "extension": "epub", "state": "inspected"}}, + {"title": title}, + ) + assert reason and "partial content" in reason + + def test_one_part_of_a_book_released_in_parts_needs_review(): group = SimpleNamespace(medium="ebook", files=[SimpleNamespace(path="book.epub")]) files = {"book.epub": {"path": "book.epub", "extension": "epub", "state": "inspected"}} diff --git a/tests/unit/test_automatic_eligibility.py b/tests/unit/test_automatic_eligibility.py index d133b866..d5566000 100644 --- a/tests/unit/test_automatic_eligibility.py +++ b/tests/unit/test_automatic_eligibility.py @@ -162,6 +162,47 @@ def test_dotted_ebook_label_does_not_replace_inspected_content(paths, allowed): assert (not reasons) is allowed, reasons +@pytest.mark.parametrize( + "warning", ["Sample01", "Excerpt1", "Preview02", "Incomplete3", "Truncated4"] +) +@pytest.mark.parametrize("evidence", ["indexer", "raw-title", "file"]) +def test_numbered_partial_markers_prevent_automatic_ebook_selection(warning, evidence): + from app.adapters.prowlarr import ProwlarrRelease + + value = release().model_copy(update={"medium": "ebook", "formats": ["epub"]}) + paths = ["Harbor.epub"] + if evidence == "indexer": + title = f"Writer.Harbor.epub.eBook-{warning}" + value = ProwlarrRelease( + source_id="fixture", + title=title, + raw_title=title, + authors=[], + medium="ebook", + language="en", + protocol="torrent", + seeders=5, + formats=["epub"], + indexer_name="Fixture", + categories=[7020], + observed_at=datetime.now(UTC), + acquisition_supported=True, + ) + elif evidence == "raw-title": + value = value.model_copy(update={"raw_title": f"Harbor-{warning}"}) + else: + paths = [f"Harbor-{warning}.epub"] + reasons = eligibility( + value, + WORK, + {**RULE, "medium": "ebook"}, + ReleasePreferences(), + descriptor=descriptor(paths), + unattended=True, + ) + assert any("partial content" in reason for reason in reasons), reasons + + @pytest.mark.parametrize( ("title", "allowed"), [ diff --git a/tests/unit/test_release_name_identity.py b/tests/unit/test_release_name_identity.py index 70469446..807b4e7b 100644 --- a/tests/unit/test_release_name_identity.py +++ b/tests/unit/test_release_name_identity.py @@ -103,6 +103,8 @@ def test_series_prefixed_releases_preserve_catalog_constraints(source, title, ac (".2012.retail.EPUB-LiBRiCiDE", True), (".ePub.eBook-LIBRICIDE", True), (".mobi.eBook-Group", True), + (".ePub.eBook-Group42", True), + (".ePub.eBook-Sampler01", True), (".ePub.eBook-sample", False), (".ePub.eBook-PART01", False), (".ePub.eBook-preview", False), @@ -120,6 +122,30 @@ def test_ebook_packaging_does_not_erase_content_warnings(suffix, accepted): ) +@pytest.mark.parametrize( + "warning", + [ + "Sample01", + "Excerpt1", + "Preview02", + "Incomplete3", + "Truncated4", + "Summary5", + "Collection6", + "Pack7", + "Omnibus8", + "Anthology9", + "Boxset10", + "Part11", + "CD12", + "Disc13", + ], +) +def test_numbered_content_warnings_are_not_release_groups(warning): + title = "Stephen.King.The.Wind.Through.The.Keyhole.ePub.eBook-" + warning + assert authors(title, "The Wind through the Keyhole", None) == [] + + def test_supported_layout_still_requires_structured_author_agreement_and_catalog_series(): title = "Dark Tower II: The Drawing of the Three - Stephen King" assert authors(title, authors=["Other Writer"]) == [] From e25e9ac1cdc0a35e9794a43f62880f2f60076089 Mon Sep 17 00:00:00 2001 From: Logan Abell Date: Thu, 15 Oct 2026 09:32:08 -0400 Subject: [PATCH 3/4] test(web): scope library review success during next search --- apps/web/tests/ui/library-review.spec.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/apps/web/tests/ui/library-review.spec.ts b/apps/web/tests/ui/library-review.spec.ts index f77cd967..371f6412 100644 --- a/apps/web/tests/ui/library-review.spec.ts +++ b/apps/web/tests/ui/library-review.spec.ts @@ -28,6 +28,10 @@ test("library review links unmatched items to Dewarr and Hardcover books", async page.on("pageerror", (e) => errors.push(e.message)); const matchSearches: string[] = []; const matches: { asset: string; body: unknown }[] = []; + let releaseNextSearch!: () => void; + const nextSearch = new Promise((resolve) => { + releaseNextSearch = resolve; + }); let imported = false; const items = [ { kind: "asset", asset: asset("a1", "Untagged Folder", ["title"]) }, @@ -115,6 +119,7 @@ test("library review links unmatched items to Dewarr and Hardcover books", async data = { items: shown, total: shown.length }; } else if (path === "/api/catalog/works") { matchSearches.push(url.searchParams.get("q") || ""); + if (url.searchParams.get("q") === "The Long Way") await nextSearch; data = { items: [work], total: 1 }; } else if (path.endsWith("/match") && method === "POST") { const id = path.split("/").at(-2)!; @@ -214,9 +219,16 @@ test("library review links unmatched items to Dewarr and Hardcover books", async part_total: null, }, }); - await expect(page.getByRole("status")).toContainText( - "Linked “Untagged Folder”", - ); + try { + // Saving and continuing can announce success while the next lookup is pending. + await expect(detail.getByRole("status")).toHaveText("Searching catalog…"); + await expect( + page.getByRole("status").filter({ hasText: "Linked “Untagged Folder”" }), + ).toBeVisible(); + } finally { + releaseNextSearch(); + } + await expect(detail.getByRole("status")).toHaveCount(0); await expect(cards).toHaveCount(2); await expect( nav.getByRole("link", { name: /^Library review/ }), From 4ae817caeef82b92fcdcb7483f4db7ba7a726743 Mon Sep 17 00:00:00 2001 From: Logan Abell Date: Thu, 15 Oct 2026 10:03:27 -0400 Subject: [PATCH 4/4] fix(web): preserve download preference drafts during refresh --- apps/web/src/pages/DownloadPreferences.tsx | 40 +++++++++++--- apps/web/tests/download-defaults.spec.ts | 62 ++++++++++++++++++++++ 2 files changed, 96 insertions(+), 6 deletions(-) diff --git a/apps/web/src/pages/DownloadPreferences.tsx b/apps/web/src/pages/DownloadPreferences.tsx index 070d2762..f08fced1 100644 --- a/apps/web/src/pages/DownloadPreferences.tsx +++ b/apps/web/src/pages/DownloadPreferences.tsx @@ -61,7 +61,7 @@ export default function DownloadPreferences({ {current.isPending && } {current.data && ( void; }) { const cache = useQueryClient(); - const [overrides, setOverrides] = useState(current.overrides); + const [draft, setDraft] = useState<{ + base: Defaults; + overrides: Defaults["overrides"]; + } | null>(null); + // A refresh must not replace the draft or authorize saving against a newer revision. + const current = draft?.base ?? latest; + const overrides = draft?.overrides ?? current.overrides; + const setOverrides = (value: Defaults["overrides"]) => + setDraft((previous) => ({ + base: previous?.base ?? latest, + overrides: value, + })); const [merge, setMerge] = useState(null); const showAudiobookImport = admin && !librariesOnly && scope === "installation"; @@ -126,11 +137,28 @@ function Editor({ }), ) : null; - return { preferences, settings }; + return { + preferences, + settings, + submittedDraft: draft, + submittedMerge: merge, + }; }, - onSuccess: async ({ preferences, settings }) => { + onSuccess: async ({ + preferences, + settings, + submittedDraft, + submittedMerge, + }) => { onSaved(); - setMerge(null); + setDraft((next) => + next === submittedDraft + ? null + : next + ? { ...next, base: preferences } + : null, + ); + setMerge((next) => (next === submittedMerge ? null : next)); cache.setQueryData(["download-defaults", scope], preferences); if (settings) cache.setQueryData(["organization-settings"], settings); await Promise.all([ diff --git a/apps/web/tests/download-defaults.spec.ts b/apps/web/tests/download-defaults.spec.ts index eed4dd46..5fd249c9 100644 --- a/apps/web/tests/download-defaults.spec.ts +++ b/apps/web/tests/download-defaults.spec.ts @@ -68,12 +68,42 @@ test("download defaults inherit per field and persist after reload", async ({ await expect(panel.getByRole("status")).toContainText( "Download defaults saved", ); + let releasePersonal!: () => void; + const personalRefresh = new Promise((resolve) => { + releasePersonal = resolve; + }); + await page.route("**/api/acquisition/preferences/personal", async (route) => { + const response = await route.fetch(); + await personalRefresh; + await route.fulfill({ response }); + }); await panel.getByLabel("Apply to").selectOption("personal"); await panel.getByText("Formats and transfer limits", { exact: true }).click(); const ebooks = panel.getByRole("group", { name: "Ebook format preference", exact: true, }); + // A refresh of cached personal defaults must not collapse or discard a draft. + await expect(ebooks).toBeVisible(); + await panel.getByLabel("Require standalone copies", { exact: true }).check(); + const refreshedPersonal = page.waitForResponse( + "**/api/acquisition/preferences/personal", + ); + releasePersonal(); + await refreshedPersonal; + await page.unroute("**/api/acquisition/preferences/personal"); + await expect( + panel.getByLabel("Require standalone copies", { exact: true }), + ).toBeChecked(); + await expect(ebooks.getByRole("listitem").first()).toContainText("EPUB"); + await panel + .getByRole("button", { name: "Save download defaults", exact: true }) + .click(); + await expect(panel.getByRole("alert")).toContainText( + "Download defaults changed; reload before saving", + ); + await page.reload(); + await panel.getByText("Formats and transfer limits", { exact: true }).click(); await expect(ebooks.getByRole("listitem").first()).toContainText("PDF"); await panel .getByRole("button", { @@ -81,13 +111,42 @@ test("download defaults inherit per field and persist after reload", async ({ exact: true, }) .press("ArrowUp"); + let releaseSave!: () => void; + const pendingSave = new Promise((resolve) => { + releaseSave = resolve; + }); + await page.route("**/api/acquisition/preferences/personal", async (route) => { + if (route.request().method() !== "PUT") return route.continue(); + const response = await route.fetch(); + await pendingSave; + await route.fulfill({ response }); + }); await panel .getByRole("button", { name: "Save download defaults", exact: true }) .click(); + // Edits made during a save belong to the next save, not the submitted snapshot. + await panel.getByLabel("Require standalone copies", { exact: true }).check(); + releaseSave(); await expect(panel.getByRole("status")).toContainText( "Download defaults saved", ); + await page.unroute("**/api/acquisition/preferences/personal"); + await expect( + panel.getByLabel("Require standalone copies", { exact: true }), + ).toBeChecked(); + await panel + .getByRole("button", { name: "Save download defaults", exact: true }) + .click(); + await expect(panel.getByRole("status")).toContainText( + "Download defaults saved", + ); + await expect( + panel.getByRole("button", { name: "Save download defaults", exact: true }), + ).toBeDisabled(); await page.reload(); + await expect( + panel.getByLabel("Require standalone copies", { exact: true }), + ).toBeChecked(); await panel.getByText("Formats and transfer limits", { exact: true }).click(); await expect(ebooks.getByRole("listitem").first()).toContainText("EPUB"); await panel @@ -97,6 +156,9 @@ test("download defaults inherit per field and persist after reload", async ({ }) .click(); await expect(ebooks.getByRole("listitem").first()).toContainText("PDF"); + await panel + .getByRole("button", { name: "Use inherited defaults", exact: true }) + .click(); await panel .getByRole("button", { name: "Save download defaults", exact: true }) .click();