From b8f9609f4f372c00eeb5b85c8abafa6c93b7ef75 Mon Sep 17 00:00:00 2001 From: selbereth Date: Tue, 17 Nov 2026 17:08:50 -0800 Subject: [PATCH] Fix four dead ends between Quick add and the library - Treat undetermined file languages (und, zxx, mul, mis) as absent tags, so an M4B whose only language marker is the MP4 default `und` no longer fails with "The files use a different language". - Ignore OS metadata files (.DS_Store, ._*, Thumbs.db, desktop.ini, ...) when checking structural destination folders, and name the folder and files when a real file does block publishing. - Say how much disk space is free, needed and reserved when a download waits for space, and where the reserve is configured. - Return an accurate 409 when a download is started while automatic import is only requested (unverified route) instead of "approval changed". Refs #109, #110 Co-Authored-By: Claude Sonnet 5.5 --- services/app/domain/automatic_dispatch.py | 7 +++++++ services/app/domain/capacity.py | 15 ++++++++++++-- services/app/importing/match_evidence.py | 13 +++++++++--- services/app/importing/publication.py | 24 +++++++++++++++++++++-- tests/unit/test_capacity_rules.py | 14 +++++++++++++ tests/unit/test_import_publication.py | 21 ++++++++++++++++++++ tests/unit/test_linked_download.py | 21 ++++++++++++++++++++ 7 files changed, 108 insertions(+), 7 deletions(-) diff --git a/services/app/domain/automatic_dispatch.py b/services/app/domain/automatic_dispatch.py index 8fa255cb..5371c811 100644 --- a/services/app/domain/automatic_dispatch.py +++ b/services/app/domain/automatic_dispatch.py @@ -107,6 +107,13 @@ async def approve_route( policy = await db.scalar(query.execution_options(populate_existing=True)) if not policy: raise HTTPException(409, "An administrator must approve automatic imports for this route") + if not policy.enabled and not get_settings().recovery_mode: + # Only "requested" until the folders are verified; nothing was approved or changed. + raise HTTPException( + 409, + "Automatic import is not active for this library folder. Verify its download " + "folder in Settings → Libraries, then turn automatic import on", + ) snapshot = { "policy_id": str(policy.id), "policy_generation": policy.generation, diff --git a/services/app/domain/capacity.py b/services/app/domain/capacity.py index e837e991..b76eb73f 100644 --- a/services/app/domain/capacity.py +++ b/services/app/domain/capacity.py @@ -194,6 +194,14 @@ async def reserved_bytes(db, *, excluding_attempt=None, excluding_entry=None): return defaultdict(int, {key: int(amount) for key, amount in rows}) +def size_label(value): + value = max(int(value), 0) + for unit, factor in (("TiB", 1024**4), ("GiB", 1024**3), ("MiB", MIB)): + if value >= factor: + return f"{value / factor:.1f} {unit}" + return f"{value} B" + + def verify_space(observation, required, reserved, limits): if datetime.now(UTC) - datetime.fromisoformat(observation["at"]) > timedelta(seconds=10): raise CapacityWait("Storage observation expired; capacity will be checked again") @@ -202,9 +210,12 @@ def verify_space(observation, required, reserved, limits): if not space or space["total"] <= 0: raise CapacityWait("Storage capacity is unknown; waiting for a fresh observation") floor = max(limits.minimum_free_bytes, space["total"] * limits.minimum_free_percent // 100) - if space["available"] - reserved.get(key, 0) - amount < floor: + free = space["available"] - reserved.get(key, 0) + if free - amount < floor: raise CapacityWait( - "Waiting for free disk space after existing reservations and reserve" + f"Waiting for free disk space: {size_label(free)} free after reservations, " + f"{size_label(amount)} needed, {size_label(floor)} kept in reserve " + "(Settings → Download clients → Storage reserve)" ) diff --git a/services/app/importing/match_evidence.py b/services/app/importing/match_evidence.py index 732ef4a2..b4f2982d 100644 --- a/services/app/importing/match_evidence.py +++ b/services/app/importing/match_evidence.py @@ -14,12 +14,21 @@ from app.importing.naming import StrictModel ISBN_KEYS = ("isbn", "isbn10", "isbn13", "isbn_10", "isbn_13") +# ISO 639 codes that say "no usable language" (MP4 streams default to "und"). +# They are absent tags, not assertions, so they must not contradict the catalog. +UNDETERMINED_LANGUAGES = frozenset({"und", "zxx", "mul", "mis"}) def language_key(value): return catalog_language(value) or "" +def language_assertions(values): + """Language keys the files actually assert; blank and undetermined tags are neutral.""" + keys = (language_key(value) for value in values if isinstance(value, str) and value.strip()) + return {key for key in keys if key and key not in UNDETERMINED_LANGUAGES} + + def isbn_key(value): if not isinstance(value, str): return None @@ -182,9 +191,7 @@ def group_evidence(snapshot, group): ) ): authors.add(names) - languages.update( - language_key(value) for value in language if isinstance(value, str) and value.strip() - ) + languages.update(language_assertions(language)) for assertion in assertions: scheme, value = assertion.get("scheme"), assertion.get("value") found = identifier(scheme, value) diff --git a/services/app/importing/publication.py b/services/app/importing/publication.py index f05994b2..5b9ea87a 100644 --- a/services/app/importing/publication.py +++ b/services/app/importing/publication.py @@ -773,6 +773,16 @@ def verify_item(folder, spec, deadline, derived=None, receipt=None): raise PublicationError("Generated metadata differs from its frozen manifest") +# File browsers drop these into shared folders. Audiobookshelf ignores them, so they +# cannot be absorbed into a book and must not block publishing. +OS_METADATA_NAMES = frozenset({".ds_store", "thumbs.db", "desktop.ini", ".localized", ".directory"}) + + +def is_os_metadata(name): + lowered = name.lower() + return lowered in OS_METADATA_NAMES or lowered.startswith("._") + + def conflicting_name(fd, name): key = collision_key(name) return next((existing for existing in os.listdir(fd) if collision_key(existing) == key), None) @@ -797,8 +807,18 @@ def destination_parent(root, relative): fd = child # Structural ancestors may not contain files that ABS could absorb into a book. with os.scandir(fd) as entries: - if any(not entry.is_dir(follow_symlinks=False) for entry in entries): - raise PublicationError("A structural destination folder already contains files") + stray = sorted( + entry.name + for entry in entries + if not entry.is_dir(follow_symlinks=False) and not is_os_metadata(entry.name) + ) + if stray: + shown = ", ".join(name[:60] for name in stray[:3]) + more = f" and {len(stray) - 3} more" if len(stray) > 3 else "" + raise PublicationError( + f"A structural destination folder already contains files: '{part[:80]}' " + f"holds {shown}{more}. Move them out of the library folder and retry" + ) yield fd, parts[-1] finally: os.close(fd) diff --git a/tests/unit/test_capacity_rules.py b/tests/unit/test_capacity_rules.py index 364f7404..7fe3caed 100644 --- a/tests/unit/test_capacity_rules.py +++ b/tests/unit/test_capacity_rules.py @@ -99,3 +99,17 @@ def test_selected_pack_reserves_selected_payload_and_only_selected_import_overhe scope["selected_paths"] = ["Pack/Unknown.m4b"] with pytest.raises(CapacityWait): download_cost(scope, snapshot()) + + +def test_waiting_for_space_says_how_much_and_where_to_change_the_reserve(): + from app.domain.capacity import size_label + + assert size_label(645 * GIB) == "645.0 GiB" and size_label(2 * 1024**4) == "2.0 TiB" + assert size_label(-5) == "0 B" and size_label(512) == "512 B" + big = {"roots": {"a": "a"}, "at": datetime.now(UTC).isoformat()} + big["filesystems"] = {"a": {"available": 645 * GIB, "total": 17 * 1024**4}} + with pytest.raises(CapacityWait) as error: + verify_space(big, {"a": GIB}, {}, Limits()) + message = str(error.value) + assert "645.0 GiB free" in message and "1.0 GiB needed" in message + assert "Storage reserve" in message and "870.4 GiB kept in reserve" in message diff --git a/tests/unit/test_import_publication.py b/tests/unit/test_import_publication.py index a2bc8868..38e9ff95 100644 --- a/tests/unit/test_import_publication.py +++ b/tests/unit/test_import_publication.py @@ -459,6 +459,27 @@ def test_casefold_collision_and_parent_media_are_held(specification): assert (spec.destination_root / "Alex Morgan/old.mp3").read_bytes() == b"existing" +def test_os_metadata_in_parent_folders_does_not_block_publishing(specification): + spec = specification + parent = spec.destination_root / "Alex Morgan" + parent.mkdir() + for name in (".DS_Store", "._cover.jpg", "Thumbs.db", "desktop.ini"): + (parent / name).write_bytes(b"junk") + publish_item(spec) + assert (spec.destination_root / spec.folder).exists() + + +def test_parent_folder_error_names_the_folder_and_files(specification): + spec = specification + parent = spec.destination_root / "Alex Morgan" + parent.mkdir() + (parent / ".DS_Store").write_bytes(b"junk") + (parent / "old.mp3").write_bytes(b"existing") + with pytest.raises(PublicationError, match=r"'Alex Morgan' holds old\.mp3") as error: + publish_item(spec) + assert ".DS_Store" not in str(error.value) + + def test_symlink_destination_parent_cannot_escape(specification, tmp_path): spec = specification outside = tmp_path.resolve() / "outside" diff --git a/tests/unit/test_linked_download.py b/tests/unit/test_linked_download.py index cdb20088..a420248b 100644 --- a/tests/unit/test_linked_download.py +++ b/tests/unit/test_linked_download.py @@ -339,3 +339,24 @@ async def test_reviewed_collection_mapping_requires_independent_file_identity(mo ) assert result is (version if case == "corroborated" else None) assert attach.await_count == (1 if case == "corroborated" else 0) + + +def test_undetermined_file_languages_are_neutral_but_real_ones_still_conflict(): + from app.importing.linked_download import request_file_conflicts + from app.importing.match_evidence import language_assertions + + assert language_assertions(["und", "ZXX", "mul", "mis", "", " ", None, 7]) == set() + assert language_assertions(["und", "eng"]) == {"en"} + work = SimpleNamespace(title="First Harbor", authors=["Alex Morgan"], language="en") + release = {"title": work.title, "authors": work.authors} + + def conflicts(languages): + facts = MatchEvidence( + titles=["first harbor"], + authors=[["alex morgan"]], + languages=sorted(language_assertions(languages)), + ) + return request_file_conflicts(work, release, facts) + + assert conflicts(["und"]) == [] + assert conflicts(["fr"]) == ["The files use a different language"]