Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions services/app/domain/automatic_dispatch.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,13 @@ async def approve_route(
policy = await db.scalar(query.execution_options(populate_existing=True))
if not policy:
raise HTTPException(409, "An administrator must approve automatic imports for this route")
if not policy.enabled and not get_settings().recovery_mode:
# Only "requested" until the folders are verified; nothing was approved or changed.
raise HTTPException(
409,
"Automatic import is not active for this library folder. Verify its download "
"folder in Settings → Libraries, then turn automatic import on",
)
snapshot = {
"policy_id": str(policy.id),
"policy_generation": policy.generation,
Expand Down
15 changes: 13 additions & 2 deletions services/app/domain/capacity.py
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,14 @@ async def reserved_bytes(db, *, excluding_attempt=None, excluding_entry=None):
return defaultdict(int, {key: int(amount) for key, amount in rows})


def size_label(value):
value = max(int(value), 0)
for unit, factor in (("TiB", 1024**4), ("GiB", 1024**3), ("MiB", MIB)):
if value >= factor:
return f"{value / factor:.1f} {unit}"
return f"{value} B"


def verify_space(observation, required, reserved, limits):
if datetime.now(UTC) - datetime.fromisoformat(observation["at"]) > timedelta(seconds=10):
raise CapacityWait("Storage observation expired; capacity will be checked again")
Expand All @@ -202,9 +210,12 @@ def verify_space(observation, required, reserved, limits):
if not space or space["total"] <= 0:
raise CapacityWait("Storage capacity is unknown; waiting for a fresh observation")
floor = max(limits.minimum_free_bytes, space["total"] * limits.minimum_free_percent // 100)
if space["available"] - reserved.get(key, 0) - amount < floor:
free = space["available"] - reserved.get(key, 0)
if free - amount < floor:
raise CapacityWait(
"Waiting for free disk space after existing reservations and reserve"
f"Waiting for free disk space: {size_label(free)} free after reservations, "
f"{size_label(amount)} needed, {size_label(floor)} kept in reserve "
"(Settings → Download clients → Storage reserve)"
)


Expand Down
13 changes: 10 additions & 3 deletions services/app/importing/match_evidence.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,21 @@
from app.importing.naming import StrictModel

ISBN_KEYS = ("isbn", "isbn10", "isbn13", "isbn_10", "isbn_13")
# ISO 639 codes that say "no usable language" (MP4 streams default to "und").
# They are absent tags, not assertions, so they must not contradict the catalog.
UNDETERMINED_LANGUAGES = frozenset({"und", "zxx", "mul", "mis"})


def language_key(value):
return catalog_language(value) or ""


def language_assertions(values):
"""Language keys the files actually assert; blank and undetermined tags are neutral."""
keys = (language_key(value) for value in values if isinstance(value, str) and value.strip())
return {key for key in keys if key and key not in UNDETERMINED_LANGUAGES}


def isbn_key(value):
if not isinstance(value, str):
return None
Expand Down Expand Up @@ -182,9 +191,7 @@ def group_evidence(snapshot, group):
)
):
authors.add(names)
languages.update(
language_key(value) for value in language if isinstance(value, str) and value.strip()
)
languages.update(language_assertions(language))
for assertion in assertions:
scheme, value = assertion.get("scheme"), assertion.get("value")
found = identifier(scheme, value)
Expand Down
24 changes: 22 additions & 2 deletions services/app/importing/publication.py
Original file line number Diff line number Diff line change
Expand Up @@ -773,6 +773,16 @@ def verify_item(folder, spec, deadline, derived=None, receipt=None):
raise PublicationError("Generated metadata differs from its frozen manifest")


# File browsers drop these into shared folders. Audiobookshelf ignores them, so they
# cannot be absorbed into a book and must not block publishing.
OS_METADATA_NAMES = frozenset({".ds_store", "thumbs.db", "desktop.ini", ".localized", ".directory"})


def is_os_metadata(name):
lowered = name.lower()
return lowered in OS_METADATA_NAMES or lowered.startswith("._")


def conflicting_name(fd, name):
key = collision_key(name)
return next((existing for existing in os.listdir(fd) if collision_key(existing) == key), None)
Expand All @@ -797,8 +807,18 @@ def destination_parent(root, relative):
fd = child
# Structural ancestors may not contain files that ABS could absorb into a book.
with os.scandir(fd) as entries:
if any(not entry.is_dir(follow_symlinks=False) for entry in entries):
raise PublicationError("A structural destination folder already contains files")
stray = sorted(
entry.name
for entry in entries
if not entry.is_dir(follow_symlinks=False) and not is_os_metadata(entry.name)
)
if stray:
shown = ", ".join(name[:60] for name in stray[:3])
more = f" and {len(stray) - 3} more" if len(stray) > 3 else ""
raise PublicationError(
f"A structural destination folder already contains files: '{part[:80]}' "
f"holds {shown}{more}. Move them out of the library folder and retry"
)
yield fd, parts[-1]
finally:
os.close(fd)
Expand Down
14 changes: 14 additions & 0 deletions tests/unit/test_capacity_rules.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,3 +99,17 @@ def test_selected_pack_reserves_selected_payload_and_only_selected_import_overhe
scope["selected_paths"] = ["Pack/Unknown.m4b"]
with pytest.raises(CapacityWait):
download_cost(scope, snapshot())


def test_waiting_for_space_says_how_much_and_where_to_change_the_reserve():
from app.domain.capacity import size_label

assert size_label(645 * GIB) == "645.0 GiB" and size_label(2 * 1024**4) == "2.0 TiB"
assert size_label(-5) == "0 B" and size_label(512) == "512 B"
big = {"roots": {"a": "a"}, "at": datetime.now(UTC).isoformat()}
big["filesystems"] = {"a": {"available": 645 * GIB, "total": 17 * 1024**4}}
with pytest.raises(CapacityWait) as error:
verify_space(big, {"a": GIB}, {}, Limits())
message = str(error.value)
assert "645.0 GiB free" in message and "1.0 GiB needed" in message
assert "Storage reserve" in message and "870.4 GiB kept in reserve" in message
21 changes: 21 additions & 0 deletions tests/unit/test_import_publication.py
Original file line number Diff line number Diff line change
Expand Up @@ -459,6 +459,27 @@ def test_casefold_collision_and_parent_media_are_held(specification):
assert (spec.destination_root / "Alex Morgan/old.mp3").read_bytes() == b"existing"


def test_os_metadata_in_parent_folders_does_not_block_publishing(specification):
spec = specification
parent = spec.destination_root / "Alex Morgan"
parent.mkdir()
for name in (".DS_Store", "._cover.jpg", "Thumbs.db", "desktop.ini"):
(parent / name).write_bytes(b"junk")
publish_item(spec)
assert (spec.destination_root / spec.folder).exists()


def test_parent_folder_error_names_the_folder_and_files(specification):
spec = specification
parent = spec.destination_root / "Alex Morgan"
parent.mkdir()
(parent / ".DS_Store").write_bytes(b"junk")
(parent / "old.mp3").write_bytes(b"existing")
with pytest.raises(PublicationError, match=r"'Alex Morgan' holds old\.mp3") as error:
publish_item(spec)
assert ".DS_Store" not in str(error.value)


def test_symlink_destination_parent_cannot_escape(specification, tmp_path):
spec = specification
outside = tmp_path.resolve() / "outside"
Expand Down
21 changes: 21 additions & 0 deletions tests/unit/test_linked_download.py
Original file line number Diff line number Diff line change
Expand Up @@ -339,3 +339,24 @@ async def test_reviewed_collection_mapping_requires_independent_file_identity(mo
)
assert result is (version if case == "corroborated" else None)
assert attach.await_count == (1 if case == "corroborated" else 0)


def test_undetermined_file_languages_are_neutral_but_real_ones_still_conflict():
from app.importing.linked_download import request_file_conflicts
from app.importing.match_evidence import language_assertions

assert language_assertions(["und", "ZXX", "mul", "mis", "", " ", None, 7]) == set()
assert language_assertions(["und", "eng"]) == {"en"}
work = SimpleNamespace(title="First Harbor", authors=["Alex Morgan"], language="en")
release = {"title": work.title, "authors": work.authors}

def conflicts(languages):
facts = MatchEvidence(
titles=["first harbor"],
authors=[["alex morgan"]],
languages=sorted(language_assertions(languages)),
)
return request_file_conflicts(work, release, facts)

assert conflicts(["und"]) == []
assert conflicts(["fr"]) == ["The files use a different language"]