Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions apps/web/tests/ui/library-review.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ test("library review links unmatched items to Dewarr and Hardcover books", async
page.on("pageerror", (e) => errors.push(e.message));
const matchSearches: string[] = [];
const matches: { asset: string; body: unknown }[] = [];
let releaseNextSearch!: () => void;
const nextSearch = new Promise<void>((resolve) => {
releaseNextSearch = resolve;
});
let imported = false;
const items = [
{ kind: "asset", asset: asset("a1", "Untagged Folder", ["title"]) },
Expand Down Expand Up @@ -115,6 +119,7 @@ test("library review links unmatched items to Dewarr and Hardcover books", async
data = { items: shown, total: shown.length };
} else if (path === "/api/catalog/works") {
matchSearches.push(url.searchParams.get("q") || "");
if (url.searchParams.get("q") === "The Long Way") await nextSearch;
data = { items: [work], total: 1 };
} else if (path.endsWith("/match") && method === "POST") {
const id = path.split("/").at(-2)!;
Expand Down Expand Up @@ -214,9 +219,16 @@ test("library review links unmatched items to Dewarr and Hardcover books", async
part_total: null,
},
});
await expect(page.getByRole("status")).toContainText(
"Linked “Untagged Folder”",
);
try {
// Saving and continuing can announce success while the next lookup is pending.
await expect(detail.getByRole("status")).toHaveText("Searching catalog…");
await expect(
page.getByRole("status").filter({ hasText: "Linked “Untagged Folder”" }),
).toBeVisible();
} finally {
releaseNextSearch();
}
await expect(detail.getByRole("status")).toHaveCount(0);
await expect(cards).toHaveCount(2);
await expect(
nav.getByRole("link", { name: /^Library review/ }),
Expand Down
2 changes: 1 addition & 1 deletion docs/LIBRARY-FOLDERS.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ Templates accept `{{` and `}}` for literal braces. `{author}/{title}[ {{{narrato

## Staging and mount boundaries

Staging must share the library's mount so complete folders can be published safely. New Audiobookshelf destinations use the reserved direct hidden child, avoiding any requirement to write to the library's parent. Grimmory uses it when its watcher is disabled; otherwise staging remains outside the watched library. Existing valid staging is retained. Only this private namespace is name-specific; user library and download names are unrestricted. New routes keep journals/locks in application storage, independent of media permissions.
Staging must share the library's mount so complete folders can be published safely. New Audiobookshelf destinations use the reserved direct hidden child, avoiding any requirement to write to the library's parent. Grimmory uses it when its watcher is disabled; otherwise staging remains outside the watched library. Existing valid staging is retained. The private staging namespace and `.book-search-root` identity marker names are reserved; ordinary library and download names are unrestricted. New routes keep journals/locks in application storage, independent of media permissions.

Two routes using the same local root reuse the same valid staging route. A saved automatic staging choice that now overlaps the selected library incorrectly is reselected. An explicit `BOOK_IMPORT_STAGING_ROOT` override remains authoritative and receives a path-specific error if invalid. Unfinished imports continue to block changing their saved storage; historical journals are retained.

Expand Down
9 changes: 9 additions & 0 deletions docs/NATIVE-LIBRARY-IMPORTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,15 @@ Activating Bookdrop changes the administrator's personal ebook destination. It p

The connection needs access to Bookdrop and its queue. Folder verification checks local publication access, the selected backend configuration, and queue readability; it does not prove that the two services share a filesystem. Grimmory does not expose its configured Bookdrop root through this API, so Dewarr cannot prove that the chosen path is watched until a delivered EPUB appears in the queue. **Refresh review status** checks for that exact intake path and size.

Bookdrop queue reads support both legacy top-level paging and Grimmory 3.5.0's
nested Spring `page` metadata, including an empty queue. Reads stop at 100 pages
or 10,000 files and reject inconsistent page sizes/counts, repeated or unordered
IDs, and malformed files. A queue changing during a read must be refreshed;
these pages do not provide a transactional snapshot. **Send to Bookdrop on
completion** controls delivery timing. Turning it off still requires Bookdrop
access and folder verification. Select the separate direct-library workflow for
direct imports.

Review metadata and approve the import in Grimmory. Sync the resulting library in Dewarr, then use **Link reviewed copy** in the download's import review. Candidates must be complete, matched copies of the selected edition on the original Grimmory server. Resolve an edition mismatch in library review before linking. Queue disappearance alone does not establish whether a file was imported, discarded or moved.

**Mark rejected** retains the receipt and suppresses another handoff; it does not delete anything in Grimmory. A confirmed or rejected handoff is never automatically resent. Publication receipts also protect against a restart after Grimmory consumes a file but before Dewarr records delivery. Uncertain outcomes require review rather than automatic replacement. Pending human review does not occupy a worker or trigger repeated scans.
Expand Down
44 changes: 44 additions & 0 deletions docs/STORAGE-COMPATIBILITY.md
Original file line number Diff line number Diff line change
Expand Up @@ -277,3 +277,47 @@ API/worker activation journeys passed through `scripts/check.py`. The latter cov
qBittorrent and slskd on local, simulated SMB, and read-only sources, including
copy selection, backend mapping, and import readiness. They used an isolated
PostgreSQL database that was removed afterward. Lint and formatting checks passed.

## NOR-94: library roots whose directory inode changes

The [Unraid report](https://github.com/logabell/dewarr/issues/61) describes the
library root switching its reported inode after a first write to a cache-backed
user share. [Unraid user shares](https://docs.unraid.net/unraid-os/using-unraid-to/manage-storage/shares/)
combine folders through FUSE. Replaying that root-only inode transition against
real local publication operations reproduced both the final publication failure
and the retry failure. This does not establish how every FUSE filesystem behaves.

New publications create a persistent `.book-search-root` file in the destination
root. Its random token, device and **file inode**, together with the root device,
identify the destination independently of the root directory inode. Initialization
writes and flushes a temporary file before publishing it without replacement;
concurrent creators use the winning marker. A malformed or linked marker is held
for review. The marker contains no credentials and is not a media file. Keep it
with the original library; do not copy it to provision another destination.

Publication, seeding moves, combining, cancellation and companion placement use
the same marker-based destination lock. New receipts require the original marker,
even if the directory inode still matches. A copied token, different device,
missing/changed marker or symlink does not authorize publication. Source files,
item folders, staged files and publication ownership still have their existing
checks. Recovery scans only read this evidence; they never initialize markers.
Stop old workers when upgrading so all writers use the same locking protocol.

Older receipts without a root marker retain strict device/inode matching. They
are never silently rebound to the current folder. If an old receipt is stale,
**Cancel import** can remove only a still-present, verified owned stage and
record cancellation; then start a new import. It preserves the source and does
not remove anything from the library. Missing stages, published receipts,
unconfirmed staging and seeding moves require review instead. Interrupted
cancellation remains retryable from its journal.

The regression checks cover root changes during publication and retry, a stable
shared lock, concurrent marker initialization, actual directory replacement,
copied/modified markers, symlinks, read-only recovery, Bookdrop restart recovery,
combining/separating, seeding moves and cancellation followed by a fresh import.
API/worker checks also retain both ebook/audiobook import orders and multiple
narrations. They simulate the reported identity transition on local files;
**no live Unraid or cache-mover interoperability test was available**. If the
filesystem also changes the marker file's identity, existing receipts remain
held for review. This fix does not trust a token alone or certify that broader
filesystem behavior.
69 changes: 65 additions & 4 deletions services/app/adapters/grimmory.py
Original file line number Diff line number Diff line change
Expand Up @@ -537,30 +537,91 @@ async def metadata_persistence(self) -> dict:

async def bookdrop_files(self) -> list[dict]:
rows, seen = [], set()
snapshot = None
for page in range(100):
value = await self.request(
"GET", "api/v1/bookdrop/files", params={"page": page, "size": 100, "sort": "id,asc"}
)
content = value.get("content")
if not isinstance(content, list) or type(value.get("last")) is not bool:
fields = ("number", "size", "totalElements", "totalPages")
nested = "page" in value
metadata = value.get("page") if nested else value
has_metadata = nested or any(field in value for field in fields)
if not isinstance(content, list) or len(content) > 100:
raise AdapterError(
FailureKind.PARSER, "Grimmory returned an invalid Bookdrop queue."
)
if has_metadata:
if not isinstance(metadata, dict) or any(
type(metadata.get(field)) is not int for field in fields
):
raise AdapterError(FailureKind.PARSER, "Bookdrop returned invalid pagination.")
number, size, total, pages = (metadata[field] for field in fields)
if (
number != page
or not 1 <= size <= 100
or total < 0
or pages != (total + size - 1) // size
or (total == 0 and page != 0)
or (total > 0 and page >= pages)
or len(content) != min(size, max(0, total - page * size))
):
raise AdapterError(
FailureKind.PARSER, "Bookdrop returned inconsistent pagination."
)
if pages > 100 or total > 10_000:
raise AdapterError(
FailureKind.PARSER, "Bookdrop queue exceeds the supported review limit."
)
current = (nested, size, total, pages)
last = page + 1 >= pages
# If both representations are present, neither may contradict the other.
if "last" in value and (type(value["last"]) is not bool or value["last"] != last):
raise AdapterError(
FailureKind.PARSER, "Bookdrop returned inconsistent pagination."
)
if nested and any(
field in value
and (type(value[field]) is not int or value[field] != metadata[field])
for field in fields
):
raise AdapterError(
FailureKind.PARSER, "Bookdrop returned inconsistent pagination."
)
else:
# Older servers expose only content/last. Retain that contract,
# with the same row/page bounds and progress checks.
last = value.get("last")
if type(last) is not bool or (not last and not content):
raise AdapterError(FailureKind.PARSER, "Bookdrop returned invalid pagination.")
current = (False, None, None, None)
if snapshot is not None and snapshot != current:
raise AdapterError(
FailureKind.UNCERTAIN,
"Bookdrop changed while reading its queue; refresh again.",
)
snapshot = current
for row in content:
if (
not isinstance(row, dict)
or type(row.get("id")) is not int
or row["id"] <= 0
or not isinstance(row.get("filePath"), str)
or not row["filePath"].startswith("/")
or type(row.get("fileSize")) is not int
or row["id"] in seen
or row["fileSize"] < 0
):
raise AdapterError(
FailureKind.PARSER,
FailureKind.PARSER, "Grimmory returned an invalid Bookdrop file."
)
if row["id"] in seen or (rows and row["id"] <= rows[-1]["id"]):
raise AdapterError(
FailureKind.UNCERTAIN,
"Bookdrop changed while reading its queue; refresh again.",
)
seen.add(row["id"])
rows.append(row)
if value["last"]:
if last:
return rows
raise AdapterError(FailureKind.PARSER, "Bookdrop queue exceeds the supported review limit.")

Expand Down
8 changes: 4 additions & 4 deletions services/app/importing/bookdrop.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@
from app.importing.publication import (
PublicationError,
entry_lock,
object_id,
private_staging,
read_receipt,
same_destination,
specification_fingerprint,
)

Expand All @@ -32,9 +32,9 @@ def recovered_handoff(spec):
if not receipt:
return None
with directory(spec.destination_root) as destination:
if receipt.get("spec_hash") != specification_fingerprint(spec) or receipt.get(
"destination_identity"
) != object_id(destination):
if receipt.get("spec_hash") != specification_fingerprint(spec) or not same_destination(
destination, receipt.get("destination_identity")
):
raise PublicationError(
"Bookdrop receipt or intake mount changed; review the existing handoff"
)
Expand Down
47 changes: 38 additions & 9 deletions services/app/importing/cancel_files.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
"""Stop one unpublished item under the same filesystem lock used by publication."""

import json
import os
from contextlib import ExitStack, nullcontext
from uuid import uuid4
Expand All @@ -9,16 +8,18 @@
from app.importing.publication import (
PUBLICATION_MARKER,
PublicationError,
destination_identity,
destination_lock,
entry_lock,
generated_files,
has_publication_marker,
needs_hardlink,
object_id,
private_staging,
publication_lock,
published_names,
read_receipt,
remove_publication_marker,
same_destination,
same_object,
specification_fingerprint,
sync_directory,
Expand Down Expand Up @@ -110,7 +111,7 @@ def cancel_files(spec, *, guard=nullcontext, checkpoint=lambda _: None):
):
with (
entry_lock(staging, spec.entry_id),
publication_lock(staging, json.dumps(object_id(root), sort_keys=True)),
destination_lock(staging, root),
guard(),
):
receipt = read_receipt(staging, name)
Expand All @@ -130,21 +131,49 @@ def cancel_files(spec, *, guard=nullcontext, checkpoint=lambda _: None):
"schema_version": 1,
"entry_id": str(spec.entry_id),
"spec_hash": specification_fingerprint(spec),
"destination_identity": object_id(root),
"destination_identity": destination_identity(root),
"stage_name": "item-" + uuid4().hex,
"state": "cancelled",
}
# A durable tombstone also fences a delayed pre-cancellation worker.
write_receipt(staging, name, receipt, create=True)
return receipt
if (
receipt.get("entry_id") != str(spec.entry_id)
or receipt.get("spec_hash") != specification_fingerprint(spec)
or receipt.get("destination_identity") != object_id(root)
):
if receipt.get("entry_id") != str(spec.entry_id) or receipt.get(
"spec_hash"
) != specification_fingerprint(spec):
raise PublicationError("Cancellation journal or destination identity changed")
if receipt["state"] == "cancelled":
return receipt
if not same_destination(root, receipt.get("destination_identity")):
# Never rebind an old receipt to a new root. An owned stage still
# present proves this atomic publication has not left staging.
# Explicit cancellation can clean only that stage, allowing a new
# plan; missing/published/rename evidence remains held for review.
if (
spec.mode == "rename"
or receipt["state"] not in {"preparing", "prepared", "cancelling"}
or not receipt.get("stage_identity")
or receipt.get("unconfirmed_stages")
):
raise PublicationError(
"Destination identity changed; review the existing publication "
"before replanning"
)
if receipt["state"] != "cancelling":
try:
with beneath(staging, receipt["stage_name"], folder=True) as folder:
if not same_object(folder, receipt["stage_identity"]):
raise PublicationError("Cancellation staging identity changed")
except FileNotFoundError:
raise PublicationError(
"Staged item is missing; publication may have completed elsewhere"
) from None
receipt["state"] = "cancelling"
write_receipt(staging, name, receipt)
remove_stage(staging, receipt, spec, checkpoint)
receipt["state"] = "cancelled"
write_receipt(staging, name, receipt)
return receipt
if spec.mode == "rename":
return cancel_renamed(root, staging, name, receipt, spec)
try:
Expand Down
5 changes: 2 additions & 3 deletions services/app/importing/colocate.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@

import asyncio
import errno
import json
import os
import time
from contextlib import nullcontext
Expand Down Expand Up @@ -38,10 +37,10 @@
from app.importing.publication import (
PublicationBusy,
PublicationError,
destination_lock,
no_replace,
object_id,
private_staging,
publication_lock,
sync_directory,
write_all,
)
Expand Down Expand Up @@ -119,7 +118,7 @@ def place_companion(
Path(target["staging_path"]),
Path(target["journal_path"]) if target.get("journal_path") else None,
) as staging,
publication_lock(staging, json.dumps(object_id(root), sort_keys=True)),
destination_lock(staging, root),
beneath(root, str(relative.parent), folder=True) as folder,
):
if object_id(folder) != configuration["target_folder_identity"]:
Expand Down
Loading
Loading