diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b099b5..2487868 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,12 @@ point is tag `upstream-baseline`. problems print a message instead of a traceback. With `--json-log`, a run that fails with an unhandled error still appends a `run` record carrying `exit_code` and `error`. +### Fixed +- `Config/dedupe_roots`: a multi-file release whose discs are already hardlinked under *different* library + folders (a prior `multibook` run, or a file moved in the library) is no longer reported as already filed. + The first folder used to win, `hardlinkUnlessFiled` skipped, and the processed marker then hid the rest of + the files on later runs. The `Title/cd1`, `Title/cd2` disc subfolders booktree creates still count as one book. + ### Changed - The cookie store is `/cookies.json` (owner-readable, written atomically) instead of `cookies.pkl`. A pickle from a shared directory was loaded on every run, which executes whatever the file contains; an existing diff --git a/CONFIG.md b/CONFIG.md index c040699..d4c27c8 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -262,8 +262,9 @@ when their media files are the same inodes. Before hardlinking a matched release in an index of the media files under the listed directories (built once per run); if one is found, the release is reported (`Already in the library at ...; not hardlinking ...`, `already_filed` in the JSON log) and left alone, but still logged as matched (with its target path) and marked processed. A release counts as filed only -when every one of its files is, so an interrupted filing is completed on the next run; symlinks in a library do -not count. Nothing is ever deleted. Typical values are your +when every one of its files is in the same library book folder (`Title/cd1`, `Title/cd2` disc subfolders count +as `Title`), so an interrupted or split filing (discs left under two different matches) is completed on the next +run; symlinks in a library do not count. Nothing is ever deleted. Typical values are your `media_path` (stops a second config from filing a clone of a book the first one already filed) or the libraries of other users on the same server. A root that contains, or lies inside, a `source_path` is refused, because the downloads themselves would then count as already filed. diff --git a/myx_library.py b/myx_library.py index 03858f8..8c23d91 100644 --- a/myx_library.py +++ b/myx_library.py @@ -14,8 +14,10 @@ Dedupe: two folders hold the same book when their media files are the same inodes (hardlinks), which is exactly how booktree files a download. Before hardlinking a matched book its source files are looked up in an index of (device, inode) built once per run from the dedupe roots; a book already present is reported and left -alone, nothing is ever deleted. A book counts as filed only when every one of its files is (a filing that was -interrupted half-way is completed, as hardlinkFile skips files that already exist). A root that contains, or lies +alone, nothing is ever deleted. A book counts as filed only when every one of its files is in the same library +book folder, where the disc subfolders booktree creates (Title/cd1, Title/cd2) belong to their parent (a filing +that was interrupted half-way, or split across two folders after a multibook run or a manual move, is completed; +hardlinkFile skips files that already exist at the new target). A root that contains, or lies inside, a source path is refused: the downloads themselves would then count as "already filed" and nothing under it would ever be hardlinked. """ @@ -25,6 +27,8 @@ import requests +import myx_utilities + TIMEOUT = 30 LIBRARY_ID = re.compile(r"[A-Za-z0-9_-]{1,64}") # ABS ids are `lib_...` or UUIDs; anything else would change the URL path MEDIA_EXTS = (".m4b", ".mp3", ".m4a", ".flac", ".ogg", ".opus", ".aac", ".wma") @@ -91,7 +95,13 @@ def _buildIndex(roots): def alreadyFiled(cfg, files): - """The library folder that already holds every one of `files` (paths of a book's media files), or None.""" + """The library folder that already holds every one of `files` (paths of a book's media files), or None. + + Every file must be present *and* they must all belong to the same book folder (disc subfolders such as + Title/cd1, Title/cd2 count as their parent): discs filed under two different matches (multibook-on, then off; + or a later move in the library) are not "already filed" — hardlinkUnlessFiled should complete the release, + where the processed marker would otherwise hide the rest of the files forever. + """ global _index, _indexRoots roots = dedupeRoots(cfg) if not roots: @@ -110,7 +120,21 @@ def alreadyFiled(cfg, files): if not folder: return None folders.append(folder) - return folders[0] if folders else None + if not folders: + return None + # booktree files a multi-disc book as Title/cd1, Title/cd2 (Config/target_path/disc_folder): those are one + # book folder. "Title" next to "Title Disc 2" (a split filing) are not: their parents differ. + homes = [_bookFolder(folder) for folder in folders] + if any(home != homes[0] for home in homes[1:]): + return None + return homes[0] + + +def _bookFolder(folder): + """The book folder a filed media file belongs to: its directory, or the parent when that is a disc subfolder.""" + if myx_utilities.isMultiCD(os.path.basename(folder)): + return os.path.dirname(folder) + return folder # ---------------------------------------------------------------- Audiobookshelf diff --git a/tests/test_operator_tools.py b/tests/test_operator_tools.py index 1b8d918..466b62d 100644 --- a/tests/test_operator_tools.py +++ b/tests/test_operator_tools.py @@ -357,6 +357,57 @@ def test_every_file_must_be_filed_and_symlinks_do_not_count(self): with contextlib.redirect_stdout(io.StringIO()): self.assertEqual(myx_library.alreadyFiled(cfg, [self.source_file, second]), target) + def test_files_in_different_library_folders_are_not_already_filed(self): + # multibook-on filed each disc as its own book; or the operator moved one disc in ABS. The inodes are + # all in the library, but no single folder holds the release — do not skip, or cacheMe would hide it. + cfg = self.cfg(**{"Config/dedupe_roots": [self.lib]}) + second = os.path.join(self.src, "Some Book", "cd2.m4b") + with open(second, "wb") as fh: + fh.write(b"\x00" * 16) + first_dir = os.path.join(self.lib, "Author", "Some Book") + other_dir = os.path.join(self.lib, "Author", "Some Book Disc 2") + os.makedirs(other_dir) + os.link(self.source_file, os.path.join(first_dir, "cd1.m4b")) + os.link(second, os.path.join(other_dir, "cd2.m4b")) + with contextlib.redirect_stdout(io.StringIO()): + self.assertIsNone(myx_library.alreadyFiled(cfg, [self.source_file, second])) + import booktree + import myx_classes + mb = myx_classes.MAMBook("Some Book") + match = myx_classes.Book(asin=ASIN, title="Some Book") + match.authors = [myx_classes.Contributor("Author")] + for name, path in (("Some Book/book.m4b", self.source_file), ("Some Book/cd2.m4b", second)): + bf = myx_classes.BookFile(name, path, self.src, self.lib) + bf.ffprobeBook = match + mb.files.append(bf) + mb.ffprobeBook = mb.bestAudibleMatch = match + mb.metadata = "audible" + mb.isMatched = True + with contextlib.redirect_stdout(io.StringIO()) as out: + self.assertTrue(booktree.hardlinkUnlessFiled(mb, cfg)) + unified = os.path.join(self.lib, "Author", "Some Book") + self.assertTrue(os.path.exists(os.path.join(unified, "book.m4b")), out.getvalue()) + self.assertTrue(os.path.exists(os.path.join(unified, "cd2.m4b")), out.getvalue()) + self.assertNotIn("Already in the library", out.getvalue()) + + def test_disc_subfolders_of_one_book_count_as_already_filed(self): + # booktree itself files a multi-disc book as Title/cd1, Title/cd2 (disc_folder): that is one filed book + cfg = self.cfg(**{"Config/dedupe_roots": [self.lib]}) + second = os.path.join(self.src, "Some Book", "cd2.m4b") + with open(second, "wb") as fh: + fh.write(b"\x00" * 16) + title = os.path.join(self.lib, "Author", "Some Book") + for disc, source in (("cd1", self.source_file), ("Disc 02", second)): + os.makedirs(os.path.join(title, disc)) + os.link(source, os.path.join(title, disc, os.path.basename(source))) + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(myx_library.alreadyFiled(cfg, [self.source_file, second]), title) + # the same discs under two different titles are still a split filing + os.rename(os.path.join(title, "Disc 02"), os.path.join(self.lib, "Author", "Other Title")) + myx_library.reset() + with contextlib.redirect_stdout(io.StringIO()): + self.assertIsNone(myx_library.alreadyFiled(cfg, [self.source_file, second])) + def test_pinned_or_refreshed_book_is_filed_even_when_a_copy_exists(self): # the copy in the library is the wrong match being corrected: dedupe must not block the pin import booktree