From 10f5db8a304bdfa8169621c0a740943f91722bfb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 17 Sep 2026 14:35:50 +0000 Subject: [PATCH 1/2] dedupe: do not treat split library folders as already filed alreadyFiled returned the first inode's folder whenever every file of a release appeared somewhere under dedupe_roots. Discs left in two different matches (a prior multibook run, or a file moved in the library) were skipped, then cacheMe hid the rest of the files on later runs. Require every file to live in the same folder so hardlinkUnlessFiled can complete the release. Co-authored-by: Mancolt --- CHANGELOG.md | 6 ++++++ CONFIG.md | 4 ++-- myx_library.py | 19 +++++++++++++++---- tests/test_operator_tools.py | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d84a019..b140d8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,12 @@ point is tag `upstream-baseline`. problems print a message instead of a traceback. With `--json-log`, a run that fails with an unhandled error still appends a `run` record carrying `exit_code` and `error`. +### Fixed +- `Config/dedupe_roots`: a multi-file release whose discs are already hardlinked under *different* library + folders (a prior `multibook` run, or a file moved in the library) is no longer reported as already filed. + The first folder used to win, `hardlinkUnlessFiled` skipped, and the processed marker then hid the rest of + the files on later runs. + ### Changed - The cookie store is `/cookies.json` (owner-readable, written atomically) instead of `cookies.pkl`. A pickle from a shared directory was loaded on every run, which executes whatever the file contains; an existing diff --git a/CONFIG.md b/CONFIG.md index 8fa3061..e98437f 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -262,8 +262,8 @@ when their media files are the same inodes. Before hardlinking a matched release in an index of the media files under the listed directories (built once per run); if one is found, the release is reported (`Already in the library at ...; not hardlinking ...`, `already_filed` in the JSON log) and left alone, but still logged as matched (with its target path) and marked processed. A release counts as filed only -when every one of its files is, so an interrupted filing is completed on the next run; symlinks in a library do -not count. Nothing is ever deleted. Typical values are your +when every one of its files is in the same library folder, so an interrupted or split filing (discs left under +two different matches) is completed on the next run; symlinks in a library do not count. Nothing is ever deleted. Typical values are your `media_path` (stops a second config from filing a clone of a book the first one already filed) or the libraries of other users on the same server. A root that contains, or lies inside, a `source_path` is refused, because the downloads themselves would then count as already filed. diff --git a/myx_library.py b/myx_library.py index 03858f8..9ec565b 100644 --- a/myx_library.py +++ b/myx_library.py @@ -14,8 +14,9 @@ Dedupe: two folders hold the same book when their media files are the same inodes (hardlinks), which is exactly how booktree files a download. Before hardlinking a matched book its source files are looked up in an index of (device, inode) built once per run from the dedupe roots; a book already present is reported and left -alone, nothing is ever deleted. A book counts as filed only when every one of its files is (a filing that was -interrupted half-way is completed, as hardlinkFile skips files that already exist). A root that contains, or lies +alone, nothing is ever deleted. A book counts as filed only when every one of its files is in the same library +folder (a filing that was interrupted half-way, or split across two folders after a multibook run or a manual +move, is completed; hardlinkFile skips files that already exist at the new target). A root that contains, or lies inside, a source path is refused: the downloads themselves would then count as "already filed" and nothing under it would ever be hardlinked. """ @@ -91,7 +92,12 @@ def _buildIndex(roots): def alreadyFiled(cfg, files): - """The library folder that already holds every one of `files` (paths of a book's media files), or None.""" + """The library folder that already holds every one of `files` (paths of a book's media files), or None. + + Every file must be present *and* they must all live in the same folder: discs filed under two different + matches (multibook-on, then off; or a later move in the library) are not "already filed" — hardlinkUnlessFiled + should complete the release, then the processed marker would otherwise hide the rest of the files forever. + """ global _index, _indexRoots roots = dedupeRoots(cfg) if not roots: @@ -110,7 +116,12 @@ def alreadyFiled(cfg, files): if not folder: return None folders.append(folder) - return folders[0] if folders else None + if not folders: + return None + first = folders[0] + if any(folder != first for folder in folders[1:]): + return None # including sibling disc_folder dirs: re-link is skip-if-exists, not a skip-forever + return first # ---------------------------------------------------------------- Audiobookshelf diff --git a/tests/test_operator_tools.py b/tests/test_operator_tools.py index 1b8d918..d42d4da 100644 --- a/tests/test_operator_tools.py +++ b/tests/test_operator_tools.py @@ -357,6 +357,39 @@ def test_every_file_must_be_filed_and_symlinks_do_not_count(self): with contextlib.redirect_stdout(io.StringIO()): self.assertEqual(myx_library.alreadyFiled(cfg, [self.source_file, second]), target) + def test_files_in_different_library_folders_are_not_already_filed(self): + # multibook-on filed each disc as its own book; or the operator moved one disc in ABS. The inodes are + # all in the library, but no single folder holds the release — do not skip, or cacheMe would hide it. + cfg = self.cfg(**{"Config/dedupe_roots": [self.lib]}) + second = os.path.join(self.src, "Some Book", "cd2.m4b") + with open(second, "wb") as fh: + fh.write(b"\x00" * 16) + first_dir = os.path.join(self.lib, "Author", "Some Book") + other_dir = os.path.join(self.lib, "Author", "Some Book Disc 2") + os.makedirs(other_dir) + os.link(self.source_file, os.path.join(first_dir, "cd1.m4b")) + os.link(second, os.path.join(other_dir, "cd2.m4b")) + with contextlib.redirect_stdout(io.StringIO()): + self.assertIsNone(myx_library.alreadyFiled(cfg, [self.source_file, second])) + import booktree + import myx_classes + mb = myx_classes.MAMBook("Some Book") + match = myx_classes.Book(asin=ASIN, title="Some Book") + match.authors = [myx_classes.Contributor("Author")] + for name, path in (("Some Book/book.m4b", self.source_file), ("Some Book/cd2.m4b", second)): + bf = myx_classes.BookFile(name, path, self.src, self.lib) + bf.ffprobeBook = match + mb.files.append(bf) + mb.ffprobeBook = mb.bestAudibleMatch = match + mb.metadata = "audible" + mb.isMatched = True + with contextlib.redirect_stdout(io.StringIO()) as out: + self.assertTrue(booktree.hardlinkUnlessFiled(mb, cfg)) + unified = os.path.join(self.lib, "Author", "Some Book") + self.assertTrue(os.path.exists(os.path.join(unified, "book.m4b")), out.getvalue()) + self.assertTrue(os.path.exists(os.path.join(unified, "cd2.m4b")), out.getvalue()) + self.assertNotIn("Already in the library", out.getvalue()) + def test_pinned_or_refreshed_book_is_filed_even_when_a_copy_exists(self): # the copy in the library is the wrong match being corrected: dedupe must not block the pin import booktree From 9529f6e40e870637fcfbb71fd1eb04ccb994c43a Mon Sep 17 00:00:00 2001 From: Mancolt Date: Fri, 18 Sep 2026 11:17:19 -0400 Subject: [PATCH 2/2] dedupe: disc subfolders of one book (Title/cd1, Title/cd2) still count as filed The same-folder rule would have reported every multi-disc book booktree filed itself (Config/target_path/disc_folder) as not filed, switching dedupe_roots off for exactly the releases that #15 now groups. Collapse a disc subfolder to its parent before comparing; a split filing across two titles still fails the check. Co-authored-by: Cursor --- CHANGELOG.md | 2 +- CONFIG.md | 5 +++-- myx_library.py | 31 ++++++++++++++++++++++--------- tests/test_operator_tools.py | 18 ++++++++++++++++++ 4 files changed, 44 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f420eaf..2487868 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,7 +37,7 @@ point is tag `upstream-baseline`. - `Config/dedupe_roots`: a multi-file release whose discs are already hardlinked under *different* library folders (a prior `multibook` run, or a file moved in the library) is no longer reported as already filed. The first folder used to win, `hardlinkUnlessFiled` skipped, and the processed marker then hid the rest of - the files on later runs. + the files on later runs. The `Title/cd1`, `Title/cd2` disc subfolders booktree creates still count as one book. ### Changed - The cookie store is `/cookies.json` (owner-readable, written atomically) instead of `cookies.pkl`. diff --git a/CONFIG.md b/CONFIG.md index 933fa2e..d4c27c8 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -262,8 +262,9 @@ when their media files are the same inodes. Before hardlinking a matched release in an index of the media files under the listed directories (built once per run); if one is found, the release is reported (`Already in the library at ...; not hardlinking ...`, `already_filed` in the JSON log) and left alone, but still logged as matched (with its target path) and marked processed. A release counts as filed only -when every one of its files is in the same library folder, so an interrupted or split filing (discs left under -two different matches) is completed on the next run; symlinks in a library do not count. Nothing is ever deleted. Typical values are your +when every one of its files is in the same library book folder (`Title/cd1`, `Title/cd2` disc subfolders count +as `Title`), so an interrupted or split filing (discs left under two different matches) is completed on the next +run; symlinks in a library do not count. Nothing is ever deleted. Typical values are your `media_path` (stops a second config from filing a clone of a book the first one already filed) or the libraries of other users on the same server. A root that contains, or lies inside, a `source_path` is refused, because the downloads themselves would then count as already filed. diff --git a/myx_library.py b/myx_library.py index 9ec565b..8c23d91 100644 --- a/myx_library.py +++ b/myx_library.py @@ -15,8 +15,9 @@ exactly how booktree files a download. Before hardlinking a matched book its source files are looked up in an index of (device, inode) built once per run from the dedupe roots; a book already present is reported and left alone, nothing is ever deleted. A book counts as filed only when every one of its files is in the same library -folder (a filing that was interrupted half-way, or split across two folders after a multibook run or a manual -move, is completed; hardlinkFile skips files that already exist at the new target). A root that contains, or lies +book folder, where the disc subfolders booktree creates (Title/cd1, Title/cd2) belong to their parent (a filing +that was interrupted half-way, or split across two folders after a multibook run or a manual move, is completed; +hardlinkFile skips files that already exist at the new target). A root that contains, or lies inside, a source path is refused: the downloads themselves would then count as "already filed" and nothing under it would ever be hardlinked. """ @@ -26,6 +27,8 @@ import requests +import myx_utilities + TIMEOUT = 30 LIBRARY_ID = re.compile(r"[A-Za-z0-9_-]{1,64}") # ABS ids are `lib_...` or UUIDs; anything else would change the URL path MEDIA_EXTS = (".m4b", ".mp3", ".m4a", ".flac", ".ogg", ".opus", ".aac", ".wma") @@ -94,9 +97,10 @@ def _buildIndex(roots): def alreadyFiled(cfg, files): """The library folder that already holds every one of `files` (paths of a book's media files), or None. - Every file must be present *and* they must all live in the same folder: discs filed under two different - matches (multibook-on, then off; or a later move in the library) are not "already filed" — hardlinkUnlessFiled - should complete the release, then the processed marker would otherwise hide the rest of the files forever. + Every file must be present *and* they must all belong to the same book folder (disc subfolders such as + Title/cd1, Title/cd2 count as their parent): discs filed under two different matches (multibook-on, then off; + or a later move in the library) are not "already filed" — hardlinkUnlessFiled should complete the release, + where the processed marker would otherwise hide the rest of the files forever. """ global _index, _indexRoots roots = dedupeRoots(cfg) @@ -118,10 +122,19 @@ def alreadyFiled(cfg, files): folders.append(folder) if not folders: return None - first = folders[0] - if any(folder != first for folder in folders[1:]): - return None # including sibling disc_folder dirs: re-link is skip-if-exists, not a skip-forever - return first + # booktree files a multi-disc book as Title/cd1, Title/cd2 (Config/target_path/disc_folder): those are one + # book folder. "Title" next to "Title Disc 2" (a split filing) are not: their parents differ. + homes = [_bookFolder(folder) for folder in folders] + if any(home != homes[0] for home in homes[1:]): + return None + return homes[0] + + +def _bookFolder(folder): + """The book folder a filed media file belongs to: its directory, or the parent when that is a disc subfolder.""" + if myx_utilities.isMultiCD(os.path.basename(folder)): + return os.path.dirname(folder) + return folder # ---------------------------------------------------------------- Audiobookshelf diff --git a/tests/test_operator_tools.py b/tests/test_operator_tools.py index d42d4da..466b62d 100644 --- a/tests/test_operator_tools.py +++ b/tests/test_operator_tools.py @@ -390,6 +390,24 @@ def test_files_in_different_library_folders_are_not_already_filed(self): self.assertTrue(os.path.exists(os.path.join(unified, "cd2.m4b")), out.getvalue()) self.assertNotIn("Already in the library", out.getvalue()) + def test_disc_subfolders_of_one_book_count_as_already_filed(self): + # booktree itself files a multi-disc book as Title/cd1, Title/cd2 (disc_folder): that is one filed book + cfg = self.cfg(**{"Config/dedupe_roots": [self.lib]}) + second = os.path.join(self.src, "Some Book", "cd2.m4b") + with open(second, "wb") as fh: + fh.write(b"\x00" * 16) + title = os.path.join(self.lib, "Author", "Some Book") + for disc, source in (("cd1", self.source_file), ("Disc 02", second)): + os.makedirs(os.path.join(title, disc)) + os.link(source, os.path.join(title, disc, os.path.basename(source))) + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(myx_library.alreadyFiled(cfg, [self.source_file, second]), title) + # the same discs under two different titles are still a split filing + os.rename(os.path.join(title, "Disc 02"), os.path.join(self.lib, "Author", "Other Title")) + myx_library.reset() + with contextlib.redirect_stdout(io.StringIO()): + self.assertIsNone(myx_library.alreadyFiled(cfg, [self.source_file, second])) + def test_pinned_or_refreshed_book_is_filed_even_when_a_copy_exists(self): # the copy in the library is the wrong match being corrected: dedupe must not block the pin import booktree