From d6f7710eb54cf67b2501185c491240b7ee6cdec1 Mon Sep 17 00:00:00 2001 From: Ervin Oro Date: Sat, 18 Apr 2026 21:39:56 +0300 Subject: [PATCH 1/5] C0: close test coverage gaps and fix check false-FAIL bug Adds pre-refactor coverage where silent regressions could land undetected in the upcoming scandir/Entry refactor: - test/test_check.py: 4 tests for the check command (Gap A) (previously 0 of 24 statements in check were covered). - test/test_walk_trees.py: 3 collision tests (Gap B) + 1 empty-dir test (Gap C). Test test_compare_dirs_raises_when_index_is_index_but_fs_is_file is marked @expectedFailure; C3 adds the symmetric isinstance check in _compare_dirs' matched-child branch, which un-marks it. Test test_check_reports_file_missing_from_disk asserts the pre-C4 "Unable to open" output from slurp(); C4's rewrite flips it to an explicit "File missing from disk" line. Latent bug surfaced by test 1: check's "missing from index" loop flagged index.txt itself on every run, so every healthy archive reported FAIL. Adds a one-line filter to skip Index.FILENAME. The filter is preserved when C4 rewrites check. 64 tests pass (was 56); isort/flake8/mypy clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- dintact.py | 2 + test/test_check.py | 85 +++++++++++++++++++++++++++++++++++++++++ test/test_walk_trees.py | 79 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 test/test_check.py diff --git a/dintact.py b/dintact.py index 68b82ff..a58c932 100644 --- a/dintact.py +++ b/dintact.py @@ -42,6 +42,8 @@ def check(args: argparse.Namespace) -> None: # Secondly, check that the index is complete for file in walk(cold_dir): rel_path: PurePath = file.relative_to(cold_dir) + if rel_path == PurePath(Index.FILENAME): + continue if rel_path not in index: print(f"File missing from index: '{rel_path}'.", file=sys.stderr) fail_count += 1 diff --git a/test/test_check.py b/test/test_check.py new file mode 100644 index 0000000..1618b26 --- /dev/null +++ b/test/test_check.py @@ -0,0 +1,85 @@ +import json +import tempfile +import unittest +from argparse import Namespace +from contextlib import redirect_stderr, redirect_stdout +from io import StringIO +from pathlib import Path +from unittest import TestCase + +import xxhash + +from dintact import check +from index import Index + + +def _xxh(data: bytes) -> str: + return xxhash.xxh3_128(data).hexdigest() + + +def _write_index(cold_dir: Path, entries: dict) -> None: + meta = {"version": 1, "algorithm": "XXH128", "coding": "utf8"} + with (cold_dir / Index.FILENAME).open('w', encoding='utf8') as f: + f.write(f"# dintact index {json.dumps(meta)}\n") + for name, h in entries.items(): + f.write(f"{h} {name}\n") + + +def _write_files(cold_dir: Path, files: dict) -> None: + for name, content in files.items(): + p = cold_dir / name + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(content) + + +def _run_check(cold_dir: Path): + out, err = StringIO(), StringIO() + with redirect_stdout(out), redirect_stderr(err): + check(Namespace(cold_dir=str(cold_dir))) + return out.getvalue(), err.getvalue() + + +class TestCheck(TestCase): + def test_check_reports_ok_when_everything_matches(self): + with tempfile.TemporaryDirectory() as tmp: + cold = Path(tmp) + _write_files(cold, {'a.txt': b'hello'}) + _write_index(cold, {'a.txt': _xxh(b'hello')}) + out, err = _run_check(cold) + self.assertIn("OK: Data is intact!", out) + self.assertNotIn("FAIL", out) + self.assertNotIn("Verification failed", err) + self.assertNotIn("missing", err) + + def test_check_reports_hash_mismatch(self): + with tempfile.TemporaryDirectory() as tmp: + cold = Path(tmp) + _write_files(cold, {'a.txt': b'real content'}) + _write_index(cold, {'a.txt': _xxh(b'different content')}) + out, err = _run_check(cold) + self.assertIn("Verification failed: 'a.txt'", err) + self.assertIn("FAIL", out) + + def test_check_reports_file_missing_from_index(self): + with tempfile.TemporaryDirectory() as tmp: + cold = Path(tmp) + _write_files(cold, {'a.txt': b'a', 'b.txt': b'b'}) + _write_index(cold, {'a.txt': _xxh(b'a')}) + out, err = _run_check(cold) + self.assertIn("File missing from index: 'b.txt'", err) + self.assertIn("FAIL", out) + + def test_check_reports_file_missing_from_disk(self): + # Pre-C4 assertion: "Unable to open" lands on stderr via slurp(). + # C4 flips this to an explicit "File missing from disk: '...'" line. + with tempfile.TemporaryDirectory() as tmp: + cold = Path(tmp) + _write_index(cold, {'gone.txt': _xxh(b'x')}) + out, err = _run_check(cold) + self.assertIn("Unable to open", err) + self.assertIn("gone.txt", err) + self.assertIn("FAIL", out) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_walk_trees.py b/test/test_walk_trees.py index d43288a..1372ae4 100644 --- a/test/test_walk_trees.py +++ b/test/test_walk_trees.py @@ -1,5 +1,6 @@ import os import sys +import tempfile import unittest from pathlib import Path, PurePath from unittest import mock @@ -51,3 +52,81 @@ def test_bundled_folder(self): Moved(PurePath('Moved2'), Index(), Removed(PurePath('Moved1'), Index())), ], key=repr), sorted(changes, key=repr)) + + +class TestCollisions(unittest.TestCase): + """Type-collision cases between filesystem and index. + + These pin the invariant that walk_trees must refuse to produce changes + when hot, cold, or the index disagree about whether a name is a file or + a directory — a collision silently classified as a regular change is a + data-loss hazard (apply() could rm a directory expecting a file, or + cp over a directory expecting a file). + """ + + def _setup(self, tmp: Path): + hot = tmp / 'hot' + cold = tmp / 'cold' + hot.mkdir() + cold.mkdir() + return hot, cold + + def test_compare_dirs_raises_on_fs_file_vs_fs_dir(self): + with tempfile.TemporaryDirectory() as tmp: + hot, cold = self._setup(Path(tmp)) + (hot / 'x').write_bytes(b'file-on-hot') + (cold / 'x').mkdir() + (cold / 'x' / 'inner.txt').write_bytes(b'inner') + index = Index() + with self.assertRaises(NotImplementedError): + walk_trees(PurePath(), index, hot, cold, mock.MagicMock()) + + @unittest.expectedFailure + def test_compare_dirs_raises_when_index_is_index_but_fs_is_file(self): + # Un-marked in C3, which adds the symmetric isinstance check in the + # matched-child branch of _compare_dirs. On current code, this + # scenario silently returns ModifiedCopied, rewriting the index + # with a file hash and discarding the sub-tree record. + with tempfile.TemporaryDirectory() as tmp: + hot, cold = self._setup(Path(tmp)) + (hot / 'x').write_bytes(b'same') + (cold / 'x').write_bytes(b'same') + index = Index() + sub = Index() + sub[PurePath('inner.txt')] = 'deadbeef' + index[PurePath('x')] = sub + with self.assertRaises(NotImplementedError): + walk_trees(PurePath(), index, hot, cold, mock.MagicMock()) + + def test_compare_dirs_raises_when_index_is_hash_but_fs_is_dir(self): + with tempfile.TemporaryDirectory() as tmp: + hot, cold = self._setup(Path(tmp)) + (hot / 'x').mkdir() + (hot / 'x' / 'a.txt').write_bytes(b'a') + (cold / 'x').mkdir() + (cold / 'x' / 'a.txt').write_bytes(b'a') + index = Index() + index[PurePath('x')] = 'deadbeef' + with self.assertRaises(NotImplementedError): + walk_trees(PurePath(), index, hot, cold, mock.MagicMock()) + + +class TestEmptyDirs(unittest.TestCase): + def test_compare_dirs_ignores_empty_dirs(self): + # An empty directory on either side produces no change — is_relevant + # filters it out before _compare_dirs sees it. + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + hot = tmp / 'hot' + cold = tmp / 'cold' + hot.mkdir() + cold.mkdir() + (hot / 'empty_only_in_hot').mkdir() + (cold / 'empty_only_in_cold').mkdir() + index = Index() + changes = walk_trees(PurePath(), index, hot, cold, mock.MagicMock()) + self.assertEqual([], changes) + + +if __name__ == "__main__": + unittest.main() From f598102c39648c8c595bc972bd01ab39d1db7b12 Mon Sep 17 00:00:00 2001 From: Ervin Oro Date: Sat, 18 Apr 2026 21:40:24 +0300 Subject: [PATCH 2/5] C1: bump Python floor to 3.14 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI matrix goes to 3.14 only across ubuntu/macos/windows. actions/checkout and setup-python bump from v2 to v4/v5 respectively — v2's setup-python predates 3.14 and would fail to install it. README's "Depends on Python 3.8+" was already a lie (the code uses 3.10+ syntax); updated to 3.14+. The stale ".gitignore" section is removed — commit 3dcd517 ripped that functionality out and the docs never caught up. requirements.txt carries no Python pin; untouched. Independent of the upcoming scandir refactor — the Entry architecture in C2-C4 doesn't lean on any 3.14-only API. Landing this first lets later commits assume 3.14 without ambiguity. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/lint-and-test.yml | 6 +++--- README.md | 6 +----- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index b1eff10..21254dd 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -15,11 +15,11 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [ '3.13' ] + python-version: [ '3.14' ] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/README.md b/README.md index b802805..6ca269a 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ It assumes that you have a hot version of a file archive (on your pc maybe), and ## Requirements -Depends on Python 3.8+ and some pip libraries. +Depends on Python 3.14+ and some pip libraries. `pip install -r requirements.txt` @@ -27,10 +27,6 @@ The entire index must fit in memory, but files are read chunk-by-chunk. `dintact sync ` -### Ignored files - -dintact respects any `.gitignore` files it finds, and does **not** back up files matched by these. - ### Testing `python -m unittest discover test` From 136b782f4512f49a526337639285148312a0be81 Mon Sep 17 00:00:00 2001 From: Ervin Oro Date: Sat, 18 Apr 2026 21:43:01 +0300 Subject: [PATCH 3/5] C2: introduce Entry dataclass, single scandir-based walk; drop DirEntryPath MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds Entry, list_dir, walk in utils.py built on os.scandir. Entry carries path, is_dir, is_file, and size — everything the kernel told us at listing time. On Windows, DirEntry.is_dir/is_file/stat read from the FindNextFile cache, so list_dir issues zero per-entry syscalls. On Linux/macOS one stat per file is unavoidable for size (d_type doesn't include it) but now paid once per run, not three times. hash_tree and cp rewritten to consume the new walk (no more rglob, no more per-file stat in the copy loop). sync's four pre-flight passes collapse to one per side; the "Calculating data size" pbar becomes count-only since we no longer pre-count files. hot_dir / cold_dir are made absolute at CLI entry via .absolute() so every Entry.path carried through the system is absolute. .absolute() is chosen over .resolve() to preserve current user-visible behavior (the latter would follow symlinks on the root path). Incidental changes required to keep C2 bisect-safe (existing tests green): - _compare_dirs' Appeared branch: sum(file.stat().st_size for file in walk) → sum(e.size for e in walk). - check's "missing from index" loop: file.relative_to → entry.path.relative_to. is_relevant keeps its Path signature at the C2 checkpoint because _compare_dirs still uses iterdir(); C3 flips both. Behavior note: broken symlinks and exotic entries (sockets, FIFOs) no longer raise ValueError from walk — they are silently skipped, matching os.walk's default. DirEntryPath.py deleted. Its Path-subclass overrides were invalidated by 3.12's with_segments routing (self.entry unset on derived paths), and 3.14's Path.info doesn't carry size so can't replace it. 71 tests pass (was 64), 1 skipped (broken-symlink test on Windows without SeCreateSymbolicLink). isort/flake8/mypy clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- DirEntryPath.py | 44 ----------------------- dintact.py | 27 ++++++-------- test/test_utils.py | 81 +++++++++++++++++++++++++++++++++++++++-- utils.py | 89 ++++++++++++++++++++++++++++++++-------------- 4 files changed, 153 insertions(+), 88 deletions(-) delete mode 100644 DirEntryPath.py diff --git a/DirEntryPath.py b/DirEntryPath.py deleted file mode 100644 index 6e39b7c..0000000 --- a/DirEntryPath.py +++ /dev/null @@ -1,44 +0,0 @@ -import os -from pathlib import Path - - -class DirEntryPath(Path): - entry: os.DirEntry - - def __init__(self, *args): - entry = args[0] - if isinstance(entry, os.DirEntry): - super().__init__(entry.path) - self.entry = entry - else: - super().__init__(*args) - - def is_symlink(self): - if self.entry: - return self.entry.is_symlink() - else: - return super().is_symlink() - - def is_dir(self, follow_symlinks=True): - if self.entry: - return self.entry.is_dir(follow_symlinks=follow_symlinks) - else: - return super().is_dir(follow_symlinks=follow_symlinks) - - def is_file(self, follow_symlinks=True): - if self.entry: - return self.entry.is_file(follow_symlinks=follow_symlinks) - else: - return super().is_file(follow_symlinks=follow_symlinks) - - def is_junction(self): - if self.entry: - return self.entry.is_junction() - else: - return super().is_junction() - - def stat(self, *, follow_symlinks=True): - if self.entry: - return self.entry.stat(follow_symlinks=follow_symlinks) - else: - return super().stat(follow_symlinks=follow_symlinks) diff --git a/dintact.py b/dintact.py index a58c932..dc70853 100644 --- a/dintact.py +++ b/dintact.py @@ -26,7 +26,7 @@ def check(args: argparse.Namespace) -> None: :param args: must have attr cold_dir: str """ - cold_dir = Path(args.cold_dir) + cold_dir = Path(args.cold_dir).absolute() assert cold_dir.is_dir(), "cold_dir not found!" index = Index(cold_dir) fail_count = 0 @@ -40,8 +40,8 @@ def check(args: argparse.Namespace) -> None: print(f"Verification failed: '{p}'.", file=sys.stderr) fail_count += 1 # Secondly, check that the index is complete - for file in walk(cold_dir): - rel_path: PurePath = file.relative_to(cold_dir) + for entry in walk(cold_dir): + rel_path: PurePath = entry.path.relative_to(cold_dir) if rel_path == PurePath(Index.FILENAME): continue if rel_path not in index: @@ -102,7 +102,7 @@ def _compare_dirs(path: PurePath, cold_index: Index, hot_dir: Path, cold_dir: Pa for cold_child in cold_children.difference(hot_children): if cold_child not in cold_index: changes.append(Appeared(cold_child)) - pbar.update(sum(file.stat().st_size for file in walk(cold_dir / cold_child))) + pbar.update(sum(e.size for e in walk(cold_dir / cold_child))) elif hash_tree(cold_dir / cold_child, pbar)[0] == cold_index[cold_child]: changes.append(Removed(cold_child, cold_index[cold_child])) else: @@ -180,24 +180,19 @@ def sync(args: argparse.Namespace) -> None: :param args: must have attrs hot_dir:str and cold_dir: str """ - hot_dir, cold_dir = Path(args.hot_dir), Path(args.cold_dir) + hot_dir, cold_dir = Path(args.hot_dir).absolute(), Path(args.cold_dir).absolute() assert hot_dir.is_dir(), "hot_dir not found!" assert cold_dir.is_dir(), "cold_dir not found!" index = Index(cold_dir) - # Set up progress bar - file_count = 0 - for _, dirs, files in os.walk(hot_dir): - file_count += len(dirs) + len(files) - for _, dirs, files in os.walk(cold_dir): - file_count += len(dirs) + len(files) + # Set up progress bar — one walk per side, size comes from Entry directly total = 0 - with tqdm(total=file_count, unit_scale=True, desc="Calculating data size") as pbar: - for file in walk(hot_dir, pbar): - total += file.stat().st_size - for file in walk(cold_dir, pbar): - total += file.stat().st_size + with tqdm(unit_scale=True, desc="Calculating data size") as pbar: + for side in (hot_dir, cold_dir): + for entry in walk(side): + total += entry.size + pbar.update() with tqdm(total=total, unit="B", unit_scale=True, desc="Detecting changes") as pbar: # Find all changes required changes = walk_trees(PurePath(), index, hot_dir, cold_dir, pbar) diff --git a/test/test_utils.py b/test/test_utils.py index 17e00ad..ce39a13 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -4,8 +4,8 @@ from pathlib import Path from unittest import TestCase, main -from utils import (cp, hash_compare_files, hash_file, hash_tree, rm, slurp, - yesno) +from utils import (Entry, cp, hash_compare_files, hash_file, hash_tree, + list_dir, rm, slurp, walk, yesno) class TestSlurp(TestCase): @@ -93,6 +93,83 @@ def test_unknown(self): hash_tree(Path('asdf_path'), mock.MagicMock()) +class TestListDir(TestCase): + def test_empty_dir(self): + with tempfile.TemporaryDirectory() as tmpdir: + self.assertEqual([], list(list_dir(Path(tmpdir)))) + + def test_files_and_dirs(self): + with tempfile.TemporaryDirectory() as tmpdir: + tmp = Path(tmpdir) + (tmp / 'a.txt').write_bytes(b'hello') + (tmp / 'b.txt').write_bytes(b'bye!') + (tmp / 'sub').mkdir() + entries = sorted(list_dir(tmp), key=lambda e: e.path.name) + self.assertEqual(3, len(entries)) + + a, b, sub = entries + self.assertTrue(a.is_file) + self.assertFalse(a.is_dir) + self.assertEqual(5, a.size) + + self.assertTrue(b.is_file) + self.assertEqual(4, b.size) + + self.assertTrue(sub.is_dir) + self.assertFalse(sub.is_file) + self.assertEqual(0, sub.size) + + def test_entries_are_instances_of_entry(self): + with tempfile.TemporaryDirectory() as tmpdir: + (Path(tmpdir) / 'x').write_bytes(b'') + entries = list(list_dir(Path(tmpdir))) + self.assertIsInstance(entries[0], Entry) + + +class TestWalk(TestCase): + def test_empty_dir(self): + with tempfile.TemporaryDirectory() as tmpdir: + self.assertEqual([], list(walk(Path(tmpdir)))) + + def test_nested_files(self): + with tempfile.TemporaryDirectory() as tmpdir: + tmp = Path(tmpdir) + (tmp / 'a.txt').write_bytes(b'a') + (tmp / 'sub').mkdir() + (tmp / 'sub' / 'b.txt').write_bytes(b'bb') + (tmp / 'sub' / 'deep').mkdir() + (tmp / 'sub' / 'deep' / 'c.txt').write_bytes(b'ccc') + files = sorted(walk(tmp), key=lambda e: e.path.name) + self.assertEqual(['a.txt', 'b.txt', 'c.txt'], [e.path.name for e in files]) + self.assertEqual([1, 2, 3], [e.size for e in files]) + for e in files: + self.assertTrue(e.is_file) + + def test_single_file_as_root(self): + # Supports walk(file) for callers like cp(file, ...) and hash_tree(file). + with tempfile.TemporaryDirectory() as tmpdir: + tmp = Path(tmpdir) + (tmp / 'a.txt').write_bytes(b'hello') + entries = list(walk(tmp / 'a.txt')) + self.assertEqual(1, len(entries)) + self.assertTrue(entries[0].is_file) + self.assertEqual(5, entries[0].size) + + def test_broken_symlink_is_skipped(self): + # Pre-refactor walk raised ValueError on broken symlinks; new walk + # skips them (matches os.walk's default). Symlink creation on Windows + # requires privilege — skip if it does. + with tempfile.TemporaryDirectory() as tmpdir: + tmp = Path(tmpdir) + (tmp / 'real.txt').write_bytes(b'x') + try: + (tmp / 'dangling').symlink_to(tmp / 'nope') + except (OSError, NotImplementedError): + self.skipTest("symlink creation not permitted on this platform") + names = sorted(e.path.name for e in walk(tmp)) + self.assertEqual(['real.txt'], names) + + class TestCp(TestCase): def test_file(self): with tempfile.TemporaryDirectory() as tmpdir: diff --git a/utils.py b/utils.py index d36708f..439d5c4 100644 --- a/utils.py +++ b/utils.py @@ -2,13 +2,13 @@ import os import shutil import sys +from dataclasses import dataclass from pathlib import Path -from typing import Generator, Optional, Tuple, Union +from typing import Generator, Iterator, Tuple, Union import xxhash from tqdm import tqdm -from DirEntryPath import DirEntryPath from index import Index # noinspection PyShadowingBuiltins @@ -17,28 +17,66 @@ CHUNK_SIZE: int = 4096 +@dataclass(frozen=True, slots=True) +class Entry: + """One filesystem entry as reported by os.scandir. + + path: joined with the scandir root; absolute iff the root was absolute. + is_dir / is_file: follow symlinks (matches pre-refactor walk behavior). + size: st_size for files; 0 for directories and non-file entries. + + Broken symlinks and exotic entries (sockets, FIFOs, ...) surface with + both is_dir=False and is_file=False. Callers filter or skip them. + """ + path: Path + is_dir: bool + is_file: bool + size: int + + +def list_dir(root: Path) -> Iterator[Entry]: + """One directory level, as Entries. + + On Windows, DirEntry.is_dir/is_file/stat read from the FindNextFile + cache — zero per-entry syscalls. On Linux/macOS, one stat per file + (unavoidable for size; d_type doesn't include it). + """ + with os.scandir(root) as scan: + for e in scan: + is_dir = e.is_dir() + is_file = e.is_file() + size = e.stat().st_size if is_file else 0 + yield Entry(Path(e.path), is_dir=is_dir, is_file=is_file, size=size) + + +def walk(root: Path) -> Iterator[Entry]: + """Recursive file Entries under root. + + Non-file, non-dir entries (broken symlinks, sockets, ...) are skipped. + The pre-refactor walk raised ValueError on those; skipping is friendlier + and matches os.walk's default. + """ + if root.is_file(): + # Support walk(file) for callers like cp(file, ...) and + # hash_tree(file). Single-entry path, one stat. + st = root.stat() + yield Entry(root, is_dir=False, is_file=True, size=st.st_size) + return + for entry in list_dir(root): + if entry.is_file: + yield entry + elif entry.is_dir: + yield from walk(entry.path) + + def is_relevant(path: Path) -> bool: + # C3 flips this to take Entry; keeping the Path signature here because + # _compare_dirs still uses iterdir() at the C2 checkpoint. if not (path.is_file() or any(path.iterdir())): return False return True -def walk(path: Path, pbar: Optional[tqdm] = None) -> Generator[Path, None, None]: - if pbar: - pbar.update() - if path.is_file(): - yield path - elif path.is_dir(): - children = list(os.scandir(path)) - if not children: - return - for child in children: - for grandchild_path in walk(DirEntryPath(child), pbar): - yield grandchild_path - else: # pragma: no cover - raise ValueError(f"Unknown thing {dir}") - - def slurp(filename: Path, pbar: tqdm, chunk_size: int = CHUNK_SIZE) -> Generator[bytes, None, None]: """Returns generator for accessing file content chunk-by-chunk""" try: @@ -79,10 +117,9 @@ def hash_tree(path: Path, pbar: tqdm) -> Tuple[Union[Index, str], int]: if path.is_dir(): i = Index() size = 0 - for f in path.rglob("*"): - if f.is_file(): - i[f.relative_to(path)] = hash_file(f, pbar) - size += f.stat().st_size + for entry in walk(path): + i[entry.path.relative_to(path)] = hash_file(entry.path, pbar) + size += entry.size return i, size elif path.is_file(): return hash_file(path, pbar), path.stat().st_size @@ -93,11 +130,11 @@ def hash_tree(path: Path, pbar: tqdm) -> Tuple[Union[Index, str], int]: def cp(source: Path, target: Path, pbar: tqdm): assert os.path.exists(source), "can't copy, doesn't exist (internal error)" assert not os.path.exists(target), "remove explicitly first (internal error)" - for src_f in walk(source): - dst_f = target / src_f.relative_to(source) + for entry in walk(source): + dst_f = target / entry.path.relative_to(source) dst_f.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(src_f, dst_f) - pbar.update(src_f.stat().st_size) + shutil.copyfile(entry.path, dst_f) + pbar.update(entry.size) def rm(target: os.PathLike): From 12774f03fc0edd180416b4b54d70228d4119bf28 Mon Sep 17 00:00:00 2001 From: Ervin Oro Date: Sat, 18 Apr 2026 21:45:15 +0300 Subject: [PATCH 4/5] C3: comparison layer on list_dir; drop hot_dir/cold_dir from _compare_files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrites _compare_dirs to key children by name with Entry values — no more Set[PurePath] strip-and-rebuild, no more is_file()/is_dir() re-stat in walk_trees after listing. Matched pairs carry types and sizes straight from the scandir listing into the match branch. The file-vs-dir dispatch that walk_trees did today migrates into _compare_dirs' H=1,C=1 branch, where the Entries are already in hand. walk_trees becomes a thin wrapper preserving its public signature (so test_walk_trees is unchanged); it checks the root sub_index type once and delegates. Collision raises now fire symmetrically at each matched child: - FS file vs FS dir (on either side) → NotImplementedError - FS file/file but index has Index sub-tree → NotImplementedError (new) - FS dir/dir but index has hash string → NotImplementedError The second case was silently returning ModifiedCopied on every run, rewriting the index with a file hash and discarding the sub-tree record. test_compare_dirs_raises_when_index_is_index_but_fs_is_file was marked @expectedFailure in C0; un-marked here. _compare_files now takes two Entry values and reads hot.size directly — four avoidable stat()/getsize() call sites gone. hot_dir and cold_dir parameters drop out of its signature. is_relevant signature flipped from Path to Entry (one less is_file() per child — the scandir listing already told us). Empty-dir detection still costs one scandir, scoped to the edge case. Result: one os.scandir per visited directory per side. Zero extra type-check syscalls on the matched-child path. Zero stats in _compare_files. On Windows, zero per-file stat end-to-end on the matched path. 71 tests pass, 1 skipped (symlink-on-Windows). isort/flake8/mypy clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- dintact.py | 107 +++++++++++++++++++++------------------- test/test_walk_trees.py | 9 ++-- utils.py | 12 ++--- 3 files changed, 67 insertions(+), 61 deletions(-) diff --git a/dintact.py b/dintact.py index dc70853..cf26ba9 100644 --- a/dintact.py +++ b/dintact.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 import argparse -import os import sys from collections import defaultdict from operator import attrgetter @@ -14,8 +13,8 @@ ModifiedCorrupted, ModifiedLost, Moved, Removed, RemovedCorrupted, RemovedLost) from index import Index -from utils import (hash_compare_files, hash_file, hash_tree, is_relevant, walk, - yesno) +from utils import (Entry, hash_compare_files, hash_file, hash_tree, + is_relevant, list_dir, walk, yesno) # noinspection PyShadowingBuiltins print = tqdm.write @@ -54,8 +53,8 @@ def check(args: argparse.Namespace) -> None: print(f"FAIL: There were {fail_count} failures!") -def _compare_files(path: PurePath, cold_index: Index, hot_dir: Path, cold_dir: Path, pbar: tqdm) -> List[Change]: - hot_hash, cold_hash, eq = hash_compare_files(hot_dir / path, cold_dir / path, pbar) +def _compare_files(path: PurePath, hot: Entry, cold: Entry, cold_index: Index, pbar: tqdm) -> List[Change]: + hot_hash, cold_hash, eq = hash_compare_files(hot.path, cold.path, pbar) if eq: if path not in cold_index: return [AddedCopied(path, hot_hash)] @@ -65,57 +64,68 @@ def _compare_files(path: PurePath, cold_index: Index, hot_dir: Path, cold_dir: P return [] else: if path not in cold_index: - return [AddedAppeared(path, hot_hash, (hot_dir / path).stat().st_size)] + return [AddedAppeared(path, hot_hash, hot.size)] elif cold_index[path] != cold_hash: if hot_hash == cold_index[path]: - return [Corrupted(path, os.path.getsize(hot_dir / path))] + return [Corrupted(path, hot.size)] else: - return [ModifiedCorrupted(path, hot_hash, os.path.getsize(hot_dir / path))] + return [ModifiedCorrupted(path, hot_hash, hot.size)] else: - return [Modified(path, hot_hash, (hot_dir / path).stat().st_size)] + return [Modified(path, hot_hash, hot.size)] -def _compare_dirs(path: PurePath, cold_index: Index, hot_dir: Path, cold_dir: Path, sub_index: Index | None, +def _compare_dirs(rel: PurePath, cold_index: Index, hot_dir: Path, cold_dir: Path, sub_index: Index | None, pbar: tqdm) -> List[Change]: + hot: dict[PurePath, Entry] = { + PurePath(e.path.name): e for e in list_dir(hot_dir / rel) if is_relevant(e) + } + cold: dict[PurePath, Entry] = { + PurePath(e.path.name): e for e in list_dir(cold_dir / rel) if is_relevant(e) + } + indexed: Set[PurePath] = set(sub_index.iterdir()) if sub_index is not None else set() changes: List[Change] = [] - hot_children: Set[PurePath] = set(map(lambda abs_path: abs_path.relative_to(hot_dir), - filter(lambda p: is_relevant(p), - (hot_dir / path).iterdir()))) - cold_children: Set[PurePath] = set(map(lambda abs_path: abs_path.relative_to(cold_dir), - filter(lambda p: is_relevant(p), - (cold_dir / path).iterdir()))) - index_children: Set[PurePath] = set(map(lambda p: path / p, - sub_index.iterdir() if sub_index is not None else [])) - - # H C I: 1 0 X - for hot_child in hot_children.difference(cold_children): - i, size = hash_tree(hot_dir / hot_child, pbar) - if hot_child not in cold_index: - changes.append(Added(hot_child, i, size)) - elif i == cold_index[hot_child]: - changes.append(Lost(hot_child, size)) + # H=1, C=0 + for name in hot.keys() - cold.keys(): + child = rel / name + i, size = hash_tree(hot[name].path, pbar) + if child not in cold_index: + changes.append(Added(child, i, size)) + elif i == cold_index[child]: + changes.append(Lost(child, size)) else: - changes.append(ModifiedLost(hot_child, i, size)) - - # H C I: 0 1 X - for cold_child in cold_children.difference(hot_children): - if cold_child not in cold_index: - changes.append(Appeared(cold_child)) - pbar.update(sum(e.size for e in walk(cold_dir / cold_child))) - elif hash_tree(cold_dir / cold_child, pbar)[0] == cold_index[cold_child]: - changes.append(Removed(cold_child, cold_index[cold_child])) + changes.append(ModifiedLost(child, i, size)) + + # H=0, C=1 + for name in cold.keys() - hot.keys(): + child = rel / name + if child not in cold_index: + changes.append(Appeared(child)) + pbar.update(sum(e.size for e in walk(cold[name].path))) + elif hash_tree(cold[name].path, pbar)[0] == cold_index[child]: + changes.append(Removed(child, cold_index[child])) else: - changes.append(RemovedCorrupted(cold_child, cold_index[cold_child])) - - # H C I: 0 0 1 - for index_child in index_children.difference(hot_children).difference(cold_children): - changes.append(RemovedLost(index_child)) - - # Recursive: (H C I: 1 1 X) - for child in hot_children & cold_children: - ch_changes = walk_trees(child, cold_index, hot_dir, cold_dir, pbar) - changes.extend(ch_changes) + changes.append(RemovedCorrupted(child, cold_index[child])) + + # H=0, C=0, I=1 + for name in indexed - hot.keys() - cold.keys(): + changes.append(RemovedLost(rel / name)) + + # H=1, C=1 + for name in hot.keys() & cold.keys(): + h, c = hot[name], cold[name] + child = rel / name + sub = cold_index[child] if child in cold_index else None + if h.is_file and c.is_file: + if isinstance(sub, Index): + raise NotImplementedError("File/Folder name collision") + changes.extend(_compare_files(child, h, c, cold_index, pbar)) + elif h.is_dir and c.is_dir: + if isinstance(sub, str): + raise NotImplementedError("File/Folder name collision") + changes.extend(_compare_dirs(child, cold_index, hot_dir, cold_dir, sub, pbar)) + else: + raise NotImplementedError("File/Folder name collision") return changes @@ -134,12 +144,9 @@ def walk_trees(path: PurePath, cold_index: Index, hot_dir: Path, cold_dir: Path, """ sub_index = cold_index[path] if path in cold_index else None - if (hot_dir / path).is_file() and (cold_dir / path).is_file(): - return _compare_files(path, cold_index, hot_dir, cold_dir, pbar) - elif not (hot_dir / path).is_dir() or not (cold_dir / path).is_dir() or isinstance(sub_index, str): + if isinstance(sub_index, str): raise NotImplementedError("File/Folder name collision") - else: - return _compare_dirs(path, cold_index, hot_dir, cold_dir, sub_index, pbar) + return _compare_dirs(path, cold_index, hot_dir, cold_dir, sub_index, pbar) def find_moveds(changes: list[Change]): diff --git a/test/test_walk_trees.py b/test/test_walk_trees.py index 1372ae4..ada01b3 100644 --- a/test/test_walk_trees.py +++ b/test/test_walk_trees.py @@ -81,12 +81,11 @@ def test_compare_dirs_raises_on_fs_file_vs_fs_dir(self): with self.assertRaises(NotImplementedError): walk_trees(PurePath(), index, hot, cold, mock.MagicMock()) - @unittest.expectedFailure def test_compare_dirs_raises_when_index_is_index_but_fs_is_file(self): - # Un-marked in C3, which adds the symmetric isinstance check in the - # matched-child branch of _compare_dirs. On current code, this - # scenario silently returns ModifiedCopied, rewriting the index - # with a file hash and discarding the sub-tree record. + # C3 added the symmetric isinstance check in the matched-child branch + # of _compare_dirs. Pre-C3, this scenario silently returned + # ModifiedCopied, rewriting the index with a file hash and discarding + # the sub-tree record. with tempfile.TemporaryDirectory() as tmp: hot, cold = self._setup(Path(tmp)) (hot / 'x').write_bytes(b'same') diff --git a/utils.py b/utils.py index 439d5c4..96bcbc8 100644 --- a/utils.py +++ b/utils.py @@ -69,12 +69,12 @@ def walk(root: Path) -> Iterator[Entry]: yield from walk(entry.path) -def is_relevant(path: Path) -> bool: - # C3 flips this to take Entry; keeping the Path signature here because - # _compare_dirs still uses iterdir() at the C2 checkpoint. - if not (path.is_file() or any(path.iterdir())): - return False - return True +def is_relevant(entry: Entry) -> bool: + if entry.is_file: + return True + if entry.is_dir: + return any(entry.path.iterdir()) # empty dir → filter out + return False # broken symlink, socket, etc. def slurp(filename: Path, pbar: tqdm, chunk_size: int = CHUNK_SIZE) -> Generator[bytes, None, None]: From 358e189cc03689489a484888109619609dcf150c Mon Sep 17 00:00:00 2001 From: Ervin Oro Date: Sat, 18 Apr 2026 21:46:03 +0300 Subject: [PATCH 5/5] C4: single-pass check; explicit missing-from-disk message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrites check to walk cold_dir exactly once, materializing {rel: entry}. Progress-bar total derives from that walk instead of a per-index stat loop with a double-stat-via-exists() guard. Index-completeness check uses dict set subtraction against the same dict — no second walk. Behavior change: files indexed-but-missing-from-disk now produce an explicit "File missing from disk: ''" failure line rather than surfacing only as "Unable to open ''" from slurp(). The latter was accidental — slurp() swallows IOError and emits the message as a side-effect of hash attempts against missing files. Now: - missing from disk → "File missing from disk: '

'" FAIL - hash mismatch → "Verification failed: '

'" FAIL - missing from idx → "File missing from index: '

'" FAIL test_check_reports_file_missing_from_disk (added in C0 asserting the "Unable to open" message) flipped to assert the new message, per plan. Syscalls on Windows: one scandir per directory, period. The double-stat via exists() in the old pbar-total computation, and the separate walk for extras detection, are both gone. 71 tests pass, 1 skipped. isort/flake8/mypy clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- dintact.py | 31 ++++++++++++++++++++----------- test/test_check.py | 5 +---- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/dintact.py b/dintact.py index cf26ba9..eaee29f 100644 --- a/dintact.py +++ b/dintact.py @@ -30,22 +30,31 @@ def check(args: argparse.Namespace) -> None: index = Index(cold_dir) fail_count = 0 - # Set up progress bar - total = sum([(cold_dir / p).stat().st_size if (cold_dir / p).exists() else 0 for p in index.keys()]) + # Single scandir-walk of cold_dir yields both the progress-bar total + # (sum of file sizes) and the set of present files (for extras-detection + # and missing-from-disk detection). + index_filename = PurePath(Index.FILENAME) + on_disk: dict[PurePath, Entry] = {} + total = 0 + for entry in walk(cold_dir): + rel = PurePath(entry.path.relative_to(cold_dir)) + if rel == index_filename: + continue + on_disk[rel] = entry + total += entry.size + with tqdm(total=total, unit="B", unit_scale=True) as pbar: - # Firstly, check that the index is correct for p, h in index.items(): - if h != hash_file(cold_dir / p, pbar): - print(f"Verification failed: '{p}'.", file=sys.stderr) + if p not in on_disk: + print(f"File missing from disk: '{p}'.", file=sys.stderr) fail_count += 1 - # Secondly, check that the index is complete - for entry in walk(cold_dir): - rel_path: PurePath = entry.path.relative_to(cold_dir) - if rel_path == PurePath(Index.FILENAME): continue - if rel_path not in index: - print(f"File missing from index: '{rel_path}'.", file=sys.stderr) + if h != hash_file(on_disk[p].path, pbar): + print(f"Verification failed: '{p}'.", file=sys.stderr) fail_count += 1 + for rel in on_disk.keys() - index.keys(): + print(f"File missing from index: '{rel}'.", file=sys.stderr) + fail_count += 1 if fail_count == 0: print("OK: Data is intact!") diff --git a/test/test_check.py b/test/test_check.py index 1618b26..49b9b99 100644 --- a/test/test_check.py +++ b/test/test_check.py @@ -70,14 +70,11 @@ def test_check_reports_file_missing_from_index(self): self.assertIn("FAIL", out) def test_check_reports_file_missing_from_disk(self): - # Pre-C4 assertion: "Unable to open" lands on stderr via slurp(). - # C4 flips this to an explicit "File missing from disk: '...'" line. with tempfile.TemporaryDirectory() as tmp: cold = Path(tmp) _write_index(cold, {'gone.txt': _xxh(b'x')}) out, err = _run_check(cold) - self.assertIn("Unable to open", err) - self.assertIn("gone.txt", err) + self.assertIn("File missing from disk: 'gone.txt'", err) self.assertIn("FAIL", out)