Optimize stat

.github/workflows/lint-and-test.yml

@@ -15,11 +15,11 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [ '3.13' ]
+        python-version: [ '3.14' ]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

README.md

@@ -13,7 +13,7 @@ It assumes that you have a hot version of a file archive (on your pc maybe), and
 
 ## Requirements
 
-Depends on Python 3.8+ and some pip libraries.
+Depends on Python 3.14+ and some pip libraries.
 
 `pip install -r requirements.txt`
 
@@ -27,10 +27,6 @@ The entire index must fit in memory, but files are read chunk-by-chunk.
 
 `dintact sync <hot_directory> <cold_directory>`
 
-### Ignored files
-
-dintact respects any `.gitignore` files it finds, and does **not** back up files matched by these.
-
 ### Testing
 
 `python -m unittest discover test`

dintact.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 import argparse
-import os
 import sys
 from collections import defaultdict
 from operator import attrgetter
@@ -14,8 +13,8 @@
                      ModifiedCorrupted, ModifiedLost, Moved, Removed,
                      RemovedCorrupted, RemovedLost)
 from index import Index
-from utils import (hash_compare_files, hash_file, hash_tree, is_relevant, walk,
-                   yesno)
+from utils import (Entry, hash_compare_files, hash_file, hash_tree,
+                   is_relevant, list_dir, walk, yesno)
 
 # noinspection PyShadowingBuiltins
 print = tqdm.write
@@ -26,34 +25,45 @@ def check(args: argparse.Namespace) -> None:
 
     :param args: must have attr cold_dir: str
     """
-    cold_dir = Path(args.cold_dir)
+    cold_dir = Path(args.cold_dir).absolute()
     assert cold_dir.is_dir(), "cold_dir not found!"
     index = Index(cold_dir)
     fail_count = 0
 
-    # Set up progress bar
-    total = sum([(cold_dir / p).stat().st_size if (cold_dir / p).exists() else 0 for p in index.keys()])
+    # Single scandir-walk of cold_dir yields both the progress-bar total
+    # (sum of file sizes) and the set of present files (for extras-detection
+    # and missing-from-disk detection).
+    index_filename = PurePath(Index.FILENAME)
+    on_disk: dict[PurePath, Entry] = {}
+    total = 0
+    for entry in walk(cold_dir):
+        rel = PurePath(entry.path.relative_to(cold_dir))
+        if rel == index_filename:
+            continue
+        on_disk[rel] = entry
+        total += entry.size
+
     with tqdm(total=total, unit="B", unit_scale=True) as pbar:
-        # Firstly, check that the index is correct
         for p, h in index.items():
-            if h != hash_file(cold_dir / p, pbar):
-                print(f"Verification failed: '{p}'.", file=sys.stderr)
+            if p not in on_disk:
+                print(f"File missing from disk: '{p}'.", file=sys.stderr)
                 fail_count += 1
-        # Secondly, check that the index is complete
-        for file in walk(cold_dir):
-            rel_path: PurePath = file.relative_to(cold_dir)
-            if rel_path not in index:
-                print(f"File missing from index: '{rel_path}'.", file=sys.stderr)
+                continue
+            if h != hash_file(on_disk[p].path, pbar):
+                print(f"Verification failed: '{p}'.", file=sys.stderr)
                 fail_count += 1
+        for rel in on_disk.keys() - index.keys():
+            print(f"File missing from index: '{rel}'.", file=sys.stderr)
+            fail_count += 1
 
     if fail_count == 0:
         print("OK: Data is intact!")
     else:
         print(f"FAIL: There were {fail_count} failures!")
 
 
-def _compare_files(path: PurePath, cold_index: Index, hot_dir: Path, cold_dir: Path, pbar: tqdm) -> List[Change]:
-    hot_hash, cold_hash, eq = hash_compare_files(hot_dir / path, cold_dir / path, pbar)
+def _compare_files(path: PurePath, hot: Entry, cold: Entry, cold_index: Index, pbar: tqdm) -> List[Change]:
+    hot_hash, cold_hash, eq = hash_compare_files(hot.path, cold.path, pbar)
     if eq:
         if path not in cold_index:
             return [AddedCopied(path, hot_hash)]
@@ -63,57 +73,68 @@ def _compare_files(path: PurePath, cold_index: Index, hot_dir: Path, cold_dir: P
             return []
     else:
         if path not in cold_index:
-            return [AddedAppeared(path, hot_hash, (hot_dir / path).stat().st_size)]
+            return [AddedAppeared(path, hot_hash, hot.size)]
         elif cold_index[path] != cold_hash:
             if hot_hash == cold_index[path]:
-                return [Corrupted(path, os.path.getsize(hot_dir / path))]
+                return [Corrupted(path, hot.size)]
             else:
-                return [ModifiedCorrupted(path, hot_hash, os.path.getsize(hot_dir / path))]
+                return [ModifiedCorrupted(path, hot_hash, hot.size)]
         else:
-            return [Modified(path, hot_hash, (hot_dir / path).stat().st_size)]
+            return [Modified(path, hot_hash, hot.size)]
 
 
-def _compare_dirs(path: PurePath, cold_index: Index, hot_dir: Path, cold_dir: Path, sub_index: Index | None,
+def _compare_dirs(rel: PurePath, cold_index: Index, hot_dir: Path, cold_dir: Path, sub_index: Index | None,
                   pbar: tqdm) -> List[Change]:
+    hot: dict[PurePath, Entry] = {
+        PurePath(e.path.name): e for e in list_dir(hot_dir / rel) if is_relevant(e)
+    }
+    cold: dict[PurePath, Entry] = {
+        PurePath(e.path.name): e for e in list_dir(cold_dir / rel) if is_relevant(e)
+    }
+    indexed: Set[PurePath] = set(sub_index.iterdir()) if sub_index is not None else set()
     changes: List[Change] = []
 
-    hot_children: Set[PurePath] = set(map(lambda abs_path: abs_path.relative_to(hot_dir),
-                                          filter(lambda p: is_relevant(p),
-                                                 (hot_dir / path).iterdir())))
-    cold_children: Set[PurePath] = set(map(lambda abs_path: abs_path.relative_to(cold_dir),
-                                           filter(lambda p: is_relevant(p),
-                                                  (cold_dir / path).iterdir())))
-    index_children: Set[PurePath] = set(map(lambda p: path / p,
-                                            sub_index.iterdir() if sub_index is not None else []))
-
-    # H C I: 1 0 X
-    for hot_child in hot_children.difference(cold_children):
-        i, size = hash_tree(hot_dir / hot_child, pbar)
-        if hot_child not in cold_index:
-            changes.append(Added(hot_child, i, size))
-        elif i == cold_index[hot_child]:
-            changes.append(Lost(hot_child, size))
+    # H=1, C=0
+    for name in hot.keys() - cold.keys():
+        child = rel / name
+        i, size = hash_tree(hot[name].path, pbar)
+        if child not in cold_index:
+            changes.append(Added(child, i, size))
+        elif i == cold_index[child]:
+            changes.append(Lost(child, size))
         else:
-            changes.append(ModifiedLost(hot_child, i, size))
-
-    # H C I: 0 1 X
-    for cold_child in cold_children.difference(hot_children):
-        if cold_child not in cold_index:
-            changes.append(Appeared(cold_child))
-            pbar.update(sum(file.stat().st_size for file in walk(cold_dir / cold_child)))
-        elif hash_tree(cold_dir / cold_child, pbar)[0] == cold_index[cold_child]:
-            changes.append(Removed(cold_child, cold_index[cold_child]))
+            changes.append(ModifiedLost(child, i, size))
+
+    # H=0, C=1
+    for name in cold.keys() - hot.keys():
+        child = rel / name
+        if child not in cold_index:
+            changes.append(Appeared(child))
+            pbar.update(sum(e.size for e in walk(cold[name].path)))
+        elif hash_tree(cold[name].path, pbar)[0] == cold_index[child]:
+            changes.append(Removed(child, cold_index[child]))
         else:
-            changes.append(RemovedCorrupted(cold_child, cold_index[cold_child]))
-
-    # H C I: 0 0 1
-    for index_child in index_children.difference(hot_children).difference(cold_children):
-        changes.append(RemovedLost(index_child))
-
-    # Recursive: (H C I: 1 1 X)
-    for child in hot_children & cold_children:
-        ch_changes = walk_trees(child, cold_index, hot_dir, cold_dir, pbar)
-        changes.extend(ch_changes)
+            changes.append(RemovedCorrupted(child, cold_index[child]))
+
+    # H=0, C=0, I=1
+    for name in indexed - hot.keys() - cold.keys():
+        changes.append(RemovedLost(rel / name))
+
+    # H=1, C=1
+    for name in hot.keys() & cold.keys():
+        h, c = hot[name], cold[name]
+        child = rel / name
+        sub = cold_index[child] if child in cold_index else None
+        if h.is_file and c.is_file:
+            if isinstance(sub, Index):
+                raise NotImplementedError("File/Folder name collision")
+            changes.extend(_compare_files(child, h, c, cold_index, pbar))
+        elif h.is_dir and c.is_dir:
+            if isinstance(sub, str):
+                raise NotImplementedError("File/Folder name collision")
+            changes.extend(_compare_dirs(child, cold_index, hot_dir, cold_dir, sub, pbar))
+        else:
+            raise NotImplementedError("File/Folder name collision")
 
     return changes
 
@@ -132,12 +153,9 @@ def walk_trees(path: PurePath, cold_index: Index, hot_dir: Path, cold_dir: Path,
     """
 
     sub_index = cold_index[path] if path in cold_index else None
-    if (hot_dir / path).is_file() and (cold_dir / path).is_file():
-        return _compare_files(path, cold_index, hot_dir, cold_dir, pbar)
-    elif not (hot_dir / path).is_dir() or not (cold_dir / path).is_dir() or isinstance(sub_index, str):
+    if isinstance(sub_index, str):
         raise NotImplementedError("File/Folder name collision")
-    else:
-        return _compare_dirs(path, cold_index, hot_dir, cold_dir, sub_index, pbar)
+    return _compare_dirs(path, cold_index, hot_dir, cold_dir, sub_index, pbar)
 
 
 def find_moveds(changes: list[Change]):
@@ -178,24 +196,19 @@ def sync(args: argparse.Namespace) -> None:
 
     :param args: must have attrs hot_dir:str and cold_dir: str
     """
-    hot_dir, cold_dir = Path(args.hot_dir), Path(args.cold_dir)
+    hot_dir, cold_dir = Path(args.hot_dir).absolute(), Path(args.cold_dir).absolute()
     assert hot_dir.is_dir(), "hot_dir not found!"
     assert cold_dir.is_dir(), "cold_dir not found!"
 
     index = Index(cold_dir)
 
-    # Set up progress bar
-    file_count = 0
-    for _, dirs, files in os.walk(hot_dir):
-        file_count += len(dirs) + len(files)
-    for _, dirs, files in os.walk(cold_dir):
-        file_count += len(dirs) + len(files)
+    # Set up progress bar — one walk per side, size comes from Entry directly
     total = 0
-    with tqdm(total=file_count, unit_scale=True, desc="Calculating data size") as pbar:
-        for file in walk(hot_dir, pbar):
-            total += file.stat().st_size
-        for file in walk(cold_dir, pbar):
-            total += file.stat().st_size
+    with tqdm(unit_scale=True, desc="Calculating data size") as pbar:
+        for side in (hot_dir, cold_dir):
+            for entry in walk(side):
+                total += entry.size
+                pbar.update()
     with tqdm(total=total, unit="B", unit_scale=True, desc="Detecting changes") as pbar:
         # Find all changes required
         changes = walk_trees(PurePath(), index, hot_dir, cold_dir, pbar)

test/test_check.py

@@ -0,0 +1,82 @@
+import json
+import tempfile
+import unittest
+from argparse import Namespace
+from contextlib import redirect_stderr, redirect_stdout
+from io import StringIO
+from pathlib import Path
+from unittest import TestCase
+
+import xxhash
+
+from dintact import check
+from index import Index
+
+
+def _xxh(data: bytes) -> str:
+    return xxhash.xxh3_128(data).hexdigest()
+
+
+def _write_index(cold_dir: Path, entries: dict) -> None:
+    meta = {"version": 1, "algorithm": "XXH128", "coding": "utf8"}
+    with (cold_dir / Index.FILENAME).open('w', encoding='utf8') as f:
+        f.write(f"# dintact index {json.dumps(meta)}\n")
+        for name, h in entries.items():
+            f.write(f"{h}  {name}\n")
+
+
+def _write_files(cold_dir: Path, files: dict) -> None:
+    for name, content in files.items():
+        p = cold_dir / name
+        p.parent.mkdir(parents=True, exist_ok=True)
+        p.write_bytes(content)
+
+
+def _run_check(cold_dir: Path):
+    out, err = StringIO(), StringIO()
+    with redirect_stdout(out), redirect_stderr(err):
+        check(Namespace(cold_dir=str(cold_dir)))
+    return out.getvalue(), err.getvalue()
+
+
+class TestCheck(TestCase):
+    def test_check_reports_ok_when_everything_matches(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            cold = Path(tmp)
+            _write_files(cold, {'a.txt': b'hello'})
+            _write_index(cold, {'a.txt': _xxh(b'hello')})
+            out, err = _run_check(cold)
+            self.assertIn("OK: Data is intact!", out)
+            self.assertNotIn("FAIL", out)
+            self.assertNotIn("Verification failed", err)
+            self.assertNotIn("missing", err)
+
+    def test_check_reports_hash_mismatch(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            cold = Path(tmp)
+            _write_files(cold, {'a.txt': b'real content'})
+            _write_index(cold, {'a.txt': _xxh(b'different content')})
+            out, err = _run_check(cold)
+            self.assertIn("Verification failed: 'a.txt'", err)
+            self.assertIn("FAIL", out)
+
+    def test_check_reports_file_missing_from_index(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            cold = Path(tmp)
+            _write_files(cold, {'a.txt': b'a', 'b.txt': b'b'})
+            _write_index(cold, {'a.txt': _xxh(b'a')})
+            out, err = _run_check(cold)
+            self.assertIn("File missing from index: 'b.txt'", err)
+            self.assertIn("FAIL", out)
+
+    def test_check_reports_file_missing_from_disk(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            cold = Path(tmp)
+            _write_index(cold, {'gone.txt': _xxh(b'x')})
+            out, err = _run_check(cold)
+            self.assertIn("File missing from disk: 'gone.txt'", err)
+            self.assertIn("FAIL", out)
+
+
+if __name__ == "__main__":
+    unittest.main()