From 6d9c6d572981fc09d85f3f9d0dcf43fcc86aa6f7 Mon Sep 17 00:00:00 2001 From: Alex Rubinsteyn Date: Tue, 29 Sep 2026 11:32:56 -0400 Subject: [PATCH] Share fixed-path versioned registry and serialize manifest updates --- CHANGELOG.md | 8 ++ README.md | 1 + datacache/__init__.py | 2 + datacache/_filesystem.py | 5 +- datacache/file_registry.py | 169 ++++++++++++++++++++++++++ datacache/version.py | 2 +- docs/api.md | 21 +++- docs/bundles.md | 5 +- docs/file_registry.md | 63 ++++++++++ requirements.txt | 1 + tasks/fixed_path_registry.md | 44 +++++++ tasks/todo.md | 4 + tests/test_file_registry.py | 225 +++++++++++++++++++++++++++++++++++ 13 files changed, 546 insertions(+), 4 deletions(-) create mode 100644 datacache/file_registry.py create mode 100644 docs/file_registry.md create mode 100644 tasks/fixed_path_registry.md create mode 100644 tests/test_file_registry.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a82177..9dc509f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 1.15.0 + +- Add `VersionedFileRegistry` for established single-file caches with fixed + version paths and root provenance manifests. Legacy files are reused offline + without relocation; transfers use the shared downloader, and new receipts + use bounded-memory hashing and atomic JSON publication (#83). Writers serialize + per root to prevent the legacy registry’s lost manifest-update race. + ## 1.14.0 - Allow `resume=True` with `expected_size` alone when the server supplies a diff --git a/README.md b/README.md index ebd201d..3d94b46 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,7 @@ lists every public signature, default, return value, exception, and example. | Task | API | Result | | --- | --- | --- | | Install or reuse a versioned dataset | `VersionedDatasetRegistry`, `install_bundle(...)` | Mapping of asset names to snapshot paths | +| Reuse an established fixed-path versioned file cache | `VersionedFileRegistry` | One Path and a legacy-compatible root receipt | | Inspect a complete dataset generation | `inspect_bundle(...)` | `BundleInspection` | | Discard retained partial download bytes | `discard_partial(destination)` | Installed file unchanged | | Download or reuse one file | `fetch_file(...)`, `Cache.fetch(...)` | Local path string | diff --git a/datacache/__init__.py b/datacache/__init__.py index 2f5e87e..300f5c0 100644 --- a/datacache/__init__.py +++ b/datacache/__init__.py @@ -33,6 +33,7 @@ from .cache import Cache from .resume import discard_partial from .bundles import BundleInspection, VersionedDatasetRegistry, inspect_bundle, install_bundle +from .file_registry import VersionedFileRegistry from .version import __version__ __all__ = [ @@ -41,6 +42,7 @@ 'discard_partial', 'BundleInspection', 'VersionedDatasetRegistry', + 'VersionedFileRegistry', 'inspect_bundle', 'install_bundle', 'expected_path', diff --git a/datacache/_filesystem.py b/datacache/_filesystem.py index 968e584..38675ef 100644 --- a/datacache/_filesystem.py +++ b/datacache/_filesystem.py @@ -74,7 +74,10 @@ def write_json(path, value, mode=0o600): handle.write('\n') handle.flush() os.fsync(handle.fileno()) - os.fchmod(handle.fileno(), mode) + if hasattr(os, 'fchmod'): + os.fchmod(handle.fileno(), mode) + else: # Windows: no descriptor chmod; staging is still private. + os.chmod(temporary, mode) os.replace(temporary, path) finally: Path(temporary).unlink(missing_ok=True) diff --git a/datacache/file_registry.py b/datacache/file_registry.py new file mode 100644 index 0000000..1ef5438 --- /dev/null +++ b/datacache/file_registry.py @@ -0,0 +1,169 @@ +"""Fixed-path single-file registry for applications with established caches. + +Unlike generation bundles, this preserves legacy files and a root manifest. +The caller owns trusted dataset definitions; writers serialize per root. Use bundles for transactional multi-file data. +""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path + +from filelock import FileLock + +from ._filesystem import write_json +from .download import fetch_file + + +class VersionedFileRegistry: + """Download + cache for versioned, version-pinned external datasets. + + Parameters + ---------- + datasets + Mapping of ``name -> spec`` where each spec has:: + + { + "filename": "local_name.tsv", # name on disk (post-decompress) + "urls": {"v23": "https://...zip", "latest": "https://..."}, + "default_version": "v23", # used when caller passes version=None + "description": "...", # optional, for status() + } + + cache_dir + Zero-arg callable returning the cache root :class:`~pathlib.Path` + (created on demand by the caller). The on-disk layout is + ``///`` plus a ``/manifest.json`` + provenance file. + error_cls + Exception type raised for unknown datasets/versions and download + failures. Defaults to :class:`RuntimeError`; consumers may pass + their own subclass to preserve their public error type. + """ + + def __init__(self, datasets, *, cache_dir, error_cls=RuntimeError): + self._datasets = datasets + self._cache_dir = cache_dir + self._error_cls = error_cls + + # -- dataset / version resolution -- + + def _dataset(self, name: str) -> dict: + try: + return self._datasets[name] + except KeyError: + known = ", ".join(sorted(self._datasets)) + raise self._error_cls(f"unknown dataset {name!r}; known: {known}") from None + + def resolve_version(self, name: str, version: str | None = None) -> str: + """Return the concrete version for *name*, applying its default.""" + spec = self._dataset(name) + if version is None: + version = spec["default_version"] + if version not in spec["urls"]: + avail = ", ".join(sorted(spec["urls"])) + raise self._error_cls(f"{name!r} has no version {version!r}; available: {avail}") + return version + + # -- cache paths / manifest -- + + def _manifest_path(self) -> Path: + return Path(self._cache_dir()) / "manifest.json" + + @staticmethod + def _read_manifest_at(path: Path) -> dict: + try: + manifest = json.loads(path.read_text()) + return manifest if isinstance(manifest, dict) else {} + except (json.JSONDecodeError, OSError): + return {} + + def local_path(self, name: str, version: str | None = None) -> Path: + """Expected cache path for *name*/*version* (may not exist yet).""" + version = self.resolve_version(name, version) + spec = self._dataset(name) + return Path(self._cache_dir()) / name / version / spec["filename"] + + def is_cached(self, name: str, version: str | None = None) -> bool: + return self.local_path(name, version).exists() + + # -- fetch -- + + def download( + self, name: str, version: str | None = None, *, force: bool = False, **download_options + ) -> Path: + """Fetch a fixed path, serializing writers and forwarding fetch_file options. + + Ordinary cache hits neither hash nor write. Explicit integrity options + validate cache hits. New bytes and the root receipt publish separately. + """ + version = self.resolve_version(name, version) + spec = self._dataset(name) + root = Path(self._cache_dir()) + dest = root / name / version / spec["filename"] + url = spec["urls"][version] + + def reuse(): + if force or not dest.exists(): + return False + if any(download_options.get(key) is not None + for key in ('expected_sha256', 'expected_size')): + acquire() + return True + + def acquire(): + try: + fetch_file(url, destination=dest, force=force, **download_options) + except Exception as error: + raise self._error_cls(f"failed to download {name} ({url}): {error}") from error + + if reuse(): + return dest + root.mkdir(parents=True, exist_ok=True) + with FileLock(str(root / '.datacache-file-registry.lock')): + if reuse(): + return dest + acquire() + digest = hashlib.sha256() + with dest.open("rb") as handle: + for chunk in iter(lambda: handle.read(2 ** 20), b""): + digest.update(chunk) + manifest_path = root / 'manifest.json' + manifest = self._read_manifest_at(manifest_path) + manifest[name] = { + "version": version, "url": url, "path": str(dest), + "bytes": dest.stat().st_size, "sha256": digest.hexdigest(), + "downloaded_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + } + write_json(manifest_path, manifest) + return dest + + def ensure(self, name: str, version: str | None = None, **download_options) -> Path: + """Return a local path to *name*/*version*, downloading if absent.""" + return self.download(name, version, **download_options) + + def status(self) -> list[dict]: + """Return one status row per dataset (for a ``... list`` CLI command).""" + root = Path(self._cache_dir()) + manifest = self._read_manifest_at(root / "manifest.json") + rows = [] + for name, spec in sorted(self._datasets.items()): + default_v = spec["default_version"] + path = root / name / default_v / spec["filename"] + record = manifest.get(name, {}) + rows.append( + { + "name": name, + "description": spec.get("description", ""), + "default_version": default_v, + "available_versions": sorted(spec["urls"]), + "cached": path.exists(), + "cached_version": record.get("version") if record else None, + "bytes": record.get("bytes") if path.exists() else None, + "downloaded_at": record.get("downloaded_at") if path.exists() else None, + "path": str(path), + } + ) + return rows diff --git a/datacache/version.py b/datacache/version.py index e4f2ad4..6b0872c 100644 --- a/datacache/version.py +++ b/datacache/version.py @@ -1 +1 @@ -__version__ = '1.14.0' +__version__ = "1.15.0" diff --git a/docs/api.md b/docs/api.md index 770129e..b336261 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,7 +1,7 @@ # Public API reference This reference covers every name exported in `datacache.__all__` and every -public `Cache` method in DataCache 1.14.0. Import these names from `datacache`. +public `Cache` method in DataCache 1.15.0. Import these names from `datacache`. Signatures below show all defaults; arguments after `*` are keyword-only. Method signatures omit `self` and are called on a `Cache` instance. @@ -1132,3 +1132,22 @@ The [bundle guide](bundles.md) includes an example and downstream migration note | `ensure(name, version=None, **download_options)` | Download/reuse, then return `local_path`. | | `is_cached(name, version=None)` | Whether verified inspection reports `available`. | | `status()` | One row per dataset's pinned default: name, version, description, available_versions and inspection. | + +## VersionedFileRegistry + +`VersionedFileRegistry(datasets, *, cache_dir, error_cls=RuntimeError)` + +Fixed-path single-file compatibility registry. Definitions contain `filename`, +`urls` (version to URL), `default_version`, and optional `description`. The +zero-argument root callable may return a string or Path. Construction creates +nothing. See the [fixed-path guide](file_registry.md) for legacy receipt semantics, +trust boundaries, and the distinction from transactional generation bundles. + +| Method | Result | +| --- | --- | +| `resolve_version(name, version=None)` | Concrete version; unknown names/versions raise the caller's error_cls. | +| `local_path(name, version=None)` | Expected fixed Path, even when absent, without writes. | +| `is_cached(name, version=None)` | Presence only, not verified integrity. | +| `download(name, version=None, *, force=False, **download_options)` | One fixed Path; fetch_file options control acquisition. Ordinary cache hits do not hash or write; explicit size/hash expectations are checked. | +| `ensure(name, version=None, **download_options)` | Same download/reuse behavior and Path result. | +| `status()` | Legacy status dicts with name, description, default_version, available_versions, cached, cached_version, bytes, downloaded_at and path. | diff --git a/docs/bundles.md b/docs/bundles.md index 435bdd5..a0eda25 100644 --- a/docs/bundles.md +++ b/docs/bundles.md @@ -158,7 +158,10 @@ progress, disk-space and partial-discard details. ## Adopting from downstream libraries -- **hitlist / tsarina:** the existing `{filename, urls, default_version}` mapping +- **hitlist / tsarina:** use [VersionedFileRegistry](file_registry.md) to retain + fixed paths, single-Path returns and legacy root manifests without moving old + caches. For a deliberate migration to generation bundles, the existing + `{filename, urls, default_version}` mapping is accepted with `verified=False`, as is the `cache_dir` root callable. This is mapping compatibility, not a drop-in filesystem or return-value migration: `download` returns asset paths, `local_path` requires an installed bundle, and diff --git a/docs/file_registry.md b/docs/file_registry.md new file mode 100644 index 0000000..6258a3a --- /dev/null +++ b/docs/file_registry.md @@ -0,0 +1,63 @@ +# Fixed-path versioned files + +`VersionedFileRegistry` supports applications whose single-file datasets already +live at `///` with a root `manifest.json`. +It reuses those files in place. Use `VersionedDatasetRegistry` and bundles when +you need immutable snapshots and atomic multi-file installation. + +```python +from pathlib import Path +from datacache import VersionedFileRegistry + +registry = VersionedFileRegistry({ + "reference": { + "filename": "records.tsv", + "default_version": "2026-09", + "urls": {"2026-09": "https://data.example.org/2026-09/records.tsv.gz"}, + "description": "Reference records", + }, +}, cache_dir=lambda: Path("existing-cache")) + +expected = registry.local_path("reference") # works before installation; no writes +status = registry.status() # presence and legacy receipt, offline +path = registry.ensure("reference", timeout=60, record_provenance=True) +``` + +The dataset mapping and cache-root callable remain application-owned. The root +is resolved for each operation, so environment-backed callables may change it. +Use trusted definitions. Writers serialize per root using a cross-platform +file lock; cache hits and inspection never acquire or create that lock. Dataset names, +version labels and filenames must describe paths within the selected root. + +Ordinary cache hits check presence and reuse the old path without reading its +bytes, networking, hashing or rewriting receipts. This deliberately preserves +legacy behavior and is not integrity verification. Explicit `expected_sha256` +or `expected_size` options validate reused files through `fetch_file`; invalid +files require `force=True`. `ensure` accepts the same options. New acquisition +forwards `fetch_file` options, including raw/decompression, timeout, bounded +retries, progress, provenance and resume. Integrity expectations describe the +installed bytes. Human cache-status messages belong in the calling application. + +After downloading, the registry hashes the installed file in bounded chunks +and atomically updates its root JSON receipt. Entries are keyed by dataset name +and retain `version`, `url`, `path`, `bytes`, `sha256` and `downloaded_at` (UTC). +The URL is the caller's original URL for legacy compatibility; avoid credentials +or signed URLs in these definitions. Local receipts describe observed bytes, +not independently trusted checksums. Missing, unreadable or malformed JSON +receipts are treated as empty, matching the legacy registry. + +`status()` returns `name`, `description`, `default_version`, `available_versions`, +`cached`, `cached_version`, `bytes`, `downloaded_at` and `path`. Presence and path +refer to the pinned default. Receipt fields describe the most recent download +for that dataset, which may name another version. This compatibility view does +not certify freshness or integrity; use `inspect_file` for the selected path. + +Unknown dataset/version errors and acquisition failures use `error_cls` +(`RuntimeError` by default). Acquisition failures retain their original cause. +Filesystem failures while hashing or writing the receipt propagate directly. +Failed acquisition leaves the previous file and receipt intact. The file and +receipt are separate publications: if receipt publication fails after a valid +download, the new file remains installed and the prior receipt remains intact. +Concurrent registry writers serialize acquisition and receipt updates, so +downloading different datasets cannot discard each other’s receipts. These are the +established single-file semantics, not the guarantees of generation bundles. diff --git a/requirements.txt b/requirements.txt index 210a747..c276f54 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ pandas>=0.15.2 appdirs>=1.4.0 requests>=2.5.1 typechecks>=0.0.2 +filelock>=3.13 diff --git a/tasks/fixed_path_registry.md b/tasks/fixed_path_registry.md new file mode 100644 index 0000000..83d7e56 --- /dev/null +++ b/tasks/fixed_path_registry.md @@ -0,0 +1,44 @@ +# #83: fixed-path single-file registry + +Add VersionedFileRegistry as the shared implementation of hitlist's established +single-file registry. Keep VersionedDatasetRegistry's generation storage +unchanged. The separate type makes the weaker single-file publication contract +explicit rather than making bundles silently use a second layout. + +Contracts: accept the filename/urls/default_version/description mapping and a +dynamic cache_dir callable; local_path returns the old fixed Path even before +installation and creates nothing; download/ensure return one Path; cache reuse +does not fetch, hash or rewrite legacy manifests. resolve_version errors and +transfer failures use an optional caller error_cls and preserve causes. status +retains hitlist's keys and root manifest semantics. Legacy manifests contain +the most recently downloaded version per dataset, independently of the pinned +default whose cache presence status reports. + +Delegate acquisition and transformation to fetch_file with forwarded keyword +options. Human cache/download messages stay downstream. Hash new installed +files in bounded chunks for the compatibility receipt, then publish JSON with +the existing atomic helper. A failed transfer leaves old files/receipts intact. +Do not create generations, symlinks or migration copies. File and root receipt +remain separate publications, not a multi-file transaction. + +Review found and reproduced an existing lost-update race: two concurrent +registry downloads install two files but retain one manifest entry. Re-plan +before adoption: use the cross-platform filelock package to serialize downloads +and receipt updates per root, checking cache presence again inside the lock. +Ordinary read-only reuse must never create/acquire that lock. Test independent +writers and verify this prevents the reproduced race. Resolve the root once per +operation so a dynamic callable cannot split one publication across roots. + +- [x] Inspect caller contracts; file compatibility gap and write specification. +- [x] Add the shared registry and public export; document behavior and examples. +- [x] Cover legacy read-only reuse, absent path resolution, all public return + shapes/errors, refresh failure, root changes, transforms and manifest writes. +- [ ] Bump 1.15.0, run lint.sh and test.sh, review diff and current-head CI. +- [ ] Merge, run deploy.sh from clean master, verify PyPI wheel and sdist. +- [ ] Adopt the released registry in hitlist's compatibility wrapper. + +Review: lint and all 764 tests pass with 95% coverage. The independent-process +regression fails without the writer lock (one receipt for two files), and passes +with it. Acquisition and receipt publication failures preserve their documented +legacy behavior; normal cache hits remain offline and create no lock files. +Minimum filelock compatibility and current-head CI remain release checks. diff --git a/tasks/todo.md b/tasks/todo.md index ab0a0b9..542ace2 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -1,3 +1,7 @@ +# Fixed-path registry adoption (#83) + +See [fixed_path_registry.md](fixed_path_registry.md) for the specification and checklist. + # Datacache #80 and #81: resumable and raw downloads ## Specification diff --git a/tests/test_file_registry.py b/tests/test_file_registry.py new file mode 100644 index 0000000..4e135f6 --- /dev/null +++ b/tests/test_file_registry.py @@ -0,0 +1,225 @@ +"""Released hitlist contracts for the shared fixed-path registry.""" + +import gzip +import hashlib +import json +import multiprocessing +from pathlib import Path +import threading + +import pytest + +from datacache import VersionedFileRegistry +from datacache import file_registry + + +@pytest.fixture +def registry(tmp_path): + source = tmp_path / 'source' + source.write_bytes(b'original') + mapping = {'thing': { + 'filename': 'thing.tsv', 'urls': {'v1': source.as_uri(), 'v2': source.as_uri()}, + 'default_version': 'v2', 'description': 'reference', + }} + return VersionedFileRegistry(mapping, cache_dir=lambda: tmp_path / 'cache'), source + + +def test_missing_lookup_and_status_are_read_only(registry, tmp_path): + reg, _ = registry + assert reg.resolve_version('thing') == 'v2' + assert reg.resolve_version('thing', 'v1') == 'v1' + assert reg.local_path('thing') == tmp_path / 'cache/thing/v2/thing.tsv' + assert not reg.is_cached('thing') + assert reg.status() == [{ + 'name': 'thing', 'description': 'reference', 'default_version': 'v2', + 'available_versions': ['v1', 'v2'], 'cached': False, 'cached_version': None, + 'bytes': None, 'downloaded_at': None, 'path': str(reg.local_path('thing')), + }] + assert not (tmp_path / 'cache').exists() + + +def test_download_receipt_and_multiple_versions(registry, tmp_path): + reg, source = registry + first = reg.download('thing', 'v1', timeout=2, record_provenance=True) + assert isinstance(first, Path) + assert first == tmp_path / 'cache/thing/v1/thing.tsv' + assert first.read_bytes() == b'original' + manifest_path = tmp_path / 'cache/manifest.json' + receipt = json.loads(manifest_path.read_text())['thing'] + assert receipt['version'] == 'v1' + assert receipt['path'] == str(first) + assert receipt['url'] == source.as_uri() + assert receipt['bytes'] == 8 + assert receipt['sha256'] == hashlib.sha256(b'original').hexdigest() + assert receipt['downloaded_at'].endswith('+00:00') + second = reg.ensure('thing') + assert first.exists() and second.exists() + assert first != second + assert reg.status()[0]['cached_version'] == 'v2' + assert not list(manifest_path.parent.glob('.datacache-json-*')) + + +def test_legacy_reuse_is_offline_without_hash_or_manifest_rewrite(registry, tmp_path, monkeypatch): + reg, source = registry + path = reg.local_path('thing') + path.parent.mkdir(parents=True) + path.write_bytes(b'legacy file') + manifest = tmp_path / 'cache/manifest.json' + manifest.write_text(json.dumps({'thing': { + 'version': 'v2', 'path': str(path), 'bytes': 11, 'downloaded_at': '2020-01-01', + }})) + before = {p: (p.read_bytes(), p.stat().st_mtime_ns) for p in (path, manifest)} + source.unlink() + monkeypatch.setattr(file_registry, 'fetch_file', lambda *a, **k: pytest.fail('fetch')) + monkeypatch.setattr(file_registry.hashlib, 'sha256', lambda *a: pytest.fail('hash')) + assert reg.download('thing') == path + assert reg.ensure('thing') == path + assert reg.is_cached('thing') + assert reg.status()[0]['downloaded_at'] == '2020-01-01' + assert before == {p: (p.read_bytes(), p.stat().st_mtime_ns) for p in (path, manifest)} + + +def test_forced_failure_preserves_legacy_file_and_receipt(registry, tmp_path): + reg, source = registry + path = reg.download('thing') + manifest = tmp_path / 'cache/manifest.json' + before = manifest.read_bytes() + source.unlink() + with pytest.raises(RuntimeError, match='failed to download') as raised: + reg.download('thing', force=True) + assert raised.value.__cause__ is not None + assert path.read_bytes() == b'original' + assert manifest.read_bytes() == before + + +def test_refresh_and_raw_archive_override(registry): + reg, source = registry + archive = source.with_suffix('.gz') + archive.write_bytes(gzip.compress(b'expanded')) + reg._datasets['thing']['urls']['v2'] = archive.as_uri() + path = reg.download('thing') + assert path.read_bytes() == b'expanded' + assert reg.download('thing', force=True, raw=True).read_bytes() == archive.read_bytes() + + +def test_callable_root_remains_dynamic(registry, tmp_path): + reg, _ = registry + selected = [tmp_path / 'first'] + reg._cache_dir = lambda: str(selected[0]) + first = reg.download('thing') + selected[0] = tmp_path / 'second' + second = reg.ensure('thing') + assert first != second + assert first.read_bytes() == second.read_bytes() + + +def test_custom_errors_and_cause(registry, monkeypatch): + reg, _ = registry + class DomainError(RuntimeError): + pass + reg._error_cls = DomainError + with pytest.raises(DomainError, match='unknown dataset'): + reg.resolve_version('absent') + with pytest.raises(DomainError, match='available: v1, v2'): + reg.resolve_version('thing', 'v3') + error = OSError('offline') + def fail(*a, **k): + raise error + monkeypatch.setattr(file_registry, 'fetch_file', fail) + with pytest.raises(DomainError) as raised: + reg.download('thing') + assert raised.value.__cause__ is error + + +@pytest.mark.parametrize('method', ['download', 'ensure']) +def test_explicit_integrity_expectations_are_not_ignored_on_reuse(registry, method): + reg, _ = registry + path = reg.download('thing') + with pytest.raises(RuntimeError, match='size'): + getattr(reg, method)('thing', expected_size=99) + assert path.read_bytes() == b'original' + + +def test_failed_manifest_publication_preserves_prior_receipt(registry, tmp_path, monkeypatch): + reg, source = registry + reg.download('thing') + manifest = tmp_path / 'cache/manifest.json' + before = manifest.read_bytes() + source.write_bytes(b'new file') + import datacache._filesystem as filesystem + replace = filesystem.os.replace + def fail_manifest(src, dest): + if Path(dest) == manifest: + raise OSError('receipt publication failed') + return replace(src, dest) + monkeypatch.setattr(filesystem.os, 'replace', fail_manifest) + with pytest.raises(OSError, match='receipt publication failed'): + reg.download('thing', force=True) + assert manifest.read_bytes() == before + # Legacy single-file contract: file publication and receipt are separate. + assert reg.local_path('thing').read_bytes() == b'new file' + assert not list(manifest.parent.glob('.datacache-json-*')) + + +def _download_in_process(root, source, name, start, read_barrier): + """Widen the old manifest race deterministically, in separate interpreters.""" + registry = VersionedFileRegistry({name: { + 'filename': 'data', 'urls': {'v1': Path(source).as_uri()}, 'default_version': 'v1', + }}, cache_dir=lambda: Path(root)) + read = registry._read_manifest_at + def overlapping_read(path): + result = read(path) + try: + read_barrier.wait(timeout=0.5) + except threading.BrokenBarrierError: + pass # a correct writer lock prevents the other reader entering + return result + registry._read_manifest_at = overlapping_read + start.wait(timeout=20) + registry.download(name) + + +def test_concurrent_processes_preserve_both_manifest_entries(tmp_path): + ctx = multiprocessing.get_context('spawn') + source = tmp_path / 'source' + source.write_bytes(b'data') + root = tmp_path / 'cache' + start, read_barrier = ctx.Barrier(2), ctx.Barrier(2) + processes = [ctx.Process(target=_download_in_process, + args=(str(root), str(source), name, start, read_barrier)) + for name in ('first', 'second')] + try: + for process in processes: + process.start() + for process in processes: + process.join(timeout=30) + assert [process.exitcode for process in processes] == [0, 0] + finally: + for process in processes: + if process.is_alive(): + process.terminate() + process.join(timeout=5) + manifest = json.loads((root / 'manifest.json').read_text()) + assert set(manifest) == {'first', 'second'} + assert all(Path(record['path']).read_bytes() == b'data' for record in manifest.values()) + + +def test_download_resolves_dynamic_root_once(registry, tmp_path): + reg, _ = registry + calls = [] + def changing_root(): + calls.append(len(calls)) + return tmp_path / str(len(calls)) + reg._cache_dir = changing_root + path = reg.download('thing') + assert len(calls) == 1 + receipt = json.loads((tmp_path / '1/manifest.json').read_text()) + assert receipt['thing']['path'] == str(path) + + +def test_atomic_receipt_supports_platform_without_fchmod(tmp_path, monkeypatch): + import datacache._filesystem as filesystem + monkeypatch.delattr(filesystem.os, 'fchmod') + path = tmp_path / 'manifest.json' + filesystem.write_json(path, {'data': 'receipt'}) + assert json.loads(path.read_text()) == {'data': 'receipt'}