diff --git a/CHANGELOG.md b/CHANGELOG.md index 0114d79..c0221cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## 1.19.0 + +- Add opt-in `verify_files=False` to `inspect_bundle`, `install_bundle` and + `VersionedDatasetRegistry`'s `inspect`, `local_path`, `is_cached`, `status`, + `download` and `ensure`: checks ownership, source identity, required file + types, readability and recorded sizes without reading payloads, so it misses + same-size corruption. Fast results never claim hash verification. Full + verification stays the default everywhere, so all registry methods agree + about a corrupted bundle. New generations and recovery always validate + bytes before publication (#88). +- `VersionedDatasetRegistry.ensure` returns the paths its download validated + instead of inspecting the bundle a second time. +- `VersionedArchiveRegistry.status` takes `verify_files` like the archive + registry's other lookups; it stays metadata-only by default. + ## 1.18.1 - Reorganize the documentation into a site (`./docs.sh`, published from diff --git a/datacache/archives.py b/datacache/archives.py index ab7087c..fa383ca 100644 --- a/datacache/archives.py +++ b/datacache/archives.py @@ -902,8 +902,12 @@ def is_cached(self, name, version=None, *, verify_files=False): return self.inspect( name, version, verify_files=verify_files).status == "available" - def status(self, name=None): - """Return one offline status row per concrete archive version.""" + def status(self, name=None, *, verify_files=False): + """Return one offline status row per concrete archive version. + + Rows check publication and source metadata without reading payloads; + verify_files=True hashes every file as inspect does by default. + """ if name is not None: self.resolve_version(name) names = [name] @@ -914,7 +918,7 @@ def status(self, name=None): archive = self._archives[archive_name] for version, definition in archive["versions"].items(): inspection = self.inspect( - archive_name, version, verify_files=False) + archive_name, version, verify_files=verify_files) identities = [source.get("url") or _local_identity(source["path"]) for source in definition["sources"]] rows.append({ diff --git a/datacache/bundles.py b/datacache/bundles.py index 4032740..23baf31 100644 --- a/datacache/bundles.py +++ b/datacache/bundles.py @@ -14,7 +14,7 @@ from ._filesystem import file_lock, open_regular, path_present, read_json, write_json from .integrity import FileValidationError, _validate_expectations -from .inspection import inspect_file +from .inspection import FileInspection, inspect_file from .provenance import redact_url, sidecar_path STORE = '.datacache-bundle.json' @@ -117,7 +117,8 @@ class BundleInspection: status is available, missing, invalid, inaccessible, or recovery-required. verified means every file matched caller-supplied trusted hashes now; - receipt-only inspection checks recorded hashes without asserting trust. + receipt-only full inspection checks recorded hashes without asserting trust. + Metadata-only inspection checks readability and recorded sizes, not hashes. """ path: str status: str @@ -127,7 +128,7 @@ class BundleInspection: error: object = None -def _inspect_generation(store, generation, assets): +def _inspect_generation(store, generation, assets, *, verify_files=True): directory = _generation(store, generation) receipt = read_json(directory / MANIFEST) recorded = _receipt_assets(receipt) @@ -152,23 +153,36 @@ def _inspect_generation(store, generation, assets): _directory(directory / parent) # Reject symlinks and special files before the ordinary inspection API. fd = open_regular(target) - os.close(fd) - inspected = inspect_file(target, expected_sha256=spec['sha256'], expected_size=spec['size']) + try: + info = None if verify_files else os.fstat(fd) + finally: + os.close(fd) + if verify_files: + inspected = inspect_file(target, expected_sha256=spec['sha256'], expected_size=spec['size']) + if inspected.status != 'available': + raise inspected.error or FileValidationError(target, 'unavailable bundle asset') + else: + # Metadata only: an open, readable regular file of the recorded size. + if info.st_size != spec['size']: + raise FileValidationError(target, 'bundle asset size disagrees with manifest') + inspected = FileInspection(str(target), 'available', size=info.st_size, mtime=info.st_mtime) files[name] = inspected - if inspected.status != 'available': - raise inspected.error or FileValidationError(target, 'unavailable bundle asset') - trusted = assets is not None and all(spec['sha256'] for spec in assets.values()) + trusted = verify_files and assets is not None and all(spec['sha256'] for spec in assets.values()) return BundleInspection(str(store), 'available', bool(trusted), str(directory), files) -def inspect_bundle(destination, assets=None): +def inspect_bundle(destination, assets=None, *, verify_files=True): """Validate one installed snapshot offline, without writes, locks or repair. Optional assets is the caller's trusted mapping (url, sha256, size). Without it, verify consistency with the per-generation receipt, verified=False. An interrupted install with completed local generations but no pointer is recovery-required; explicit install_bundle can recover it without network. + verify_files=False checks the receipt, source expectations, required file + types, readability and sizes without reading payloads; verified stays False. """ + if not isinstance(verify_files, bool): + raise ValueError('verify_files must be a boolean') path = Path(destination) expected = _assets(assets, verified=False) if assets is not None else None try: @@ -190,7 +204,7 @@ def inspect_bundle(destination, assets=None): if any((path / 'generations').iterdir()): return BundleInspection(str(path), 'recovery-required') return BundleInspection(str(path), 'missing') - return _inspect_generation(path, pointer['generation'], expected) + return _inspect_generation(path, pointer['generation'], expected, verify_files=verify_files) except PermissionError as error: return BundleInspection(str(path), 'inaccessible', error=error) except (OSError, ValueError, KeyError, TypeError, RecursionError) as error: @@ -240,7 +254,7 @@ def _recover(path, assets): return None -def install_bundle(destination, assets, *, force=False, verified=True, download_options=None): +def install_bundle(destination, assets, *, force=False, verified=True, verify_files=True, download_options=None): """Install all assets before publishing one atomic generation pointer. assets maps relative names to {url, sha256, size, decompress?}. Trusted @@ -253,10 +267,12 @@ def install_bundle(destination, assets, *, force=False, verified=True, download_ retained, so these paths survive later force installs. Generated outputs belong outside this managed source store. Installation requires a POSIX local filesystem with flock and atomic sibling os.replace. + verify_files controls existing-generation checks only. New generations and + explicit recovery always validate payloads before publication. """ from .download import fetch_file - if not isinstance(verified, bool) or not isinstance(force, bool): - raise ValueError('verified and force must be booleans') + if not all(isinstance(value, bool) for value in (verified, force, verify_files)): + raise ValueError('verified, force and verify_files must be booleans') expected = _assets(assets, verified) options = dict(download_options or {}) allowed = {'timeout', 'chunk_size', 'progress_callback', 'show_progress', @@ -264,7 +280,7 @@ def install_bundle(destination, assets, *, force=False, verified=True, download_ if set(options) - allowed: raise ValueError('unsupported bundle download options: %s' % sorted(set(options) - allowed)) path = Path(destination) - inspection = inspect_bundle(path, expected) + inspection = inspect_bundle(path, expected, verify_files=verify_files) if not force and inspection.status == 'available': return _paths(inspection) if inspection.status == 'inaccessible': @@ -277,7 +293,7 @@ def install_bundle(destination, assets, *, force=False, verified=True, download_ key = hashlib.sha256(os.fsencode(path.name)).hexdigest()[:32] with file_lock(path.parent / ('.datacache-bundle-lock-' + key)): _initialize(path) - inspection = inspect_bundle(path, expected) + inspection = inspect_bundle(path, expected, verify_files=verify_files) if not force and inspection.status == 'available': return _paths(inspection) if inspection.status == 'invalid' and not force: @@ -395,23 +411,27 @@ def bundle_path(self, name, version=None): _directory(parent) return parent / version - def inspect(self, name, version=None): + def inspect(self, name, version=None, *, verify_files=True): version = self.resolve_version(name, version) - return inspect_bundle(self.bundle_path(name, version), self._datasets[name]['versions'][version]) + return inspect_bundle(self.bundle_path(name, version), self._datasets[name]['versions'][version], + verify_files=verify_files) - def download(self, name, version=None, *, force=False, **download_options): + def download(self, name, version=None, *, force=False, verify_files=True, **download_options): """Explicitly install/repair and return a mapping of asset snapshot paths.""" version = self.resolve_version(name, version) return install_bundle(self.bundle_path(name, version), self._datasets[name]['versions'][version], - force=force, verified=self.verified, download_options=download_options) + force=force, verified=self.verified, verify_files=verify_files, + download_options=download_options) - def local_path(self, name, version=None, *, asset=None): + def local_path(self, name, version=None, *, asset=None, verify_files=True): """Resolve an installed snapshot; no writes/network. Missing raises. For one asset, return its Path; for multiple assets return the generation - directory, or select an individual asset with asset=. + directory, or select an individual asset with asset=. Assets are hashed + by default; verify_files=False checks metadata and sizes only, which + cannot detect same-size corruption. """ - inspected = self.inspect(name, version) + inspected = self.inspect(name, version, verify_files=verify_files) if inspected.status == 'missing': raise FileNotFoundError(inspected.path) if inspected.status != 'available': @@ -423,15 +443,23 @@ def local_path(self, name, version=None, *, asset=None): return Path(inspected.generation) def ensure(self, name, version=None, **download_options): - self.download(name, version, **download_options) - return self.local_path(name, version) - - def is_cached(self, name, version=None): - return self.inspect(name, version).status == 'available' - - def status(self): - """One read-only status row per dataset's pinned default.""" + """Download/reuse, then return what local_path would: the single asset's + Path, or the generation directory of several assets.""" + paths = self.download(name, version, **download_options) + if len(paths) == 1: + return Path(next(iter(paths.values()))) + # The paths download validated: no second inspection, so a concurrent + # refresh cannot swap the generation between the two. + asset_name, asset_path = next(iter(paths.items())) + return Path(asset_path).parents[len(Path(asset_name).parts) - 1] + + def is_cached(self, name, version=None, *, verify_files=True): + """Whether inspection reports available; verify_files=False skips hashing.""" + return self.inspect(name, version, verify_files=verify_files).status == 'available' + + def status(self, *, verify_files=True): + """One read-only status row per pinned default; verify_files=False skips hashing.""" return [dict(name=name, version=self.resolve_version(name), description=self._datasets[name]['description'], available_versions=sorted(self._datasets[name]['versions']), - inspection=self.inspect(name)) for name in sorted(self._datasets)] + inspection=self.inspect(name, verify_files=verify_files)) for name in sorted(self._datasets)] diff --git a/datacache/version.py b/datacache/version.py index 4a7bff5..d84d79d 100644 --- a/datacache/version.py +++ b/datacache/version.py @@ -1 +1 @@ -__version__ = "1.18.1" +__version__ = "1.19.0" diff --git a/docs/api.md b/docs/api.md index 6eceed9..d7f1e02 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,7 +1,7 @@ # Public API reference This reference covers every name exported in `datacache.__all__` and every -public `Cache` method in DataCache 1.18.1. Import these names from `datacache`. +public `Cache` method in DataCache 1.19.0. Import these names from `datacache`. Signatures below show all defaults; arguments after `*` are keyword-only. Method signatures omit `self` and are called on a `Cache` instance. @@ -1247,30 +1247,37 @@ DataCache. `verified=False` explicitly enables historical unpinned catalogues. | `local_path(name, version=None, *, verify_files=False)` | Resolve a published generation without network, writes, or repair. | | `ensure(name, version=None, **download_options)` | Install if needed and return the generation. | | `is_cached(name, version=None, *, verify_files=False)` | Whether a published generation is available; full verification is optional. | -| `status(name=None)` | One read-only row per concrete version, including catalogue and downloaded source URLs, store/generation paths, default flag, status, fetch time, observed archive size/hash and inspection. | +| `status(name=None, *, verify_files=False)` | One read-only row per concrete version, including catalogue and downloaded source URLs, store/generation paths, default flag, status, fetch time, observed archive size/hash and inspection. Metadata-only by default; `verify_files=True` hashes every file. | ## install_bundle ```text -install_bundle(destination, assets, *, force=False, verified=True, download_options=None) +install_bundle(destination, assets, *, force=False, verified=True, verify_files=True, download_options=None) ``` Atomically install a mapping of relative asset names to `{url, sha256, size, decompress?}` metadata. Returns a dict of asset names to immutable generation paths. Invalid caches require `force=True`. `verified=False` explicitly permits unpinned sources; their observed hashes are recorded without authenticating them. +`verify_files=False` avoids payload reads for existing-generation checks; it +checks readability, file types and recorded sizes but cannot detect same-size +corruption. New generations and explicit recovery always validate bytes before +publication, regardless of this cache-hit setting. `download_options` accepts timeout, chunk size, progress, retry settings and resume. Publication requires a POSIX local filesystem. [Complete guide](bundles.md). ## inspect_bundle ```text -inspect_bundle(destination, assets=None) +inspect_bundle(destination, assets=None, *, verify_files=True) ``` Return a `BundleInspection` without writes, network, locks or recovery. Supply trusted assets to verify against the registry; omit them to check consistency against the installed generation's own recorded hashes (`verified=False`). +`verify_files=False` validates metadata, source expectations, required file types, +readability and recorded sizes without reading payloads. Its bundle and file +results always have `verified=False`, even with trusted hashes in the registry. ## BundleInspection @@ -1294,12 +1301,12 @@ The [bundle guide](bundles.md) includes an example and downstream migration note | --- | --- | | `resolve_version(name, version=None)` | Concrete version label, applying the pinned default. Unknown names/versions raise `ValueError`. | | `bundle_path(name, version=None)` | Expected managed store `Path`, without creating it. | -| `inspect(name, version=None)` | Read-only `BundleInspection`. | -| `download(name, version=None, *, force=False, **download_options)` | Install/reuse and return a dict of asset snapshot paths. | -| `local_path(name, version=None, *, asset=None)` | Installed single asset's `Path`, or a multi-file generation directory; select one asset by name. Missing raises `FileNotFoundError`; invalid/recovery-required raises `FileValidationError`. | -| `ensure(name, version=None, **download_options)` | Download/reuse, then return `local_path`. | -| `is_cached(name, version=None)` | Whether verified inspection reports `available`. | -| `status()` | One row per dataset's pinned default: name, version, description, available_versions and inspection. | +| `inspect(name, version=None, *, verify_files=True)` | Read-only `BundleInspection`; full payload verification by default. | +| `download(name, version=None, *, force=False, verify_files=True, **download_options)` | Install/reuse and return a dict of asset snapshot paths; the flag selects cache-hit cost only. | +| `local_path(name, version=None, *, asset=None, verify_files=True)` | Installed single asset's `Path`, or a multi-file generation directory; select one asset by name. Assets are hashed by default; `verify_files=False` checks metadata and sizes only. Missing raises `FileNotFoundError`; invalid/recovery-required raises `FileValidationError`. | +| `ensure(name, version=None, **download_options)` | Download/reuse, then return what `local_path` would, from the paths the download validated. `verify_files=False` among the options skips hashing on reuse. | +| `is_cached(name, version=None, *, verify_files=True)` | Whether inspection reports `available`; `verify_files=False` skips hashing. | +| `status(*, verify_files=True)` | One row per pinned default: name, version, description, available_versions and inspection. `verify_files=False` skips hashing. | ## VersionedFileRegistry diff --git a/docs/bundles.md b/docs/bundles.md index 60c125f..ed0b8e4 100644 --- a/docs/bundles.md +++ b/docs/bundles.md @@ -80,8 +80,8 @@ not visit those outputs. generation, and `generation` is that generation's directory. The `error` field retains the cause of a failed inspection. -Inspection hashes every asset. With trusted registry metadata, `verified=True` -means every file matched the supplied SHA-256 just now. Every completed generation +Explicit inspection hashes every asset by default. With trusted registry +metadata, `verified=True` means every file matched the supplied SHA-256 just now. Every completed generation also has its own manifest recording sizes, observed SHA-256 digests, source URLs, and fetch time. `inspect_bundle(directory)` can check those recorded hashes without a registry or network, but returns `verified=False`: a local receipt is @@ -90,9 +90,26 @@ fragments; a separate SHA-256 fingerprint identifies the full source URL without storing that omitted text. Paths are retained, so do not use secret-bearing URL paths. +Use `inspect_bundle(directory, assets, verify_files=False)` or +`registry.inspect(name, verify_files=False)` for metadata-only checks. This +validates ownership, the selected pointer/manifest, registry expectations, the +required inventory, regular file types, readability and recorded sizes without +reading payloads. Both bundle and per-file results have `verified=False`. +Same-size corruption requires full verification to detect. + +Every registry method hashes by default, so `inspect`, `local_path`, +`is_cached`, `status`, `download` and `ensure` agree about a corrupted bundle. +Pass `verify_files=False` to any of them for metadata-only checks where speed +matters more than detecting same-size corruption. On `install_bundle`, +`registry.download` or `ensure` it skips payload reads on reuse only: new +acquisitions and recovery always validate bytes before publishing. All these +checks remain offline and read-only. Archive registries differ: their +`local_path`, `is_cached` and `status` have been metadata-only by default since +they were added, and take `verify_files=True` for full checks. + `verified=False` on installation or registry construction explicitly permits acquiring assets without trusted hashes/sizes. Their observed hashes are still -recorded and checked on reuse, but cannot authenticate the original download. +recorded and checked on full verification, but cannot authenticate the original download. Use this mode only for upstream data without immutable integrity metadata. Without a trusted hash, reuse requires the same full URL fingerprint and decompression setting. Changing either reports an invalid installation; use diff --git a/tests/test_bundle_metadata.py b/tests/test_bundle_metadata.py new file mode 100644 index 0000000..ca6cd76 --- /dev/null +++ b/tests/test_bundle_metadata.py @@ -0,0 +1,271 @@ +"""Fast bundle reads retain snapshot, ownership and explicit trust boundaries.""" + +import builtins +from concurrent.futures import ThreadPoolExecutor +from hashlib import sha256 +import io +import json +import os +from pathlib import Path +import threading + +import pytest + +from datacache import FileValidationError, VersionedDatasetRegistry, inspect_bundle, install_bundle +from datacache import bundles, download + +pytestmark = pytest.mark.skipif(os.name != 'posix', reason='POSIX bundle installation') + + +@pytest.fixture +def installed(tmp_path): + upstream = tmp_path / 'upstream' + upstream.mkdir() + assets = {} + for name, payload in [('records.txt', b'records'), ('nested/metadata.txt', b'metadata')]: + source = upstream / name + source.parent.mkdir(exist_ok=True) + source.write_bytes(payload) + assets[name] = dict(url=source.as_uri(), sha256=sha256(payload).hexdigest(), size=len(payload)) + registry = VersionedDatasetRegistry( + {'reference': dict(default_version='v1', versions={'v1': assets})}, cache_root=tmp_path / 'cache') + paths = registry.download('reference') + return registry, assets, paths + + +@pytest.mark.parametrize('trusted', [True, False]) +def test_fast_resolution_and_cache_hits_read_no_payloads_or_write(installed, monkeypatch, trusted): + registry, assets, paths = installed + if not trusted: + assets = {name: {'url': spec['url']} for name, spec in assets.items()} + registry = VersionedDatasetRegistry( + {'reference': dict(default_version='v1', versions={'v1': assets})}, + cache_root=registry.bundle_path('reference').parents[1], verified=False) + payloads = set(map(Path, paths.values())) + original_open, original_os_open, original_read = builtins.open, os.open, os.read + payload_fds = set() + + def guarded_open(path, *args, **kwargs): + if isinstance(path, (str, os.PathLike)) and Path(path) in payloads: + raise AssertionError('fast lookup opened a payload for content reading') + return original_open(path, *args, **kwargs) + + def tracked_os_open(path, *args, **kwargs): + # Opening a payload to check its type and readability is allowed; + # reading from it is not. + fd = original_os_open(path, *args, **kwargs) + if isinstance(path, (str, os.PathLike)) and Path(path) in payloads: + payload_fds.add(fd) + return fd + + def guarded_read(fd, *args, **kwargs): + if fd in payload_fds: + raise AssertionError('fast lookup read payload bytes') + return original_read(fd, *args, **kwargs) + + def forbidden(*args, **kwargs): + raise AssertionError('fast lookup tried full inspection, mutation or acquisition') + + monkeypatch.setattr(builtins, 'open', guarded_open) + monkeypatch.setattr(io, 'open', guarded_open) + monkeypatch.setattr(os, 'open', tracked_os_open) + monkeypatch.setattr(os, 'read', guarded_read) + monkeypatch.setattr(bundles, 'inspect_file', forbidden) + monkeypatch.setattr(bundles, 'file_lock', forbidden) + monkeypatch.setattr(bundles, 'write_json', forbidden) + monkeypatch.setattr(download, 'fetch_file', forbidden) + monkeypatch.setattr(Path, 'mkdir', forbidden) + state = registry.inspect('reference', verify_files=False) + assert state.status == 'available' and not state.verified + assert {name: item.path for name, item in state.files.items()} == paths + assert all(not item.verified and item.size is not None for item in state.files.values()) + assert registry.local_path('reference', verify_files=False) == Path(state.generation) + assert registry.local_path( + 'reference', asset='records.txt', verify_files=False) == Path(paths['records.txt']) + assert registry.is_cached('reference', verify_files=False) + assert not registry.status(verify_files=False)[0]['inspection'].verified + assert registry.download('reference', verify_files=False) == paths + assert registry.ensure('reference', verify_files=False) == Path(state.generation) + assert inspect_bundle(registry.bundle_path('reference'), verify_files=False).status == 'available' + with pytest.raises(AssertionError, match='full inspection'): + registry.inspect('reference') + + +def test_defaults_agree_on_same_size_corruption_that_only_fast_checks_miss(installed): + registry, assets, paths = installed + path = Path(paths['records.txt']) + path.write_bytes(b'changed') + assert len(b'changed') == assets['records.txt']['size'] + # Every lookup verifies by default, so they agree the bundle is invalid. + assert registry.inspect('reference').status == 'invalid' + assert not registry.is_cached('reference') + assert registry.status()[0]['inspection'].status == 'invalid' + with pytest.raises(FileValidationError): + registry.local_path('reference') + with pytest.raises(FileValidationError, match='force=True'): + registry.download('reference') + with pytest.raises(FileValidationError, match='force=True'): + registry.ensure('reference') + # Opting into metadata-only checks trades that detection for speed. + assert registry.inspect('reference', verify_files=False).status == 'available' + assert registry.is_cached('reference', verify_files=False) + assert registry.download('reference', verify_files=False) == paths + + +def test_fast_resolution_on_a_read_only_store(installed): + registry, assets, paths = installed + store = registry.bundle_path('reference') + entries = [store, *store.rglob('*')] + modes = {path: path.stat().st_mode & 0o777 for path in entries} + try: + for path in entries: + path.chmod(0o555 if path.is_dir() else 0o444) + state = registry.inspect('reference', verify_files=False) + assert state.status == 'available' and not state.verified + assert registry.local_path('reference', verify_files=False) == Path(state.generation) + assert registry.is_cached('reference', verify_files=False) + assert registry.status(verify_files=False)[0]['inspection'].status == 'available' + assert registry.download('reference', verify_files=False) == paths + finally: + for path, mode in modes.items(): + path.chmod(mode) + + +@pytest.mark.parametrize('damage', ['missing', 'size', 'symlink', 'hardlink', 'fifo', + 'parent-link', 'pointer', 'manifest', 'inventory', 'expectations']) +def test_fast_lookup_rejects_invalid_metadata_and_unsafe_files(installed, tmp_path, damage): + registry, assets, paths = installed + store = registry.bundle_path('reference') + target = Path(paths['records.txt']) + generation = target.parent + manifest = generation / bundles.MANIFEST + if damage == 'missing': + target.unlink() + elif damage == 'size': + target.write_bytes(b'short') + elif damage == 'symlink': + target.unlink() + target.symlink_to(tmp_path / 'upstream/records.txt') + elif damage == 'hardlink': + os.link(target, tmp_path / 'hardlink') + elif damage == 'fifo': + target.unlink() + os.mkfifo(target) + elif damage == 'parent-link': + nested = generation / 'nested' + nested.rename(generation / 'original-nested') + nested.symlink_to(generation / 'original-nested', target_is_directory=True) + elif damage == 'pointer': + (store / bundles.CURRENT).write_text(json.dumps({'generation': '../outside'})) + elif damage == 'manifest': + manifest.write_text('{') + elif damage == 'inventory': + receipt = json.loads(manifest.read_text()) + del receipt['assets']['records.txt'] + manifest.write_text(json.dumps(receipt)) + else: + assets['records.txt']['sha256'] = '0' * 64 + assert inspect_bundle(store, assets, verify_files=False).status == 'invalid' + + +def test_fast_lookup_retains_unverified_source_identity_checks(installed): + registry, assets, paths = installed + untrusted = {name: dict(url=spec['url']) for name, spec in assets.items()} + untrusted['records.txt']['url'] = 'https://different.invalid/records.txt' + assert inspect_bundle(registry.bundle_path('reference'), untrusted, verify_files=False).status == 'invalid' + + +def test_fast_lookup_reports_permission_errors(installed, monkeypatch): + registry, assets, paths = installed + original = bundles.open_regular + + def denied(path, *args, **kwargs): + if str(path) == paths['records.txt']: + raise PermissionError('payload is not readable') + return original(path, *args, **kwargs) + + monkeypatch.setattr(bundles, 'open_regular', denied) + state = registry.inspect('reference', verify_files=False) + assert state.status == 'inaccessible' + assert isinstance(state.error, PermissionError) + + +def test_fast_install_flag_never_weakens_new_generation_validation(installed, monkeypatch): + registry, assets, paths = installed + store = registry.bundle_path('reference') + pointer = (store / bundles.CURRENT).read_bytes() + + def bad_download(url, *, destination, **kwargs): + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(b'x' * kwargs['expected_size']) + return str(destination) + + monkeypatch.setattr(download, 'fetch_file', bad_download) + with pytest.raises(FileValidationError, match='SHA-256'): + registry.download('reference', force=True, verify_files=False) + assert (store / bundles.CURRENT).read_bytes() == pointer + assert registry.local_path('reference', asset='records.txt') == Path(paths['records.txt']) + + +def test_fast_install_flag_never_weakens_recovery_validation(installed, monkeypatch): + registry, assets, paths = installed + store = registry.bundle_path('reference') + (store / bundles.CURRENT).unlink() + Path(paths['records.txt']).write_bytes(b'changed') + assert registry.inspect('reference', verify_files=False).status == 'recovery-required' + + def forbidden(*args, **kwargs): + raise RuntimeError('recovery rejected corrupted bytes and attempted acquisition') + + monkeypatch.setattr(download, 'fetch_file', forbidden) + with pytest.raises(RuntimeError, match='recovery rejected'): + registry.download('reference', verify_files=False) + assert not (store / bundles.CURRENT).exists() + + +def test_fast_readers_resolve_one_generation_during_refresh(installed, monkeypatch): + registry, assets, paths = installed + entered, release = threading.Event(), threading.Event() + original = bundles.write_json + + def paused(path, value, **kwargs): + if Path(path).name == bundles.CURRENT: + entered.set() + assert release.wait(5) + return original(path, value, **kwargs) + + monkeypatch.setattr(bundles, 'write_json', paused) + with ThreadPoolExecutor() as pool: + future = pool.submit(registry.download, 'reference', force=True) + assert entered.wait(5) + try: + state = registry.inspect('reference', verify_files=False) + assert {name: item.path for name, item in state.files.items()} == paths + finally: + release.set() + refreshed = future.result() + state = registry.inspect('reference', verify_files=False) + assert {name: item.path for name, item in state.files.items()} == refreshed + assert all(Path(path).exists() for path in paths.values()) + + +@pytest.mark.parametrize('flag', [None, 0, 'false']) +def test_nonboolean_verification_modes_fail_without_creating_paths(tmp_path, flag): + dest = tmp_path / 'missing' + with pytest.raises(ValueError, match='verify_files'): + inspect_bundle(dest, verify_files=flag) + with pytest.raises(ValueError, match='verify_files'): + install_bundle(dest, {'file': {'url': 'https://example.invalid/data'}}, + verified=False, verify_files=flag) + assert not dest.exists() + + +def test_ensure_returns_the_downloaded_snapshot_without_inspecting_again(installed, monkeypatch): + registry, assets, paths = installed + generation = Path(paths['records.txt']).parent + monkeypatch.setattr(registry, 'inspect', lambda *args, **kwargs: pytest.fail('inspected again')) + monkeypatch.setattr(bundles, 'install_bundle', lambda *args, **kwargs: paths) + assert registry.ensure('reference') == generation + single = {'records.txt': paths['records.txt']} + monkeypatch.setattr(bundles, 'install_bundle', lambda *args, **kwargs: single) + assert registry.ensure('reference') == Path(paths['records.txt']) diff --git a/tests/test_bundles.py b/tests/test_bundles.py index a5b9b53..449ca24 100644 --- a/tests/test_bundles.py +++ b/tests/test_bundles.py @@ -220,6 +220,7 @@ def test_registry_versions_are_independent_and_pinned(tmp_path, assets): reg.download('example', '2025-01') assert reg.local_path('example', '2025-01').is_file() assert reg.status()[0]['inspection'].verified + assert not reg.status(verify_files=False)[0]['inspection'].verified with pytest.raises(ValueError): reg.resolve_version('example', 'latest')