diff --git a/CHANGELOG.md b/CHANGELOG.md index 67e583e..0a82177 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 1.14.0 + +- Allow `resume=True` with `expected_size` alone when the server supplies a + strong ETag. Resume uses `If-Range` and validates the returned range and + validator; servers without a strong ETag still require `expected_sha256`. + Complete size-only partials restart instead of being published without a + fresh response. Size-only validation does not verify content checksums (#80). +- Add `raw=True` to `fetch_file` and `Cache.fetch` to disable archive + decompression and HTML conversion at arbitrary output names. It composes + with integrity checks, atomic replacement, progress, provenance, and resume; + existing suffix inference is unchanged when raw mode is omitted (#81). + ## 1.13.0 - Add integrity-pinned resumable raw HTTP downloads with `resume=True`, private diff --git a/README.md b/README.md index e468282..ebd201d 100644 --- a/README.md +++ b/README.md @@ -147,8 +147,11 @@ read-only offline inspection, and keeps returned paths usable during refreshes. The [bundle guide](docs/bundles.md) covers shared OpenVax roots, generated indices, and migration from downstream registries. -For large raw HTTP files, pass `resume=True` with `expected_sha256` and -`expected_size` to `fetch_file` or `Cache.fetch`. Interrupted transfers retain +For large raw HTTP files, pass `resume=True` and `expected_size` to `fetch_file` +or `Cache.fetch`. Supply `expected_sha256` when available; otherwise the server +must provide a strong ETag for safe range continuation. Size-only validation +does not verify a content checksum. Pass `raw=True` to preserve gzip, ZIP, or +HTML payloads at arbitrary output names without conversion. Interrupted transfers retain private partials and continue on retry. See [resumable downloads](docs/downloads.md#resumable-http-downloads). ## Choose the right API diff --git a/datacache/cache.py b/datacache/cache.py index 7c25d35..c59111a 100644 --- a/datacache/cache.py +++ b/datacache/cache.py @@ -125,7 +125,8 @@ def fetch( show_progress=False, record_provenance=False, allow_empty=False, - resume=False): + resume=False, + raw=False): """ Return the local path to the downloaded copy of a given URL. Don't download the file again if it's already present, @@ -138,6 +139,9 @@ def fetch( record_provenance=True records where the file came from, for inspect_file, as described for fetch_file. allow_empty=True accepts an empty file, which is otherwise rejected. + raw=True disables decompression and HTML conversion even for an explicit + filename. It cannot be combined with decompress=True. Resume options + have the same size, hash, and strong-ETag requirements as fetch_file. `use_wget_if_available` is deprecated and ignored (datacache always uses its streaming Python downloader now); passing it emits a warning. @@ -162,7 +166,8 @@ def fetch( show_progress=show_progress, record_provenance=record_provenance, allow_empty=allow_empty, - resume=resume) + resume=resume, + raw=raw) self._local_paths[key] = path return path diff --git a/datacache/download.py b/datacache/download.py index 27dd95e..507a587 100644 --- a/datacache/download.py +++ b/datacache/download.py @@ -533,7 +533,8 @@ def fetch_file( show_progress=False, record_provenance=False, allow_empty=False, - resume=False): + resume=False, + raw=False): """ Download a remote file and store it locally in a cache directory. Don't download it again if it's already present (unless `force` is True.) @@ -553,6 +554,12 @@ def fetch_file( under a distinct cache key. An explicit filename or destination lacking the source's .zip/.gz suffix still implies decompression for compatibility. + raw : bool, optional + Disable archive decompression and HTML-to-CSV conversion regardless of + the output name. Incompatible with decompress=True. Defaults to False, + preserving legacy output-name inference. Integrity expectations then + describe the unchanged payload, after HTTP transfer decoding. + subdir : str, optional Application name selecting a platform cache directory, "datacache" by default. It is not nested inside the default cache. Ignored when @@ -619,8 +626,12 @@ def fetch_file( resume : bool, optional Keep private partials and resume raw HTTP(S) transfers, default False. - Requires both expected_sha256 and expected_size, and a POSIX local - filesystem. Does not support decompression or HTML conversion. Retries + Requires expected_size and a POSIX local filesystem. If expected_sha256 + is omitted, the server must supply a strong ETag on every accepted + response; it is sent as If-Range when resuming. ETags prevent mixing + representations, but do not verify a trusted checksum. Cache hits with + size alone check only their byte count. Does not support decompression + or HTML conversion; raw=True permits arbitrary output names. Retries report cumulative bytes, resetting only when the server requires a fresh transfer. discard_partial(destination) explicitly drops partials. Interrupted downloads retain at most expected_size partial bytes. @@ -654,6 +665,10 @@ def fetch_file( """ if not isinstance(resume, bool): raise ValueError("resume must be a boolean") + if not isinstance(raw, bool): + raise ValueError("raw must be a boolean") + if raw and decompress: + raise ValueError("raw=True cannot be combined with decompress=True") if resume: from .resume import validate_resume validate_resume(download_url, expected_sha256, expected_size) @@ -683,11 +698,14 @@ def fetch_file( destination=destination, cache_root=cache_root) source_suffix = _source_suffix(download_url) output_suffix = os.path.splitext(full_path)[1].lower() - archive_decompression = bool(decompress or (explicit_output and output_suffix != source_suffix)) + archive_decompression = not raw and bool( + decompress or (explicit_output and output_suffix != source_suffix)) + html_conversion = (not raw and explicit_output and source_suffix in (".htm", ".html") + and output_suffix == ".csv") if resume and (decompress or (source_suffix in (".gz", ".zip") and archive_decompression) or - (explicit_output and source_suffix in (".htm", ".html") and output_suffix == ".csv")): - raise ValueError("resume=True supports raw downloads only; retain the archive suffix") + html_conversion): + raise ValueError("resume=True supports raw downloads only; use raw=True or retain the archive suffix") if not force: try: validate_file(full_path, expected_sha256, expected_size) @@ -728,7 +746,7 @@ def fetch_file( chunk_size=chunk_size, progress_callback=progress_callback, decompress=archive_decompression, - convert_html=explicit_output, + convert_html=html_conversion, explicit_output=explicit_output, expected_sha256=expected_sha256, expected_size=expected_size, diff --git a/datacache/resume.py b/datacache/resume.py index fb43332..13239f4 100644 --- a/datacache/resume.py +++ b/datacache/resume.py @@ -1,4 +1,4 @@ -"""Integrity-pinned raw HTTP downloads with private, persistent partials.""" +"""Hash- or strong-ETag-validated HTTP downloads with private partials.""" from contextlib import closing import hashlib @@ -20,8 +20,8 @@ def validate_resume(download_url, expected_sha256, expected_size): _validate_expectations(expected_sha256, expected_size) - if expected_sha256 is None or expected_size is None: - raise ValueError('resume=True requires expected_sha256 and expected_size') + if expected_size is None: + raise ValueError('resume=True requires expected_size') if urlsplit(download_url).scheme.lower() not in ('http', 'https'): raise ValueError('resume=True supports only raw HTTP/HTTPS downloads') if os.name != 'posix': @@ -61,9 +61,15 @@ def discard_partial(destination): (directory / name).unlink(missing_ok=True) +def _strong_etag(value): + # RFC 9110 entity-tag syntax, without the weak W/ prefix. Embedded quotes, + # whitespace, and control characters cannot authorize an If-Range request. + return isinstance(value, str) and re.fullmatch(r'"[\x21\x23-\x7e\x80-\xff]*"', value) is not None + + def _validator(headers): etag = headers.get('ETag', '') - if etag.startswith('"') and etag.endswith('"'): + if _strong_etag(etag): return {'header': 'ETag', 'value': etag} # Last-Modified is not always strong, but changes still invalidate a partial. modified = headers.get('Last-Modified') @@ -80,7 +86,8 @@ def download_resumable(download_url, destination, *, expected_sha256, expected_s directory = _prepare_directory(destination) partial = directory / 'partial' metadata_path = directory / 'metadata.json' - identity = {'sha256': expected_sha256.lower(), 'size': expected_size, + identity = {'sha256': expected_sha256.lower() if expected_sha256 is not None else None, + 'size': expected_size, 'url_hash': hashlib.sha256(download_url.encode()).hexdigest()} with file_lock(directory / 'lock', private=True): # Another installer may have completed while we waited for its lock. @@ -100,10 +107,14 @@ def download_resumable(download_url, destination, *, expected_sha256, expected_s # Unlink, never truncate an unknown path planted in the state dir. partial.unlink(missing_ok=True) validator = metadata.get('validator') - if validator is not None and not ( + valid_validator = ( isinstance(validator, dict) and validator.get('header') in ('ETag', 'Last-Modified') and isinstance(validator.get('value'), str) - and '\r' not in validator['value'] and '\n' not in validator['value']): + and '\r' not in validator['value'] and '\n' not in validator['value'] + and (validator['header'] != 'ETag' or _strong_etag(validator['value']))) + strong_validator = (valid_validator and validator['header'] == 'ETag') + if ((validator is not None and not valid_validator) + or (expected_sha256 is None and not strong_validator)): metadata['validator'] = None partial.unlink(missing_ok=True) write_json(metadata_path, metadata) @@ -122,16 +133,20 @@ def report(): offset = output.tell() if offset >= expected_size: output.flush() - try: - validate_file(partial, expected_sha256, expected_size) - except FileValidationError: - output.seek(0) - output.truncate() - offset = 0 - metadata['validator'] = None - else: - report() - break + if expected_sha256 is not None: + try: + validate_file(partial, expected_sha256, expected_size) + except FileValidationError: + pass + else: + report() + break + # Size alone cannot validate a completed persistent + # partial, including an empty one. Request fresh bytes. + output.seek(0) + output.truncate() + offset = 0 + metadata['validator'] = None callback_failed = False try: # One protocol restart per attempt, independent of retries. @@ -175,6 +190,10 @@ def report(): offset = 0 else: raise FileValidationError(destination, 'expected HTTP 200 or 206') + if expected_sha256 is None and ( + current_validator is None or current_validator['header'] != 'ETag'): + raise FileValidationError( + destination, 'resume without expected_sha256 requires a strong ETag') metadata['validator'] = current_validator write_json(metadata_path, metadata) for chunk in response.iter_content(chunk_size=chunk_size): diff --git a/datacache/version.py b/datacache/version.py index 84c54b7..e4f2ad4 100644 --- a/datacache/version.py +++ b/datacache/version.py @@ -1 +1 @@ -__version__ = '1.13.0' +__version__ = '1.14.0' diff --git a/docs/api.md b/docs/api.md index e27058a..770129e 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,7 +1,7 @@ # Public API reference This reference covers every name exported in `datacache.__all__` and every -public `Cache` method in DataCache 1.13.0. Import these names from `datacache`. +public `Cache` method in DataCache 1.14.0. Import these names from `datacache`. Signatures below show all defaults; arguments after `*` are keyword-only. Method signatures omit `self` and are called on a `Cache` instance. @@ -60,7 +60,8 @@ fetch_file( timeout=None, use_wget_if_available=None, chunk_size=1048576, progress_callback=None, *, destination=None, cache_root=None, expected_sha256=None, expected_size=None, max_retries=2, retry_backoff=1.0, retry_max_delay=30.0, - show_progress=False, record_provenance=False, allow_empty=False, resume=False + show_progress=False, record_provenance=False, allow_empty=False, resume=False, + raw=False ) ``` @@ -75,6 +76,7 @@ replacement leaves the previous file intact. | `download_url` | Source URL string: HTTP, HTTPS, FTP, or `file://`. | | `filename` | Optional string cache key. Omitted names are inferred from the full URL. Names are normalized by [build_local_filename](#build_local_filename), not treated as exact paths. | | `decompress` | Request gzip/ZIP contents. With inferred filenames, `False` retains the archive and `True` selects a separate decompressed key. An explicit output lacking the source's compression suffix also implies decompression. | +| `raw` | Boolean disabling archive decompression and HTML conversion regardless of the output name. Default `False` preserves legacy inference; cannot be combined with `decompress=True`. Expectations describe the unchanged payload after HTTP transfer decoding. | | `subdir` | Application name selecting a platform cache directory; `None` selects `datacache`. See [get_data_dir](#get_data_dir). | | `force` | Download even when a cached file exists. Does not automatically broaden its permissions. | | `timeout` | Per-attempt timeout in seconds; `None` supplies no timeout. HTTP also accepts a Requests `(connect, read)` tuple. Timeouts limit connection/read inactivity, not overall elapsed time. | @@ -89,7 +91,7 @@ replacement leaves the previous file intact. | `retry_backoff` | Initial retry delay in seconds, doubled for subsequent retries. Must be finite and non-negative. | | `retry_max_delay` | Maximum retry delay in seconds, finite and non-negative. A server `Retry-After` exceeding this limit stops retries. | | `show_progress` | Boolean enabling optional tqdm download, decompression, and hash bars. Requires `datacache[progress]` when a bar is needed. Cache hits are quiet. | -| `resume` | Boolean retaining private partials for raw integrity-pinned HTTP transfers. Requires SHA-256, size and a POSIX local filesystem. See [resumable downloads](downloads.md#resumable-http-downloads). | +| `resume` | Boolean retaining private partials for raw HTTP transfers. Requires size and a POSIX local filesystem. Without an expected SHA-256, every accepted response must supply a strong ETag. Size-only cache hits check only byte count. See [resumable downloads](downloads.md#resumable-http-downloads). | | `record_provenance` | Boolean; after publishing a download, also write a hidden `..datacache.json` record of the source URL (without user name, password, query string, or fragment; the path is kept as is, so avoid recording URLs with secrets in their path), the fetch time, the size, and the SHA-256 when `expected_sha256` verified it. [inspect_file](#inspect_file) reports these offline. The record has the file's permissions. Default `False`: a caller that downloads to a temporary name and then moves the file would leave the record behind. Cache hits never write one, any new download removes a previous record first, and a failure to write one never fails the download. | | `allow_empty` | Boolean accepting an empty installed file; default `False`. A complete but empty response, such as a withdrawn upstream record, is otherwise never published: HTTP retries it as transient, then raises `FileValidationError`. An empty cached file is likewise an invalid hit. `expected_size=0` also allows an empty file. | @@ -867,7 +869,8 @@ Cache.fetch( url, filename=None, decompress=False, force=False, timeout=None, use_wget_if_available=None, *, chunk_size=1048576, progress_callback=None, expected_sha256=None, expected_size=None, max_retries=2, retry_backoff=1.0, - retry_max_delay=30.0, show_progress=False, record_provenance=False, allow_empty=False, resume=False + retry_max_delay=30.0, show_progress=False, record_provenance=False, allow_empty=False, + resume=False, raw=False ) ``` diff --git a/docs/downloads.md b/docs/downloads.md index 1cce759..e74bb18 100644 --- a/docs/downloads.md +++ b/docs/downloads.md @@ -32,15 +32,37 @@ cached file can be reused offline in a readable, non-writable installation. Both expectations always describe **installed bytes**, after decompression or HTML-to-CSV conversion. They do not describe HTTP wire bytes or a compressed -archive when its contents are being installed. To verify and retain an archive, -keep its `.gz` or `.zip` suffix at the destination and leave `decompress=False`. +archive when its contents are being installed. To verify and retain an archive +at any output name, pass `raw=True`. This also disables HTML-to-CSV conversion: + +```python +path = fetch_file( + archive_url, + destination="reference.bin", + raw=True, + expected_sha256=release_metadata["archive_sha256"], + expected_size=release_metadata["archive_size"], + record_provenance=True, +) +``` + +`Cache.fetch` accepts the same `raw=True` option with `filename`. Raw mode is +incompatible with `decompress=True`. It preserves the downloaded payload after +normal HTTP transfer decoding, not encoded HTTP wire bytes. Hashes and sizes +refer to that payload. A cache hit still uses the supplied local validation +checks; changing transformation options does not replace an existing output. +Use distinct names, trusted expectations, or `force=True` when switching modes. + +Without `raw=True`, keeping the source's `.gz` or `.zip` suffix at the destination +and leaving `decompress=False` retains an archive as before. With an inferred filename, archives are retained by default, including URLs with query strings or fragments; their existing cache keys are preserved. `decompress=True` uses a distinct key for the decompressed contents, keeping the full URL in the key's digest. When a download endpoint's inferred filename has no removable archive suffix, `.decompressed` distinguishes its output. With an explicit `filename` or `destination`, a -missing compression suffix still implies decompression for compatibility. +missing compression suffix still implies decompression for compatibility unless +`raw=True` is supplied. `decompress=True` explicitly requests decompression while preserving an explicit destination's exact name. Format detection prefers a supported extension on the URL path (case-insensitively). For download endpoints without a supported @@ -249,24 +271,49 @@ APIs. Use `fetch_file(..., resume=True, expected_sha256=sha256, expected_size=size)` (or the same options on `Cache.fetch`) for large immutable raw HTTP/HTTPS files. -Both trusted expectations are required. Keep an archive's suffix to download -its raw bytes; resumable decompression and HTML conversion are not supported. -The option is off by default and requires a POSIX local filesystem. +`expected_size` is required. Prefer a trusted `expected_sha256` when available. +For sources such as Ensembl that do not publish SHA-256 digests, omit the hash: + +```python +path = fetch_file( + ensembl_dna_url, + destination="reference.download", + raw=True, + resume=True, + expected_size=ensembl_dna_size, +) +``` + +Without `expected_sha256`, every accepted response must supply a syntactically +valid strong ETag. Missing or weak ETags (including a Last-Modified value alone) +raise `FileValidationError` before response bytes are retained or published. +Strong ETags prevent joining bytes from different server representations; they +are not independently verified checksums. Callers can still apply their source's +checksum to the completed archive. Size-only cache hits remain offline and check +only the local byte count, not remote freshness or same-size corruption. Optional +provenance records no verified SHA-256 unless the caller supplied one. + +Use `raw=True` for an arbitrary destination name, or keep the source archive's +suffix. Resumable decompression and HTML conversion are not supported. Resume is +off by default and requires a POSIX local filesystem. Each destination gets a private, owner-only working directory beside it, with one bounded partial file, metadata and a permanent advisory lock. Threads and processes serialize on that destination for the same user; other users retain separate private state. A valid destination remains intact throughout transfer, -verification, and publication. Complete partials are verified and reused without -network; oversized or corrupt ones restart. Interruptions and exhausted transport +verification, and publication. With an expected SHA-256, complete partials are +verified and reused without network; oversized or corrupt ones restart. Without +one, a complete-size partial restarts to obtain a fresh validated response, and +an incomplete partial without a saved strong ETag is discarded. Interruptions and exhausted transport retries keep safely written partial bytes for the next call. Range requests use `Accept-Encoding: identity`. DataCache checks the exact -`Content-Range` offset and total, carries `If-Range` for a strong ETag, compares Last-Modified when no strong -ETag is available, and restarts on changed validators, ignored or -incompatible ranges, or 416 responses. Final SHA-256 and size checks are always -required. A Last-Modified value alone is not a cryptographic identity; these -checks remain the authority. HTTP range behavior follows +`Content-Range` offset and total, carries `If-Range` for a strong ETag, and only +appends bytes from a matching representation. A 200 response replaces the +partial; incompatible ranges, changed validators, or 416 responses restart it. +When a trusted SHA-256 is supplied, downloads can also resume without strong +ETags, with Last-Modified changes triggering restarts and the final digest +authorizing publication. The final byte count is always checked. HTTP range behavior follows [RFC 9110](https://www.rfc-editor.org/rfc/rfc9110.html#name-range-requests). Progress reports bytes present in the current transfer, including the retained diff --git a/examples/basic_usage.py b/examples/basic_usage.py index ea46b3a..6fbdabc 100644 --- a/examples/basic_usage.py +++ b/examples/basic_usage.py @@ -21,6 +21,12 @@ def main(): assert Path(path).read_bytes() == data assert cache.inspect(filename="records.csv", expected_sha256=digest).verified + archive = source.read_bytes() + raw_path = cache.fetch(source.as_uri(), filename="archive.bin", raw=True, + expected_sha256=hashlib.sha256(archive).hexdigest(), + expected_size=len(archive)) + assert Path(raw_path).read_bytes() == archive + with closing(fetch_csv_db( "records", source.as_uri(), csv_filename="records.csv", download_options={"cache_root": root / "cache", "expected_sha256": digest})) as connection: diff --git a/tasks/todo.md b/tasks/todo.md new file mode 100644 index 0000000..ab0a0b9 --- /dev/null +++ b/tasks/todo.md @@ -0,0 +1,63 @@ +# Datacache #80 and #81: resumable and raw downloads + +## Specification + +Release 1.14.0 adds two compatible capabilities to `fetch_file` and `Cache.fetch`. + +1. `raw=True` disables archive decompression and HTML-to-CSV conversion regardless + of the explicit output name. It is a keyword-only boolean, incompatible with + `decompress=True`. Existing calls retain suffix-based inference. Integrity + expectations describe the unchanged downloaded payload; HTTP transfer decoding + retains its existing semantics. Reuse, staging, atomic publication, progress, + permissions, and optional provenance use the existing paths. Raw mode composes + with resume and arbitrary exact destination names. +2. `resume=True` requires `expected_size`, with `expected_sha256` optional only + when each accepted response supplies a syntactically valid strong ETag. + Persist that validator, send it as `If-Range`, and append only an exact 206 + range with the same validator. A 200 response replaces the private partial; + mismatched ranges/validators and 416 retain the existing bounded restart + behavior. Weak, absent, or malformed validators cannot authorize size-only + resume. Existing SHA-256-pinned behavior remains supported, including servers + without ETags. Reject encoded responses and validate the final byte count. +3. Persistent size-only partials require a valid strong ETag in their metadata. + A complete-size partial cannot take the hash-verified publication shortcut: + restart it so the new request validates its server representation. Existing + installed cache hits still perform only the caller's requested local checks; + size-only validation does not imply cryptographic integrity or freshness. +4. Document the new APIs, examples for Ensembl and raw archive adapters, and + the limitations of size-only validation. Do not expand this release into + downstream pyensembl/Hitlist migrations or new checksum formats. + +## Plan + +- [x] Inspect source, issues, repository state, tests, and HTTP validator rules. +- [x] Create an isolated feature checkout and check in the implementation plan. +- [x] Reproduce both missing capabilities with focused regression tests. +- [x] Add raw mode and pass it through Cache.fetch, preserving legacy behavior. +- [x] Extend resumable transfer validation and persistent state for strong ETags. +- [x] Cover interruption/retry, changed and missing validators, bad ranges, + complete partials, raw gzip/ZIP/HTML outputs, cache reuse, provenance, and + atomic preservation of existing destinations. +- [x] Update documentation and changelog; bump to 1.14.0. +- [x] Run ./lint.sh and ./test.sh; review behavior and compatibility. +- [ ] Open PR linking #80 and #81; merge after CI passes. +- [ ] Deploy through ./deploy.sh from clean master; verify PyPI artifacts and + installed-package behavior. +- [ ] Review remaining dependency/urgency groups and report next work. + +## Review + +The initial focused suite passes all 95 raw/resume tests using real local HTTP +servers. Both 1.13.0 limitations were reproduced: raw is not an accepted keyword, +and resume validation requires a SHA-256 even before making a request. The new +paths preserve existing destinations and record no verified digest for size-only +downloads. The first full run passed 749 tests and flagged two documentation +consistency checks: the API reference version and Cache.fetch signature. Both +references have been updated. Final `./lint.sh` and `./test.sh` pass: 751 tests, +95% coverage on Python 3.9.22. The 95 focused raw/resume tests also passed on +Python 3.12.6, and both offline public examples passed. The diff preserves +existing transformation defaults, hash-pinned servers without validators, +private staging, and atomic publication. API docs describe cache reuse and the +checksum limitations of size-only transfers. +Release verification will be recorded in the PR after merge so the +deployed source remains a clean master checkout. diff --git a/tests/test_raw_downloads.py b/tests/test_raw_downloads.py new file mode 100644 index 0000000..5f9288f --- /dev/null +++ b/tests/test_raw_downloads.py @@ -0,0 +1,98 @@ +"""Raw downloads keep payload bytes at exact destinations without conversion.""" + +import gzip +from hashlib import sha256 +from pathlib import Path +import zipfile + +import pytest + +from datacache import Cache, FileValidationError, fetch_file, inspect_file +from datacache import download, provenance + + +@pytest.fixture(params=['gz', 'zip', 'html']) +def source(tmp_path, request): + path = tmp_path / ('source.' + request.param) + payload = b'
name
gene
' + if request.param == 'gz': + path.write_bytes(gzip.compress(payload)) + elif request.param == 'zip': + with zipfile.ZipFile(path, 'w') as archive: + archive.writestr('inside.txt', payload) + else: + path.write_bytes(payload) + return path + + +@pytest.mark.parametrize('api', ['destination', 'filename', 'cache', 'inferred']) +def test_raw_payload_validation_progress_provenance_and_reuse(source, tmp_path, api): + payload = source.read_bytes() + digest = sha256(payload).hexdigest() + events = [] + options = dict(raw=True, expected_sha256=digest, expected_size=len(payload), + record_provenance=True, chunk_size=7, + progress_callback=lambda *args: events.append(args)) + if api == 'destination': + call = lambda: fetch_file(source.as_uri(), destination=tmp_path / 'result.csv', **options) + elif api == 'filename': + call = lambda: fetch_file(source.as_uri(), filename='result.csv', + cache_root=tmp_path / 'cache', **options) + elif api == 'cache': + cache = Cache(cache_root=tmp_path / 'cache') + call = lambda: cache.fetch(source.as_uri(), filename='result.csv', **options) + else: + call = lambda: fetch_file(source.as_uri(), cache_root=tmp_path / 'cache', **options) + path = Path(call()) + assert path.read_bytes() == payload + assert api == 'inferred' or path.name == 'result.csv' + assert events[-1] == (len(payload), len(payload)) + info = inspect_file(path, expected_sha256=digest) + assert info.verified + assert info.recorded_sha256 == digest + record = Path(provenance.sidecar_path(path)).read_bytes() + events.clear() + source.unlink() + assert call() == str(path) + assert not events + assert Path(provenance.sidecar_path(path)).read_bytes() == record + if api == 'cache': + cache.delete_url(source.as_uri()) + assert not path.exists() + assert not Path(provenance.sidecar_path(path)).exists() + + +def test_raw_validation_failure_preserves_destination_and_provenance(source, tmp_path): + dest = tmp_path / 'result.csv' + fetch_file(source.as_uri(), destination=dest, raw=True, record_provenance=True) + original = dest.read_bytes() + record = Path(provenance.sidecar_path(dest)).read_bytes() + source.write_bytes(b'changed source') + with pytest.raises(FileValidationError, match='SHA-256'): + fetch_file(source.as_uri(), destination=dest, raw=True, force=True, + expected_sha256=sha256(original).hexdigest(), record_provenance=True) + assert dest.read_bytes() == original + assert Path(provenance.sidecar_path(dest)).read_bytes() == record + assert not list(tmp_path.glob('.datacache-*')) + + +def test_raw_html_does_not_invoke_parser(tmp_path, monkeypatch): + source = tmp_path / 'not-a-table.html' + source.write_bytes(b'

No table to convert

') + def fail(*args, **kwargs): + pytest.fail('raw mode attempted HTML conversion') + monkeypatch.setattr(download.pd, 'read_html', fail) + path = fetch_file(source.as_uri(), destination=tmp_path / 'result.csv', raw=True) + assert Path(path).read_bytes() == source.read_bytes() + + +@pytest.mark.parametrize('options', [dict(raw=True, decompress=True), dict(raw='yes')]) +@pytest.mark.parametrize('api', ['fetch_file', 'cache']) +def test_invalid_raw_options_have_no_side_effects(tmp_path, options, api): + root = tmp_path / 'missing' + with pytest.raises(ValueError, match='raw'): + if api == 'fetch_file': + fetch_file('https://example.invalid/data.gz', destination=root / 'data', **options) + else: + Cache(cache_root=root).fetch('https://example.invalid/data.gz', **options) + assert not root.exists() diff --git a/tests/test_resume.py b/tests/test_resume.py index 740e9d3..c2d3acd 100644 --- a/tests/test_resume.py +++ b/tests/test_resume.py @@ -5,6 +5,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path import os +import json import socket import threading @@ -78,26 +79,28 @@ def fetch(url, dest, **options): return fetch_file(url, destination=dest, **defaults) -def interrupt(url, dest): +def interrupt(url, dest, **options): def callback(done, total): if done >= 16: raise KeyboardInterrupt with pytest.raises(KeyboardInterrupt): - fetch(url, dest, progress_callback=callback) + fetch(url, dest, progress_callback=callback, **options) -def test_dropped_connection_resumes_and_records(server, tmp_path): +@pytest.mark.parametrize('expected_sha256', [DIGEST, None]) +def test_dropped_connection_resumes_and_records(server, tmp_path, expected_sha256): url, requests = server({'drop': 32}) events = [] dest = tmp_path / 'data' - fetch(url, dest, progress_callback=lambda *args: events.append(args), record_provenance=True) + fetch(url, dest, progress_callback=lambda *args: events.append(args), record_provenance=True, + expected_sha256=expected_sha256) assert requests[1]['Range'] == 'bytes=32-' assert requests[1]['If-Range'] == '"v1"' assert all(r['Accept-Encoding'] == 'identity' for r in requests) assert events[-1] == (len(PAYLOAD), len(PAYLOAD)) assert [e[0] for e in events] == list(range(8, len(PAYLOAD) + 1, 8)) assert dest.read_bytes() == PAYLOAD - assert inspect_file(dest).recorded_sha256 == DIGEST + assert inspect_file(dest).recorded_sha256 == expected_sha256 assert not (_state_directory(dest) / 'partial').exists() @@ -105,30 +108,32 @@ def test_dropped_connection_resumes_and_records(server, tmp_path): {'status': 200}, {'range': 'bytes 0-127/128'}, {'range': 'nonsense'}, {'status': 416}, {'etag': '"v2"'}, ]) -def test_protocol_changes_restart_safely(server, tmp_path, action): +@pytest.mark.parametrize('expected_sha256', [DIGEST, None]) +def test_protocol_changes_restart_safely(server, tmp_path, action, expected_sha256): url, requests = server({}, action) dest = tmp_path / 'data' - interrupt(url, dest) - fetch(url, dest, max_retries=0) + interrupt(url, dest, expected_sha256=expected_sha256) + fetch(url, dest, max_retries=0, expected_sha256=expected_sha256) assert requests[1]['Range'] == 'bytes=16-' assert dest.read_bytes() == PAYLOAD if action.get('status') != 200: assert 'Range' not in requests[2] -def test_interruption_preserves_private_partial_and_old_destination(server, tmp_path): +@pytest.mark.parametrize('expected_sha256', [DIGEST, None]) +def test_interruption_preserves_private_partial_and_old_destination(server, tmp_path, expected_sha256): url, requests = server() dest = tmp_path / 'data' dest.write_bytes(b'old') def callback(done, total): raise KeyboardInterrupt with pytest.raises(KeyboardInterrupt): - fetch(url, dest, force=True, progress_callback=callback) + fetch(url, dest, force=True, progress_callback=callback, expected_sha256=expected_sha256) assert dest.read_bytes() == b'old' partial = _state_directory(dest) / 'partial' assert partial.read_bytes() == PAYLOAD[:8] assert partial.stat().st_mode & 0o077 == 0 - fetch(url, dest, force=True) + fetch(url, dest, force=True, expected_sha256=expected_sha256) assert requests[1]['Range'] == 'bytes=8-' assert dest.read_bytes() == PAYLOAD @@ -180,11 +185,12 @@ def test_partial_persists_after_exhausted_retries(server, tmp_path): assert requests[1]['Range'] == 'bytes=24-' -def test_concurrent_first_calls_download_once(server, tmp_path): +@pytest.mark.parametrize('expected_sha256', [DIGEST, None]) +def test_concurrent_first_calls_download_once(server, tmp_path, expected_sha256): url, requests = server() dest = tmp_path / 'data' with ThreadPoolExecutor(max_workers=4) as pool: - paths = list(pool.map(lambda _: fetch(url, dest), range(4))) + paths = list(pool.map(lambda _: fetch(url, dest, expected_sha256=expected_sha256), range(4))) assert paths == [str(dest)] * 4 assert dest.read_bytes() == PAYLOAD assert len(requests) == 1 @@ -212,7 +218,7 @@ def test_unsafe_partial_is_rejected(server, tmp_path, kind): @pytest.mark.parametrize('options', [ - {'expected_sha256': None}, {'expected_size': None}, {'decompress': True}, + {'expected_size': None}, {'decompress': True}, {'resume': 'yes'}, ]) def test_invalid_resume_options_have_no_side_effects(tmp_path, options): @@ -248,11 +254,12 @@ def callback(done, total): assert inspect_bundle(dest, assets).verified -def test_encoded_http_body_is_rejected(server, tmp_path): +@pytest.mark.parametrize('expected_sha256', [DIGEST, None]) +def test_encoded_http_body_is_rejected(server, tmp_path, expected_sha256): url, _ = server({'encoding': 'gzip'}) dest = tmp_path / 'data' with pytest.raises(FileValidationError, match='identity'): - fetch(url, dest) + fetch(url, dest, expected_sha256=expected_sha256) assert not dest.exists() @@ -383,3 +390,123 @@ def fail_generation_rename(source, target): assert Path(paths['data']).read_bytes() == PAYLOAD assert len(requests) == 1 assert not working.exists() + + +@pytest.mark.parametrize('etag', [None, 'W/"v1"', 'v1', '"v1", "v2"', '"has space"', '"has\ttab"']) +def test_size_only_requires_strong_etag_before_reading_body(server, tmp_path, etag): + url, requests = server({'etag': etag, 'modified': 'Tue, 01 Sep 2026 00:00:00 GMT'}) + dest = tmp_path / 'data' + dest.write_bytes(b'old') + with pytest.raises(FileValidationError, match='requires a strong ETag'): + fetch(url, dest, expected_sha256=None, force=True) + assert dest.read_bytes() == b'old' + assert len(requests) == 1 + assert (_state_directory(dest) / 'partial').stat().st_size == 0 + + +@pytest.mark.parametrize('status', [200, 206]) +def test_size_only_changed_representation_restarts_with_new_bytes(server, tmp_path, status): + new = b'x' * len(PAYLOAD) + url, requests = server({}, {'status': status, 'etag': '"v2"', 'body': new}, + {'etag': '"v2"', 'body': new}) + dest = tmp_path / 'data' + interrupt(url, dest, expected_sha256=None) + fetch(url, dest, expected_sha256=None, max_retries=0) + assert requests[1]['If-Range'] == '"v1"' + assert dest.read_bytes() == new + if status == 206: + assert 'Range' not in requests[2] + + +@pytest.mark.parametrize('etag', [None, 'W/"v1"']) +def test_size_only_missing_resumed_validator_restarts_then_rejects(server, tmp_path, etag): + url, requests = server({}, {'etag': etag}, {'etag': etag}) + dest = tmp_path / 'data' + dest.write_bytes(b'old') + interrupt(url, dest, expected_sha256=None, force=True) + with pytest.raises(FileValidationError, match='requires a strong ETag'): + fetch(url, dest, expected_sha256=None, force=True, max_retries=0) + assert requests[1]['If-Range'] == '"v1"' + assert 'Range' not in requests[2] + assert dest.read_bytes() == b'old' + assert (_state_directory(dest) / 'partial').stat().st_size == 0 + + +@pytest.mark.parametrize('validator', [ + None, {}, 'not a validator', + {'header': 'Last-Modified', 'value': 'Tue, 01 Sep 2026 00:00:00 GMT'}, + {'header': 'ETag', 'value': 'W/"v1"'}, + {'header': 'ETag', 'value': '"v1"\r\nInjected: yes'}, +]) +def test_size_only_unvalidated_saved_partial_restarts(server, tmp_path, validator): + url, requests = server() + dest = tmp_path / 'data' + interrupt(url, dest, expected_sha256=None) + state = _state_directory(dest) + metadata = json.loads((state / 'metadata.json').read_text()) + metadata['validator'] = validator + (state / 'metadata.json').write_text(json.dumps(metadata)) + (state / 'partial').write_bytes(b'x' * 16) + fetch(url, dest, expected_sha256=None) + assert 'Range' not in requests[1] + assert dest.read_bytes() == PAYLOAD + + +@pytest.mark.parametrize('content', [PAYLOAD, b'x' * len(PAYLOAD), PAYLOAD + b'oversized']) +def test_size_only_complete_partial_always_requests_fresh_bytes(server, tmp_path, content): + new = b'y' * len(PAYLOAD) + url, requests = server({}, {'etag': '"v2"', 'body': new}) + dest = tmp_path / 'data' + interrupt(url, dest, expected_sha256=None) + (_state_directory(dest) / 'partial').write_bytes(content) + fetch(url, dest, expected_sha256=None) + assert len(requests) == 2 + assert 'Range' not in requests[1] + assert dest.read_bytes() == new + + +@pytest.mark.parametrize('etag', ['"empty"', None]) +def test_size_only_empty_download_still_requires_response_validator(server, tmp_path, etag): + url, requests = server({'body': b'', 'etag': etag}) + dest = tmp_path / 'data' + if etag is None: + with pytest.raises(FileValidationError, match='strong ETag'): + fetch(url, dest, expected_sha256=None, expected_size=0) + assert not dest.exists() + else: + fetch(url, dest, expected_sha256=None, expected_size=0) + assert dest.read_bytes() == b'' + assert len(requests) == 1 + + +def test_hash_pinned_download_can_resume_without_validator(server, tmp_path): + url, requests = server({'drop': 24, 'etag': None}, {'etag': None}) + dest = tmp_path / 'data' + fetch(url, dest) + assert requests[1]['Range'] == 'bytes=24-' + assert 'If-Range' not in requests[1] + assert dest.read_bytes() == PAYLOAD + + +@pytest.mark.parametrize('api', ['fetch_file', 'cache']) +@pytest.mark.parametrize('suffix', ['.gz', '.zip', '.html']) +@pytest.mark.parametrize('expected_sha256', [DIGEST, None]) +def test_raw_resume_at_arbitrary_output_name(server, tmp_path, api, suffix, expected_sha256): + url, requests = server({'drop': 16}) + url += suffix + '?token=secret' + options = dict(raw=True, resume=True, expected_sha256=expected_sha256, + expected_size=len(PAYLOAD), chunk_size=8, retry_backoff=0, + record_provenance=True) + if api == 'fetch_file': + call = lambda: fetch_file(url, destination=tmp_path / 'data.csv', **options) + else: + cache = Cache(cache_root=tmp_path) + call = lambda: cache.fetch(url, filename='data.csv', **options) + path = Path(call()) + assert path.name == 'data.csv' + assert path.read_bytes() == PAYLOAD + assert requests[1]['Range'] == 'bytes=16-' + assert requests[1]['If-Range'] == '"v1"' + assert inspect_file(path).recorded_sha256 == expected_sha256 + assert call() == str(path) + assert len(requests) == 2