diff --git a/bench/run.py b/bench/run.py index bead561..529b1c6 100644 --- a/bench/run.py +++ b/bench/run.py @@ -26,7 +26,8 @@ from tabulate import tabulate from compresso import _core -from compresso.introspect.benchmark import BenchmarkResult, print_results +from compresso.cli._render.benchmark import benchmark_results +from compresso.introspect.benchmark import BenchmarkResult REPO_ROOT = Path(__file__).resolve().parent.parent WORK_DIR = REPO_ROOT / "build" / "bench" @@ -326,7 +327,7 @@ def main() -> None: args = parser.parse_args() results = run(args.formats.split(","), args.size_mb, args.repeats, args.cool) - print_results(results) + benchmark_results(results) if args.compare: compare(results, args.compare) diff --git a/pyproject.toml b/pyproject.toml index f12082d..90a517d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,6 @@ dev = [ [project.scripts] compresso = "compresso.cli:main" -comp = "compresso.cli:main" [tool.meson-python.args] # Meson only activates MSVC when no other compiler is on PATH, which the MinGW diff --git a/src/compresso/__init__.py b/src/compresso/__init__.py index f2b1bce..1752459 100644 --- a/src/compresso/__init__.py +++ b/src/compresso/__init__.py @@ -9,6 +9,7 @@ ExtractionPolicyError, HeaderError, IncompleteExtractionError, + SourceRemovalError, TrailingDataWarning, compress_file, decompress_file, @@ -20,8 +21,12 @@ CompressionPlan, DecompressionJob, DecompressionPlan, + Strategy, + VerificationJob, + VerificationPlan, plan_compression, plan_decompression, + plan_verification, ) from .frontend.archive_api import ( ArchiveEntry, @@ -36,7 +41,7 @@ plan_archive, plan_extraction, ) -from .introspect.benchmark import benchmark_file, print_results +from .introspect.benchmark import benchmark_file from .introspect.capabilities import list_capabilities from .introspect.file_inspect import InspectResult, inspect from .introspect.speeds import get_estimated_speeds @@ -67,8 +72,12 @@ "JobResult", "OverwriteMode", "ProgressCallback", + "SourceRemovalError", + "Strategy", "SymlinkMode", "TrailingDataWarning", + "VerificationJob", + "VerificationPlan", "benchmark_file", "compress_file", "decompress_file", @@ -79,5 +88,5 @@ "plan_compression", "plan_decompression", "plan_extraction", - "print_results", + "plan_verification", ] diff --git a/src/compresso/_core.pyi b/src/compresso/_core.pyi index 7400e69..a5e65a5 100644 --- a/src/compresso/_core.pyi +++ b/src/compresso/_core.pyi @@ -27,6 +27,19 @@ class IncompleteExtractionError(Error): Raises on completion, anything extracted is kept. """ +class SourceRemovalError(Error): + """An archive was written, but some of its sources could not be deleted. + + The archive is complete, and every other source is deleted. + + Attributes: + output: The path the archive was written to. + failures: Each path left behind, with the reason. + """ + + output: str + failures: list[tuple[str, str]] + class CorruptDataError(Error): """The input is corrupt, truncated, or not the format it claims to be.""" @@ -153,14 +166,27 @@ def create_archive( threads: int = ..., progress: ProgressFn | None = ..., cancel: CancelToken | None = ..., + include: Sequence[str] = ..., + exclude: Sequence[str] = ..., + remove_sources: bool = ..., + dereference: bool = ..., ) -> str: """Create an archive from the given input paths in the given format. `overwrite` is 0 = error (default), 1 = skip, 2 = overwrite, 3 = rename; - applied to the single destination archive. + applied to the single destination archive; `remove_sources` with skip + raises ValueError. Returns the path actually written, which RENAME may have changed from `output_path`; SKIP returns `output_path` unchanged and writes nothing. + + `include` and `exclude` are globs matched against entry names; an exclude + wins and isn't descended into, and with any includes, only matching entries + and the directories holding them are stored. + + `dereference` stores what symlinks lead to; one that's dangling, or that + leads to a directory above it, stays a link; `remove_sources` then deletes + the links, and not their targets. """ def extract_archive( @@ -179,6 +205,8 @@ def extract_archive( threads: int = ..., progress: ProgressFn | None = ..., cancel: CancelToken | None = ..., + include: Sequence[str] = ..., + exclude: Sequence[str] = ..., ) -> None: """Extract an archive to output_dir; an empty `files` extracts every entry. @@ -187,12 +215,31 @@ def extract_archive( `overwrite` is 0 = error, 1 = skip, 2 = overwrite, 3 = rename; `allow_symlinks` is 0 = deny, 1 = allow if the target stays inside - output_dir, 2 = rewrite to regular files, -1 = the format's default (2 for zip, 0 otherwise); - `max_total_size` and `max_depth` treat 0 as unlimited; + output_dir, 2 = rewrite to regular files, -1 = the format's default (2 for zip, + 0 otherwise); `max_total_size` and `max_depth` treat 0 as unlimited; `overwrite_dir_metadata` also restores the mode and mtime of a directory that already exists, as GNU tar does, except when `overwrite` is 1 (skip); `exact_permissions` restores modes verbatim, where by default setuid/setgid/sticky are dropped and the umask applies. + + `include` and `exclude` are matched as for `create_archive`: as in bsdtar, + an entry is extracted if `files` names it or an include matches it, and an + exclude overrides both; a directory entry is kept only if it matches + itself; an include that no entry matches raises `IncompleteExtractionError`, + as a name in `files` does, after the rest is extracted. + """ + +def verify_file( + path: StrOrBytesPath, + *, + threads: int = ..., + progress: ProgressFn | None = ..., + cancel: CancelToken | None = ..., +) -> None: + """Decode all of an archive or compressed file, writing nothing. + + Raises what extracting it would for damaged data, so it proves only what + the format checks: a plain tar's file data carries no checksum. """ def detect_format(file_path: StrOrBytesPath) -> str: @@ -204,9 +251,22 @@ def gzip_stored_name(file_path: StrOrBytesPath) -> bytes | None: Always None for a non-gzip file. """ +def stored_content_size(file_path: StrOrBytesPath) -> int | None: + """Return the uncompressed size an xz, zstd, lz4 or snappy file states, or None. + + Reads no compressed data, so a damaged file can still state a size; None for + any other format, a zstd or lz4 frame written from a stream, or trailing data. + """ + def format_is_archive(format: str) -> bool: """Return whether a format's container can hold more than one entry.""" +def format_stores_hardlinks(format: str) -> bool: + """Return whether an archive format stores a file's later names as links.""" + +def path_matches(path: str, patterns: Sequence[str]) -> bool: + """Return whether any glob matches an entry path, as `include` and `exclude` do.""" + class _StandaloneFormatDict(TypedDict): """One single file container format, as defined by `standalone_formats`.""" diff --git a/src/compresso/cli/__init__.py b/src/compresso/cli/__init__.py index 84816e8..b4857c3 100644 --- a/src/compresso/cli/__init__.py +++ b/src/compresso/cli/__init__.py @@ -2,7 +2,13 @@ from __future__ import annotations -from . import algos, benchmark, compress, extract, inspect +import io +import re +import sys +from typing import Any, TextIO + +from . import benchmark, compress, extract, formats, info, verify +from . import list as list_ from ._app import app __all__ = ["app", "main"] @@ -11,18 +17,92 @@ _COMMAND_MODULES = ( compress, extract, - inspect, + verify, + list_, + info, + formats, benchmark, - algos, ) for _module in _COMMAND_MODULES: _module.register() +# Styling and spaces, so Rich's padded blank lines count as blank +_INVISIBLE = re.compile(r"\x1b\[[0-9;]*m| ") + + +class _Padded: + """A terminal stream that gives a run's output one blank line above it. + + main() adds the one below. Output that already starts or ends with a blank + line, as `--help` does, gets no extra. + """ + + # Shared, so stdout and stderr pad once between them + tail: str | None = None # the last two visible characters written + last: _Padded | None = None + + def __init__(self, stream: TextIO) -> None: + """Initialise a padded stream. + + Args: + stream: The underlying stream to write to. + """ + self._stream = stream + + def write(self, data: str) -> int: + """Write to the underlying stream, with a blank line above if needed. + + Args: + data: The string to write. + + Returns: + The number of characters written. + """ + # Typer/Click probes for a binary stream with write(b"") + visible = _INVISIBLE.sub("", data) if isinstance(data, str) else "" + if visible: + if _Padded.tail is None and not visible.startswith("\n"): + self._stream.write("\n") + + _Padded.tail = ((_Padded.tail or "") + visible)[-2:] + _Padded.last = self + + return self._stream.write(data) + + def __getattr__(self, name: str) -> Any: + """Delegate attribute access to the underlying stream. + + Args: + name: The name of the attribute to access. + + Returns: + The value of the requested attribute. + """ + return getattr(self._stream, name) + + def main() -> None: """Entry point for the CLI.""" - app() + # A pipe on Windows takes the locale code page, which can't encode markers + # like '→'; escape rather than raise + if isinstance(sys.stdout, io.TextIOWrapper): + sys.stdout.reconfigure(errors="backslashreplace") + + # Not when piped, as tar and gzip don't pad what a script reads + if sys.stdout.isatty(): + sys.stdout = _Padded(sys.stdout) + + if sys.stderr.isatty(): + sys.stderr = _Padded(sys.stderr) + + try: + app() + + finally: + if _Padded.last and _Padded.tail != "\n\n": + _Padded.last.write("\n") if __name__ == "__main__": diff --git a/src/compresso/cli/_dispatch.py b/src/compresso/cli/_dispatch.py deleted file mode 100644 index 484c1bc..0000000 --- a/src/compresso/cli/_dispatch.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Dispatcher for the two smart commands `compress` and `extract`.""" - -from __future__ import annotations - -from pathlib import Path - -from .._core import detect_format, format_is_archive -from ..frontend.api import STANDALONE_FORMATS, canonical_standalone_format - -# What `compress` reaches for when given several inputs and no format -DEFAULT_ARCHIVE_FORMAT = "zip" - - -def infer_format(output: Path | None) -> str | None: - """Infer a format from an output filename. - - Args: - output: The path the user asked to write, or None. - - Returns: - The format named by the suffix, or None if it names none — which - includes `.comp`, whose container is the default anyway. - """ - if output is None: - return None - - # Mirrors the core (`x.tar.lz4` is tar in lz4); unwritable formats - # still match, so the default is never written under their suffix - suffix: str = output.suffix.lower().lstrip(".") - if Path(output.stem).suffix.lower() == ".tar" and format_is_archive( - f"tar.{suffix}" - ): - return f"tar.{suffix}" - - if suffix in STANDALONE_FORMATS or format_is_archive(suffix): - return suffix - - return None - - -def looks_like_archive(path: Path) -> bool: - """Return whether `path`'s own bytes say it holds many entries. - - Args: - path: The path to check. - - Returns: - True if the path looks like an archive, False otherwise. - """ - try: - return bool(format_is_archive(detect_format(path))) - - except Exception: - return False - - -def is_archive_format(fmt: str | None) -> bool: - """Return whether `fmt` names a container that holds many entries. - - Args: - fmt: The format string to check. - - Returns: - True if `fmt` names an archive format, False otherwise. - """ - # Includes currently unwritable formats (e.g. 7z) - return fmt is not None and format_is_archive(fmt.lower()) - - -def default_archive_output(first_input: Path, fmt: str) -> Path: - """Name an archive the caller did not name. - - Args: - first_input: The first source path, which the archive is named after. - fmt: The archive format, used as the suffix. - - Returns: - A path beside the input, e.g. `src/` in tar.zst becomes `src.tar.zst`. - """ - # A trailing separator would otherwise make the name empty - stem: str = first_input.resolve().name - return first_input.resolve().parent / f"{stem}.{fmt}" - - -def resolve_single_output(output: Path | None, default: Path) -> Path: - """Work out where a single-file job should write. - - `-o` means a directory for an archive, but for one file it is the file itself; - an existing directory receives the file, anything else is the file. - - Args: - output: The `-o` value, or None. - default: Where the job would write if not told otherwise. - - Returns: - The destination path. - """ - if output is None: - return default - - if output.is_dir(): - return output / default.name - - return output - - -def format_is_single_file(fmt: str | None) -> bool: - """Return whether `fmt` names a container holding exactly one file. - - Args: - fmt: The format string to check. - - Returns: - True if `fmt` names a single-file format, False otherwise. - """ - return canonical_standalone_format(fmt) is not None diff --git a/src/compresso/cli/_render.py b/src/compresso/cli/_render/__init__.py similarity index 60% rename from src/compresso/cli/_render.py rename to src/compresso/cli/_render/__init__.py index 1a598d4..3ccf82f 100644 --- a/src/compresso/cli/_render.py +++ b/src/compresso/cli/_render/__init__.py @@ -1,14 +1,21 @@ -"""Shared rendering for the CLI: formatting, progress and exit codes.""" +"""Everything the CLI prints. + +The package root holds formatting, progress and exit codes; `jobs` and `info` hold +each command's reports. +""" from __future__ import annotations +import functools import sys -from collections.abc import Generator +from collections.abc import Callable, Generator from contextlib import contextmanager -from typing import NoReturn +from pathlib import Path +from typing import NoReturn, ParamSpec -from ..frontend._job import JobResult, ProgressCallback -from ._app import app +from ..._core import Error +from ...frontend._job import ProgressCallback +from .._app import app # Exit codes EXIT_OK = 0 @@ -60,6 +67,28 @@ def format_time(seconds: float) -> str: return f"{mins}m {secs:.1f}s" +def report_error(message: str) -> None: + """Print an error to stderr. + + Args: + message: Text to show, without the leading marker. + """ + # A piped stdout is block-buffered, so without this an error could + # land before the outputs ahead of it + sys.stdout.flush() + app.echo(message=app.style(text=f"✗ {message}", fg="red"), err=True) + + +def report_warning(message: str) -> None: + """Print a warning to stderr. + + Args: + message: Text to show, without the leading marker. + """ + sys.stdout.flush() + app.echo(message=app.style(text=f"! {message}", fg="yellow"), err=True) + + def fail(message: str, code: int = EXIT_FAILED) -> NoReturn: """Print an error to stderr and exit. @@ -67,7 +96,7 @@ def fail(message: str, code: int = EXIT_FAILED) -> NoReturn: message: Text to show, without the leading marker. code: Process exit code; see the EXIT_* constants. """ - app.echo(message=app.style(text=f"✗ {message}", fg="red"), err=True) + report_error(message) sys.exit(code) @@ -80,6 +109,19 @@ def succeed(message: str) -> None: app.echo(message=app.style(text=f"✓ {message}", fg="green")) +def input_header(path: Path, *, first: bool) -> None: + """Print the `==> name <==` line that head puts before each of several inputs. + + Args: + path: The input, as given. + first: False puts a blank line above, to separate it from the last. + """ + if not first: + print() + + print(f"==> {path} <==") + + def cancelled(what: str) -> NoReturn: """Report an interrupted operation and exit with the SIGINT code. @@ -90,21 +132,37 @@ def cancelled(what: str) -> NoReturn: sys.exit(EXIT_CANCELLED) -def exit_for_result(result: JobResult, what: str) -> None: - """Exit with the right code if `result` did not succeed. +P = ParamSpec("P") - A cancelled job is not a failure, so it gets the conventional SIGINT - code rather than the generic failure one. + +def handles_errors(what: str) -> Callable[[Callable[P, None]], Callable[P, None]]: + """Give a command the standard Ctrl-C and error exits. Args: - result: The outcome of a frontend job. - what: Operation name for the message, e.g. "Compression". + what: Operation name for the messages, e.g. "Compression". + + Returns: + A decorator wrapping the command. """ - if result.cancelled: - cancelled(what) - if not result.ok: - fail(f"\n{what} failed: {result.error}") + def decorate(fn: Callable[P, None]) -> Callable[P, None]: + @functools.wraps(fn) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> None: + try: + fn(*args, **kwargs) + + except KeyboardInterrupt: + cancelled(what) + + except Error as e: + fail(f"{what} error: {e}") + + except Exception as e: + fail(f"Unexpected error: {e}") + + return wrapper + + return decorate @contextmanager diff --git a/src/compresso/cli/_render/benchmark.py b/src/compresso/cli/_render/benchmark.py new file mode 100644 index 0000000..a96864e --- /dev/null +++ b/src/compresso/cli/_render/benchmark.py @@ -0,0 +1,77 @@ +"""What `benchmark` prints: the run's settings, then a table of results.""" + +from __future__ import annotations + +from collections.abc import Sequence +from pathlib import Path + +from ...introspect.benchmark import BenchmarkResult + + +def _level_name(level: int | None) -> str | int: + return "auto" if level is None else level + + +def benchmark_header( + file: Path, + repeats: int, + algos: Sequence[str] | None, + strategies: Sequence[str] | None, + levels: list[int | None], +) -> None: + """Print what the run covers, then a blank line. + + Args: + file: The file being benchmarked. + repeats: How many times each combination runs. + algos: The algorithms asked for, or None for all. + strategies: The strategies asked for, or None for all. + levels: The levels asked for, None meaning the strategy's own. + """ + lines: list[str] = [f"Running benchmarks on: {file}", f"Repeats: {repeats}"] + if algos: + lines.append(f"Algorithms: {', '.join(algos)}") + + if strategies: + lines.append(f"Strategies: {', '.join(strategies)}") + + lines.append(f"Levels: {', '.join(str(_level_name(lv)) for lv in levels)}") + print("\n".join(lines) + "\n") + + +def benchmark_results(results: list[BenchmarkResult]) -> None: + """Print one table row per algorithm, strategy and level. + + Args: + results: The benchmark's results, in run order. + """ + headers: list[str] = [ + "Algo", + "Strategy", + "Level", + "Comp Time (s)", + "Decomp Time (s)", + "MB/s (Comp)", + "MB/s (Decomp)", + "Ratio (comp/orig)", + ] + + table_data: list[list[str | int]] = [ + [ + r.algo, + r.strategy, + _level_name(r.level), + f"{r.compress_time:.4f}", + f"{r.decompress_time:.4f}", + f"{r.comp_mb_s:.2f}", + f"{r.decomp_mb_s:.2f}", + f"{r.ratio:.3f}", + ] + for r in results + ] + + # Deferred: tabulate pulls in wcwidth, which every CLI command would + # otherwise pay for + from tabulate import tabulate + + print(tabulate(tabular_data=table_data, headers=headers, tablefmt="grid")) diff --git a/src/compresso/cli/_render/formats.py b/src/compresso/cli/_render/formats.py new file mode 100644 index 0000000..432c94d --- /dev/null +++ b/src/compresso/cli/_render/formats.py @@ -0,0 +1,71 @@ +"""What `formats` prints: the algorithms and formats this build can write.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ...introspect.capabilities import BackendCapabilities +from .._app import app + +if TYPE_CHECKING: + from ..._core import _StandaloneFormatDict + + +def _heading(text: str, count: int) -> None: + """Print a section title with its item count, then a blank line. + + Args: + text: The section title, e.g. `Native algorithms`. + count: How many items the section lists. + """ + app.echo(message=f"{text}: {count}\n") + + +def _item(name: str, details: dict[str, str]) -> None: + """Print a bulleted name, then its details as indented `label: value` lines. + + The values line up in one column, and empty `details` prints only the name. + + Args: + name: The algorithm or format name, shown in bold green. + details: Labels mapped to their values, printed in order. + """ + app.echo(message=app.style(text=f"- {name}", fg="green", bold=True)) + width = max(map(len, details), default=0) + 2 + for label, value in details.items(): + app.echo(message=f" {label + ':':<{width}}{value}") + + app.echo() + + +def list_formats( + algos: list[BackendCapabilities], + standalone: list[_StandaloneFormatDict], + archives: dict[str, list[str]], +) -> None: + """Print the .comp algorithms, then the standalone and archive formats. + + Args: + algos: The backends a .comp container can use. + standalone: The single-file container formats. + archives: Each archive format, mapped to the compressed variants + written as `.`, e.g. `tar.zst`. + """ + _heading("Native algorithms", len(algos)) + for cap in algos: + levels = "none" if cap.min_level is None else f"{cap.min_level}-{cap.max_level}" + _item(cap.name, {"ID": str(cap.id), "Levels": levels}) + + _heading("Standalone formats", len(standalone)) + for fmt in standalone: + _item( + fmt["name"], + { + "Aliases": ", ".join(fmt["aliases"]) or "none", + "Extensions": ", ".join(f".{ext}" for ext in fmt["exts"]), + }, + ) + + _heading("Archive formats", len(archives)) + for name, compressed in archives.items(): + _item(name, {"Compressed": ", ".join(compressed)} if compressed else {}) diff --git a/src/compresso/cli/_render/info.py b/src/compresso/cli/_render/info.py new file mode 100644 index 0000000..1d3903a --- /dev/null +++ b/src/compresso/cli/_render/info.py @@ -0,0 +1,293 @@ +"""What describes an archive or a single-file container: its entries and metadata.""" + +from __future__ import annotations + +import stat +import time +from pathlib import Path + +import orjson + +from ...frontend.archive_api import ArchiveEntry +from ...introspect.file_inspect import InspectResult +from .._app import app +from . import format_size, format_time, succeed + + +def _format_mtime(mtime: float) -> str: + """Format a modification time for display. + + Args: + mtime: Seconds since the epoch. + + Returns: + The local date and time, to the minute. + """ + return time.strftime("%Y-%m-%d %H:%M", time.localtime(mtime)) + + +def list_entries(entries: list[ArchiveEntry], *, long: bool = False) -> None: + """Print one line per entry: its size, then its path. + + Directories and symlinks leave the size blank, a symlink is followed by + its target, and each entry is indented once for every directory above it + that was already listed. + + Args: + entries: The archive's entries, in archive order. + long: If True, print each entry's full detail line instead, unindented. + """ + if long: + for entry in entries: + print(_entry_detail_line(entry)) + + return + + listed_dirs: set[str] = set() + + for entry in entries: + parts = entry.path.rstrip("/").split("/") + depth = sum("/".join(parts[:i]) in listed_dirs for i in range(1, len(parts))) + + if entry.is_dir: + listed_dirs.add("/".join(parts)) + + # A symlink's stored size is the length of its target; hardlink's have no data + size = ( + "" + if entry.is_dir or entry.is_symlink or entry.is_hardlink + else format_size(size_bytes=entry.size) + ) + target = f" -> {entry.link_target}" if entry.is_symlink else "" + + # "1023.99 KB" is the widest format_size gives + print(f"{size:>10} {' ' * depth}{entry.path}{target}") + + +def _entry_detail_line(entry: ArchiveEntry) -> str: + """Format one entry's full detail: mode, mtime, size, path, and target. + + Args: + entry: The archive entry to format. + + Returns: + A single formatted line for the entry. + """ + if entry.is_dir: + type_bit = stat.S_IFDIR + elif entry.is_symlink: + type_bit = stat.S_IFLNK + else: + type_bit = stat.S_IFREG + + mode = stat.filemode(type_bit | entry.mode) + mtime = _format_mtime(entry.mtime) + has_no_data = entry.is_dir or entry.is_symlink or entry.is_hardlink + size = "" if has_no_data else format_size(size_bytes=entry.size) + + if entry.is_symlink: + target = f" -> {entry.link_target}" + elif entry.is_hardlink: + target = f" link to {entry.link_target}" + else: + target = "" + + detail = "" + if entry.compressed_size is not None: + detail = f" (compressed {format_size(size_bytes=entry.compressed_size)}" + if entry.crc is not None: + detail += f", crc {entry.crc:08x}" + if entry.method is not None: + detail += f", method {entry.method}" + detail += ")" + + return f"{mode} {mtime} {size:>10} {entry.path}{target}{detail}" + + +def print_json(items: list[dict[str, object]]) -> None: + """Print a JSON array, one item per input. + + Args: + items: Each input's description. + """ + print(orjson.dumps(items, option=orjson.OPT_INDENT_2).decode()) + + +def archive_info( + file: Path, + fmt: str, + entries: list[ArchiveEntry], + *, + as_json: bool, +) -> dict[str, object]: + """Describe an archive, printing the text form unless JSON is wanted. + + Args: + file: The archive's path. + fmt: The archive's detected format. + entries: The archive's entries. + as_json: If True, print nothing; the caller prints the returned item. + + Returns: + The archive's JSON item. + """ + total_size = sum(entry.size for entry in entries) + archive_size: int = file.stat().st_size + data: dict[str, object] = { + "path": str(file), + "format": fmt, + "is_archive": True, + "entry_count": len(entries), + "total_size": total_size, + "archive_size": archive_size, + } + + if as_json: + return data + + lines: list[str] = [ + f"File: {file}", + f"Format: {fmt}", + f"Entries: {len(entries)}", + f"Total size: {format_size(size_bytes=total_size)}", + f"Archive size: {format_size(size_bytes=archive_size)}", + ] + + if total_size: + ratio: int | float = (archive_size / total_size) * 100 + lines.append(f"Compression: {ratio:.1f}% of original") + + print("\n".join(lines)) + + return data + + +def codec_info( + file: Path, + fmt: str, + *, + stored_name: str | None = None, + mtime: int | None = None, + orig_size: int | None = None, + as_json: bool, +) -> dict[str, object]: + """Describe a standalone codec file, e.g. a `.gz`, by its format and size. + + Args: + file: The file's path. + fmt: The file's detected format. + stored_name: The name its header stores, if any. + mtime: The modification time its header stores, if any. + orig_size: The uncompressed size its metadata states, if any. + as_json: If True, print nothing; the caller prints the returned item. + + Returns: + The file's JSON item. + """ + size: int = file.stat().st_size + data: dict[str, object] = { + "path": str(file), + "format": fmt, + "size": size, + "stored_name": stored_name, + "mtime": mtime, + "orig_size": orig_size, + } + + if as_json: + return data + + lines: list[str] = [ + f"File: {file}", + f"Format: {fmt}", + f"Size: {format_size(size_bytes=size)}", + ] + + if stored_name is not None: + lines.append(f"Stored name: {stored_name}") + + if mtime is not None: + lines.append(f"Modified: {_format_mtime(mtime)}") + + if orig_size is not None: + lines.append(f"Original size: {format_size(size_bytes=orig_size)}") + + if orig_size: + lines.append(f"Compression: {size / orig_size * 100:.1f}% of original") + + print("\n".join(lines)) + + return data + + +def file_info(result: InspectResult, *, as_json: bool) -> dict[str, object]: + """Describe a `.comp` file with a valid header from its inspection. + + Args: + result: The container's inspection. + as_json: If True, print nothing; the caller prints the returned item. + + Returns: + The file's JSON item. + """ + data: dict[str, object] = { + "path": str(object=result.path), + "format": "compresso", + "is_compresso": result.is_compresso, + "header_ok": result.header_ok, + "version": result.version, + "algo_id": result.algo_id, + "algo_name": result.algo_name, + "level": result.level, + "flags": result.flags, + "orig_size": result.orig_size, + "backend_available": result.backend_available, + "can_decompress": result.can_decompress, + "estimated_decomp_s": result.estimated_decomp_s, + "reason": result.reason, + } + + if as_json: + return data + + print(f"File: {result.path}\n") + succeed("Valid Compresso file\n") + + lines: list[str] = [ + f"Algorithm: {result.algo_name or 'Unknown'} (ID: {result.algo_id})", + f"Version: {result.version}", + "", + ] + + if result.level is not None: + lines.append(f"Level: {result.level}") + else: + lines.append("Level: auto") + + if result.orig_size: + lines.append(f"Original size: {format_size(size_bytes=result.orig_size)}") + + compressed_size: int = result.path.stat().st_size + lines.append(f"Compressed size: {format_size(size_bytes=compressed_size)}") + + if result.orig_size: + ratio: int | float = (compressed_size / result.orig_size) * 100 + lines.append(f"Compression: {ratio:.1f}% of original") + + lines += [ + "", + f"Backend available: {'Yes' if result.backend_available else 'No'}", + f"Can decompress: {'Yes' if result.can_decompress else 'No'}", + ] + + if result.estimated_decomp_s: + lines.append( + f"Est. decomp time: {format_time(seconds=result.estimated_decomp_s)}" + ) + + print("\n".join(lines)) + + if not result.can_decompress and result.reason: + app.echo(message=app.style(text=f"\n⚠ {result.reason}", fg="yellow")) + + return data diff --git a/src/compresso/cli/_render/jobs.py b/src/compresso/cli/_render/jobs.py new file mode 100644 index 0000000..6683507 --- /dev/null +++ b/src/compresso/cli/_render/jobs.py @@ -0,0 +1,202 @@ +"""What a compress, archive, extract or decompress job prints before and after.""" + +from __future__ import annotations + +from pathlib import Path + +from ...frontend.api import CompressionPlan, DecompressionPlan +from ...frontend.archive_api import ArchivePlan, ExtractPlan +from . import format_size, format_time, succeed + + +def summary( + name: Path, + before: int, + after: int, + elapsed: float, + renamed: Path | None, + *, + ratio: bool = True, +) -> None: + """Print the one line a finished input gets by default. + + Args: + name: The output for compress, the input for extract. + before: Bytes read. + after: Bytes written. + elapsed: Seconds the job took. + renamed: Where the output went instead, if its name was taken. + ratio: Whether to show the (de)compression ratio in the summary. + """ + percent: str = f" ({after / before * 100:.0f}%)" if ratio and before else "" + note: str = f", output renamed to {renamed}" if renamed else "" + print( + f"{name} {format_size(before)} → {format_size(after)}{percent} " + f"in {format_time(elapsed)}{note}\n" + ) + + +def skipped(existing: Path) -> None: + """Print the line an input gets when `--on-conflict skip` left its output alone. + + Args: + existing: The output that was already there. + """ + print(f"{existing} skipped, already exists\n") + + +def compress_plan(plan: CompressionPlan) -> None: + """Print what a single-file compression is about to do. + + Args: + plan: The planned compression. + """ + lines: list[str] = [ + f"Compressing: {plan.src}", + f"Output: {plan.dest}", + f"Algorithm: {plan.backend_name}", + f"Input size: {format_size(size_bytes=plan.input_size)}", + ] + + if not plan.options.format: + lines.append(f"Strategy: {plan.options.strategy}") + + if plan.options.level is not None: + lines.append(f"Level: {plan.options.level}") + + print("\n".join(lines) + "\n") + + +def compress_result(plan: CompressionPlan, written: Path, elapsed: float) -> None: + """Print a finished single-file compression's stats. + + Args: + plan: The plan as it was before the job ran. + written: The path actually written, which a rename may have moved. + elapsed: Seconds the job took. + """ + compressed_size: int = written.stat().st_size + ratio: int | float = ( + (compressed_size / plan.input_size) * 100 if plan.input_size > 0 else 0 + ) + + succeed("Compression successful!\n") + if written != plan.dest: + print(f" Renamed to: {written}") + + print( + f" Output size: {format_size(size_bytes=compressed_size)} ({ratio:.1f}%)\n" + f" Time: {format_time(seconds=elapsed)}\n" + ) + + +def archive_plan(plan: ArchivePlan) -> None: + """Print what an archive job is about to do. + + Args: + plan: The planned archive. + """ + print( + f"Archiving: {plan.entry_count} source(s)\n" + f"Output: {plan.output}\n" + f"Format: {plan.options.format}\n" + f"Input size: {format_size(size_bytes=plan.total_input_size)}\n" + ) + + +def archive_result(plan: ArchivePlan, written: Path, elapsed: float) -> None: + """Print a finished archive's stats. + + Args: + plan: The plan as it was before the job ran. + written: The path actually written, which a rename may have moved. + elapsed: Seconds the job took. + """ + compressed_size: int = written.stat().st_size + ratio: int | float = ( + (compressed_size / plan.total_input_size) * 100 + if plan.total_input_size > 0 + else 0 + ) + + succeed("Archive created!\n") + if written != plan.output: + print(f" Renamed to: {written}") + + print( + f" Archive size: {format_size(size_bytes=written.stat().st_size)} ({ratio:.1f}%)\n" + f" Time: {format_time(seconds=elapsed)}\n", + ) + + +def extract_plan(plan: ExtractPlan) -> None: + """Print what an extraction is about to do. + + Args: + plan: The planned extraction. + """ + print( + f"Extracting: {plan.archive}\n" + f"Output dir: {plan.output_dir}\n" + f"Entries: {len(plan.entries)}\n" + ) + + +def extract_result(plan: ExtractPlan, elapsed: float) -> None: + """Print a finished extraction's stats. + + Args: + plan: The extraction's plan. + elapsed: Seconds the job took. + """ + succeed("Extraction successful!\n") + print( + f" Entries: {len(plan.entries)}\n Time: {format_time(seconds=elapsed)}\n" + ) + + +def decompress_plan(plan: DecompressionPlan) -> None: + """Print what a single-file decompression is about to do. + + Args: + plan: The planned decompression. + """ + lines: list[str] = [ + f"Decompressing: {plan.src}", + f"Output: {plan.dest}", + f"Format: {plan.format or plan.inspection.algo_name}", + ] + if plan.inspection.orig_size: + lines.append( + f"Original size: {format_size(size_bytes=plan.inspection.orig_size)}" + ) + + print("\n".join(lines) + "\n") + + +def decompress_result( + plan: DecompressionPlan, written: Path, compressed_size: int, elapsed: float +) -> None: + """Print a finished single-file decompression's stats. + + Args: + plan: The plan as it was before the job ran. + written: The path actually written, which a rename may have moved. + compressed_size: The source's size in bytes. + elapsed: Seconds the job took. + """ + decompressed_size: int = written.stat().st_size + speed_mbs: int | float = ( + (decompressed_size / (1024 * 1024)) / elapsed if elapsed > 0 else 0 + ) + + succeed("Decompression successful!\n") + if written != plan.dest: + print(f" Renamed to: {written}") + + print( + f" Compressed size: {format_size(size_bytes=compressed_size)}\n" + f" Decompressed size: {format_size(size_bytes=decompressed_size)}\n" + f" Time: {format_time(seconds=elapsed)}\n" + f" Speed: {speed_mbs:.2f} MB/s\n" + ) diff --git a/src/compresso/cli/_run.py b/src/compresso/cli/_run.py new file mode 100644 index 0000000..37fcc17 --- /dev/null +++ b/src/compresso/cli/_run.py @@ -0,0 +1,183 @@ +"""Running a command's jobs one input at a time, carrying on past a failure.""" + +from __future__ import annotations + +import os +import time +from collections.abc import Callable, Iterable +from pathlib import Path +from typing import Any, TypeVar + +from .._core import Error, SourceRemovalError +from ..frontend._job import JobResult, ThreadedJob +from ..frontend.archive_api import OverwriteMode +from ._render import ( + BAR_THRESHOLD, + EXIT_FAILED, + EXIT_OK, + EXIT_USAGE, + cancelled, + fail, + progress_bar, + report_error, + report_warning, +) + +J = TypeVar("J") + + +class InputFailed(Exception): + """One input failed; the runner reports it and goes on to the next.""" + + +class NotRemoved(Exception): + """An output is written, but `--rm` left inputs; args are (path, reason).""" + + +def require_files(paths: Iterable[Path]) -> None: + """Exit 2 before any input runs if one is missing or not a file. + + A stat shows these; what only an input's bytes show fails that input alone. + + Args: + paths: The inputs, as given. + """ + for path in paths: + if not path.is_file(): + fail(f"Error: {path} does not exist or is not a file", EXIT_USAGE) + + +def conflict_mode( + on_conflict: OverwriteMode | None, force: bool, remove: bool +) -> OverwriteMode: + """Resolve `--on-conflict` and `-f` into one mode, exiting 2 if they disagree. + + Args: + on_conflict: The mode given, or None if it was not. + force: Whether `-f`, shorthand for `--on-conflict overwrite`, was given. + remove: Whether `--rm`, which can't follow an output that was skipped, + was given. + + Returns: + The mode to run with; RENAME if neither was given. + """ + if not force: + if remove and on_conflict is OverwriteMode.SKIP: + fail( + "Error: --rm contradicts --on-conflict skip; " + "use --on-conflict error to keep inputs whose output exists", + EXIT_USAGE, + ) + + return on_conflict or OverwriteMode.RENAME + + if on_conflict not in (None, OverwriteMode.OVERWRITE): + fail(f"Error: -f contradicts --on-conflict {on_conflict}", EXIT_USAGE) + + return OverwriteMode.OVERWRITE + + +def remove_source(source: Path, before: os.stat_result) -> None: + """Delete an input now its output is written when '--rm' flag is used. + + Args: + source: The input. + before: Its stat from before the job; anything else at that path now + is the output, written over it, so it stays. + + Raises: + NotRemoved: If the input can't be deleted. + """ + try: + if os.path.samestat(source.stat(), before): + source.unlink() + + except OSError as e: + raise NotRemoved((str(source), e.strerror or str(e))) from e + + +def run_job( + job: ThreadedJob[Any], + source: Path, + label: str, + total: int, + *, + quiet: bool, + what: str, + verbose: bool = False, +) -> tuple[float, JobResult]: + """Run `job` under a progress bar. + + Args: + job: The planned job, already checked as runnable. + source: The input, named in the error if the job fails. + label: Text shown beside the bar. + total: Expected byte count; small jobs get no bar. + quiet: Whether to suppress the bar. + what: Operation name for messages, e.g. "Compression". + verbose: Whether a blank line follows the bar, to set it apart from + the stats block. + + Returns: + Seconds the job took, and its result; a `SourceRemovalError` in that + result is left for the caller, as the output is still there to report. + + Raises: + InputFailed: If the job failed. + """ + start_time: float = time.time() + + bar: bool = not quiet and total > BAR_THRESHOLD + with progress_bar(label, total, enabled=bar) as on_progress: + result = job.run(progress=on_progress) + + if bar and verbose: + print() + + elapsed: float = time.time() - start_time + + if result.cancelled: + cancelled(what) + + if not result.ok and not isinstance(result.error, SourceRemovalError): + raise InputFailed(f"{source}: {what} failed: {result.error}") + + return elapsed, result + + +def run_all(jobs: Iterable[J], run_one: Callable[[J], None], what: str) -> int: + """Run each job in turn, reporting a failed one and going on to the next. + + Args: + jobs: The planned jobs, one per input. + run_one: Runs one job, raising `InputFailed`, `Error` or `OSError` if + it fails; anything else is a bug and stops the run. + what: Operation name for messages, e.g. "Compression". + + Returns: + EXIT_FAILED if any job failed, else EXIT_OK. + """ + failed = False + for job in jobs: + try: + run_one(job) + + except InputFailed as e: + report_error(str(e)) + failed = True + + except NotRemoved as e: + for path, reason in e.args: + report_warning(f"{path}: not removed: {reason}") + failed = True + + except Error as e: + report_error(f"{what} error: {e}") + failed = True + + # As gzip does, e.g. an input removed or made unreadable since the stat + except OSError as e: + report_error(f"{e.filename}: {e.strerror}" if e.filename else str(e)) + failed = True + + return EXIT_FAILED if failed else EXIT_OK diff --git a/src/compresso/cli/algos.py b/src/compresso/cli/algos.py deleted file mode 100644 index 68e81de..0000000 --- a/src/compresso/cli/algos.py +++ /dev/null @@ -1,37 +0,0 @@ -"""The `list` command, showing the available compression backends.""" - -from __future__ import annotations - -from ..introspect.capabilities import list_capabilities -from ._app import app -from ._render import cancelled, fail - - -def list_algos() -> None: - """List all available compression algorithms.""" - try: - caps = list_capabilities() - - if not caps: - fail("No compression backends available") - - print(f"Available compression algorithms: {len(caps)}\n") - - for cap in caps: - app.echo(message=app.style(text=f"- {cap.name}", fg="green", bold=True)) - levels: str = ( - "none" if cap.min_level is None else f"{cap.min_level}-{cap.max_level}" - ) - app.echo(message=app.style(text=f" ID: {cap.id}")) - app.echo(message=app.style(text=f" Levels: {levels}\n")) - - except KeyboardInterrupt: - cancelled("Listing") - - except Exception as e: - fail(f"Error listing algorithms: {e}") - - -def register() -> None: - """Attach this command to the app.""" - app.command(name="list", aliases=["l", "ls"])(list_algos) diff --git a/src/compresso/cli/benchmark.py b/src/compresso/cli/benchmark.py index 64449f9..d394085 100644 --- a/src/compresso/cli/benchmark.py +++ b/src/compresso/cli/benchmark.py @@ -2,33 +2,77 @@ from __future__ import annotations +from enum import StrEnum from pathlib import Path from typing import Annotated -from ..introspect.benchmark import benchmark_file, print_results +from ..frontend.api import Strategy +from ..introspect.benchmark import benchmark_file +from ..introspect.capabilities import list_capabilities from ._app import app -from ._render import EXIT_USAGE, cancelled, fail, succeed +from ._render import EXIT_USAGE, fail, handles_errors, succeed +from ._render.benchmark import benchmark_header, benchmark_results +# Built from the core so `--help` lists only what this build compiled in +Algo = StrEnum("Algo", [c.name for c in list_capabilities()]) + +def _parse_level(value: str) -> int | None: + """Parse one `-l` value, exiting 2 if it's neither a number nor `auto`. + + Args: + value: The text given. + + Returns: + The level, or None for `auto`, the strategy's own. + """ + if value.lower() == "auto": + return None + + try: + return int(value) + + except ValueError: + fail(f"Error: Invalid level {value!r}; use an integer or 'auto'", EXIT_USAGE) + + +@handles_errors("Benchmark") def benchmark( file: Annotated[Path, app.Argument(help="File to benchmark")], - algos: Annotated[ - str | None, app.Option("--algos", help="Comma-separated list of algorithms") - ] = "all", - strategies: Annotated[ - str | None, - app.Option("--strategies", help="Comma-separated list of strategies"), - ] = "all", - levels: Annotated[ - str | None, + algo: Annotated[ + list[Algo] | None, app.Option( - "--levels", - help="Comma-separated list of levels, or 'auto'; a level outside an " + "--algo", + "-a", + case_sensitive=False, + help="Algorithm to run; repeatable (default: all)", + ), + ] = None, + strategy: Annotated[ + list[Strategy] | None, + app.Option( + "--strategy", + "-s", + case_sensitive=False, + help="Strategy to run; repeatable (default: all)", + ), + ] = None, + level: Annotated[ + list[str] | None, + app.Option( + "--level", + "-l", + metavar="N|auto", + help="Level to run, or 'auto'; repeatable; a level outside an " "algorithm's range is skipped for that algorithm", + show_default="auto", ), - ] = "auto", + ] = None, repeats: Annotated[ - int, app.Option("--repeats", help="Number of times to repeat each benchmark") + int, + app.Option( + "--repeats", "-r", help="Number of times to repeat each benchmark", min=1 + ), ] = 1, temp_dir: Annotated[ Path | None, @@ -47,90 +91,41 @@ def benchmark( Args: file: The path to the file to benchmark. - algos: Comma-separated list of algorithms to use (default: all available). - strategies: Comma-separated list of strategies to use (default: all). - levels: Comma-separated list of levels to use (default: all). + algo: Algorithms to run (default: all available). + strategy: Strategies to run (default: all). + level: Levels to run, or `auto` for the strategy's own (default: auto). repeats: Number of times to repeat each benchmark (default: 1). temp_dir: Temporary directory for benchmark files (default: None). update_cache: If True, update the speed estimates cache with benchmark results (default: False). """ + levels: list[int | None] = [_parse_level(v) for v in level or ["auto"]] + try: - algo_list: list[str] = [] - if algos and algos.lower() != "all": - algo_list = [a.strip() for a in algos.split(",") if a.strip()] - - strategy_list: list[str] = [] - if strategies and strategies.lower() != "all": - strategy_list = [s.strip() for s in strategies.split(",") if s.strip()] - - level_list: list[int | None] = [] - if levels: - for level in levels.split(sep=","): - level: str = level.strip() - if not level: - continue - - if level.lower() in ("auto", "default"): - level_list.append(None) - - else: - try: - level_list.append(int(level)) - - except ValueError: - fail( - f"Invalid level: {level}. Use integers or 'auto'", - EXIT_USAGE, - ) - - lines: list[str] = [ - f"Running benchmarks on: {file}", - f"Repeats: {repeats}", - ] - if algo_list: - lines.append(f"Algorithms: {', '.join(algo_list)}") - - if strategy_list: - lines.append(f"Strategies: {', '.join(strategy_list)}") - - if level_list is not None: - level_str: str = ", ".join( - str(object=level) if level is not None else "auto" - for level in level_list - ) - lines.append(f"Levels: {level_str}") - print("\n".join(lines) + "\n") + benchmark_header(file, repeats, algo, strategy, levels) results = benchmark_file( src=file, - algos=algo_list or None, - strategies=strategy_list or None, - levels=level_list or None, + algos=algo, + strategies=strategy, + levels=levels, repeats=repeats, temp_dir=temp_dir, update_cache=update_cache, ) if not results: - fail("No benchmark results generated\n") + fail("No benchmark results generated") - print_results(results) + benchmark_results(results) if update_cache: print() - succeed("Speed cache updated\n") - print() - - except KeyboardInterrupt: - cancelled("Benchmark") + succeed("Speed cache updated") except FileNotFoundError as e: - fail(f"File not found: {e}\n", EXIT_USAGE) - - except Exception as e: - fail(f"Benchmark error: {e}\n") + fail(f"File not found: {e}", EXIT_USAGE) def register() -> None: """Attach this command to the app.""" - app.command(aliases=["b", "bench"])(benchmark) + app.command(aliases=["b"])(benchmark) diff --git a/src/compresso/cli/compress.py b/src/compresso/cli/compress.py index 00f53a4..0125feb 100644 --- a/src/compresso/cli/compress.py +++ b/src/compresso/cli/compress.py @@ -1,208 +1,136 @@ -"""The `compress` command, which also answers to `archive`.""" +"""The `compress` command, packing files into archives and compressed files.""" from __future__ import annotations -import time +import sys +from enum import StrEnum from pathlib import Path from typing import Annotated -from .._core import BackendError, Error, HeaderError -from ..frontend.api import CompressionJob, CompressionOptions -from ..frontend.archive_api import ArchiveJob, ArchiveOptions, OverwriteMode +from .._core import SourceRemovalError +from ..frontend.api import CompressionJob, Strategy +from ..frontend.archive_api import ArchiveJob, OverwriteMode +from ..frontend.dispatch import CompressRequest, plan_compress +from ..introspect.capabilities import list_capabilities from ._app import app -from ._dispatch import ( - DEFAULT_ARCHIVE_FORMAT, - default_archive_output, - format_is_single_file, - infer_format, - is_archive_format, - resolve_single_output, -) -from ._render import ( - BAR_THRESHOLD, - EXIT_USAGE, - cancelled, - exit_for_result, - fail, - format_size, - format_time, - progress_bar, - succeed, +from ._render import EXIT_USAGE, fail, handles_errors +from ._render.jobs import ( + archive_plan, + archive_result, + compress_plan, + compress_result, + skipped, + summary, ) +from ._run import NotRemoved, conflict_mode, remove_source, run_all, run_job + +# Built from the core so `--help` lists only what this build compiled in +Algo = StrEnum("Algo", ["auto"] + [c.name for c in list_capabilities()]) def _compress_one_file( - source: Path, - output: Path | None, - fmt: str | None, - algo: str | None, - strategy: str, - level: int | None, - overwrite: OverwriteMode, - threads: int, - quiet: bool, + job: CompressionJob, quiet: bool, verbose: bool, remove: bool ) -> None: """Compress a single file into a one-file container. Args: - source: The path to the file to compress. - output: The path to the output file or directory. - fmt: The format to use for compression. - algo: The algorithm to use for compression. - strategy: The strategy to use for compression. - level: The compression level to use. - overwrite: What to do when the destination file already exists. - threads: Worker threads, or 0 for one per core. - quiet: Whether to suppress progress output. + job: The planned job. + quiet: Whether to suppress all output. + verbose: Whether to print the plan and stats blocks. + remove: Whether to delete the input once the output is written. """ - algo_lower: str | None = algo.lower() if algo else None - options = CompressionOptions( - algo=None if algo_lower == "auto" else algo_lower, - strategy=strategy.lower(), - level=level, - format=fmt, - overwrite=overwrite, - threads=threads, - ) - - job = CompressionJob.from_file(src=source, dest=None, options=options) - # Planned once to learn the default name, then repointed if `-o` moves it - dest: Path = resolve_single_output(output, job.plan.dest) - if dest != job.plan.dest: - job = CompressionJob.from_file(src=source, dest=dest, options=options) - plan = job.plan if not plan.can_compress: fail(f"Error: {plan.reason_if_unavailable}", EXIT_USAGE) - if not quiet: - lines: list[str] = [ - f"Compressing: {plan.src}", - f"Output: {plan.dest}", - f"Algorithm: {plan.backend_name}", - ] - if not fmt: - lines.append(f"Strategy: {strategy}") - if level is not None: - lines.append(f"Level: {level}") - print("\n".join(lines) + "\n") - - start_time: float = time.time() - - with progress_bar( + if verbose: + compress_plan(plan) + + skips: bool = plan.options.overwrite is OverwriteMode.SKIP and plan.dest.exists() + before = plan.src.stat() + + elapsed, _ = run_job( + job, + plan.src, "Compressing", plan.input_size, - enabled=not quiet and plan.input_size > BAR_THRESHOLD, - ) as on_progress: - result = job.run(progress=on_progress) + quiet=quiet, + what="Compression", + verbose=verbose, + ) - elapsed: float = time.time() - start_time + # RENAME may have written somewhere other than `plan.dest`; job.plan is + # replaced on success with the path actually written + written: Path = job.plan.dest + if verbose: + compress_result(plan, written, elapsed) - exit_for_result(result, "Compression") + elif skips and not quiet: + skipped(plan.dest) - # RENAME may have written somewhere other than `dest`; job.plan is updated - # in place on success to reflect the path actually written - actual_dest: Path = job.plan.dest - compressed_size: int = actual_dest.stat().st_size - ratio: int | float = ( - (compressed_size / plan.input_size) * 100 if plan.input_size > 0 else 0 - ) - speed_mbs: int | float = ( - (plan.input_size / (1024 * 1024)) / elapsed if elapsed > 0 else 0 - ) + elif not quiet: + renamed: Path | None = written if written != plan.dest else None + summary(plan.dest, plan.input_size, written.stat().st_size, elapsed, renamed) - if not quiet: - print() - succeed("Compression successful!\n") - if actual_dest != dest: - print(f" Renamed to: {actual_dest}") - print( - f" Original size: {format_size(size_bytes=plan.input_size)}\n" - f" Compressed size: {format_size(size_bytes=compressed_size)}\n" - f" Ratio: {ratio:.1f}% of original\n" - f" Time: {format_time(seconds=elapsed)}\n" - f" Speed: {speed_mbs:.2f} MB/s\n" - f" Saved: {format_size(size_bytes=plan.input_size - compressed_size)}\n" - ) + if remove: + remove_source(plan.src, before) -def _archive_many( - sources: list[Path], - output: Path | None, - fmt: str, - level: int | None, - overwrite: OverwriteMode, - threads: int, - quiet: bool, -) -> None: +def _archive_many(job: ArchiveJob, quiet: bool, verbose: bool) -> None: """Pack several paths, or a directory, into one archive. Args: - sources: The paths to the files or directories to archive. - output: The path to the output file or directory. - fmt: The format to use for compression. - level: The compression level to use. - overwrite: What to do when the destination archive already exists. - threads: Worker threads, or 0 for one per core. - quiet: Whether to suppress progress output. + job: The planned job. + quiet: Whether to suppress all output. + verbose: Whether to print the plan and stats blocks. """ - destination: Path = ( - output if output is not None else default_archive_output(sources[0], fmt) - ) - - options = ArchiveOptions( - format=fmt.lower(), - compression_level=level, - overwrite=overwrite, - threads=threads, - ) - job = ArchiveJob.from_paths(sources=sources, output=destination, options=options) plan = job.plan if not plan.can_run: fail(f"Error: {plan.reason_if_unavailable}", EXIT_USAGE) - if not quiet: - print( - f"Archiving: {plan.entry_count} source(s)\n" - f"Output: {plan.output}\n" - f"Format: {plan.options.format}\n" - f"Input size: {format_size(size_bytes=plan.total_input_size)}\n" - ) + if verbose: + archive_plan(plan) - start_time: float = time.time() + skips: bool = plan.options.overwrite is OverwriteMode.SKIP and plan.output.exists() # A format with a codec stage reads the data twice, so the job reports a # larger total than this once it starts; the bar follows it - with progress_bar( + elapsed, result = run_job( + job, + plan.output, "Archiving", plan.total_input_size, - enabled=not quiet and plan.total_input_size > BAR_THRESHOLD, - ) as on_progress: - result = job.run(progress=on_progress) - - elapsed: float = time.time() - start_time - - exit_for_result(result, "Archive") - - # RENAME may have written somewhere other than `destination`; job.plan is - # updated in place on success to reflect the path actually written - actual_output: Path = job.plan.output - archive_size: int = actual_output.stat().st_size - - if not quiet: - print() - succeed("Archive created!") - if actual_output != destination: - print(f" Renamed to: {actual_output}") - print( - f" Input size: {format_size(size_bytes=plan.total_input_size)}\n" - f" Archive size: {format_size(size_bytes=archive_size)}\n" - f" Time: {format_time(seconds=elapsed)}\n" + quiet=quiet, + what="Archive", + verbose=verbose, + ) + + # RENAME may have written somewhere other than `plan.output`; job.plan is + # replaced with the path actually written once the archive is committed + written: Path = job.plan.output + if verbose: + archive_result(plan, written, elapsed) + + elif skips and not quiet: + skipped(plan.output) + + elif not quiet: + renamed: Path | None = written if written != plan.output else None + summary( + plan.output, + plan.total_input_size, + written.stat().st_size, + elapsed, + renamed, ) + if isinstance(result.error, SourceRemovalError): + raise NotRemoved(*result.error.failures) + +@handles_errors("Compression") def compress( inputs: Annotated[ list[Path], app.Argument(help="Files and directories to compress") @@ -210,33 +138,38 @@ def compress( output: Annotated[ Path | None, app.Option( - "--output", "-o", help="Output path (default: named after the input)" + "--output", "-o", help="Output path", show_default="named after the input" ), ] = None, format: Annotated[ str | None, app.Option( "--format", - "-f", + "-F", case_sensitive=False, - help="Container to write, e.g. gz, zst, tar.zst, zip", + help="Container to write, e.g. comp, gz, zst, tar.zst, zip", ), ] = None, algo: Annotated[ - str | None, + Algo, app.Option( - "--algo", "-a", case_sensitive=False, help="Compression algorithm to use" + "--algo", + "-a", + case_sensitive=False, + help="Compression algorithm to use", + show_default=False, ), - ] = "auto", + ] = Algo.auto, strategy: Annotated[ - str, + Strategy | None, app.Option( "--strategy", "-s", case_sensitive=False, - help="Compression strategy to use (fast/balanced/max_ratio)", + help="Compression strategy to use", + show_default="balanced", ), - ] = "balanced", + ] = None, level: Annotated[ int | None, app.Option( @@ -245,23 +178,48 @@ def compress( help="Compression level; each algorithm's range is shown by `list`", ), ] = None, - overwrite: Annotated[ - bool, + include: Annotated[ + list[str] | None, + app.Option( + "--include", + help="Archive only what matches this, and the dirs holding it; repeatable", + ), + ] = None, + exclude: Annotated[ + list[str] | None, + app.Option( + "--exclude", + help="Leave out what matches this glob and all below it, overriding " + "--include; repeatable", + ), + ] = None, + on_conflict: Annotated[ + OverwriteMode | None, app.Option( - "--overwrite", help="Replace an archive that already exists at the output" + "--on-conflict", + case_sensitive=False, + help="What to do when the output exists", + show_default="rename", ), + ] = None, + force: Annotated[ + bool, + app.Option("--force", "-f", help="Replace an existing output"), ] = False, - skip_existing: Annotated[ + remove: Annotated[ bool, app.Option( - "--skip-existing", help="Leave an existing output archive untouched" + "--rm/--keep", + " /-k", + help="Delete the input once the output is written, or keep it", ), ] = False, - error_on_conflict: Annotated[ + dereference: Annotated[ bool, app.Option( - "--error-on-conflict", - help="Fail instead of renaming a clashing output archive", + "--dereference", + "-h", + help="Archive what symlinks lead to, rather than the links", ), ] = False, threads: Annotated[ @@ -273,6 +231,10 @@ def compress( help="Worker threads for zstd and xz, or 0 for one per core", ), ] = 0, + verbose: Annotated[ + bool, + app.Option("--verbose", "-v", help="Show the plan and full stats"), + ] = False, quiet: Annotated[ bool, app.Option("--quiet", "-q", help="Suppress all output") ] = False, @@ -281,93 +243,87 @@ def compress( \f One file becomes a single-file container: a Compresso `.comp` by default, - or `-f gz`/`zst`/`xz`/`bz2`/`lz4` for a standalone one; a directory, or + or `-F gz`/`zst`/`xz`/`bz2`/`lz4` for a standalone one; a directory, or more than one input, becomes an archive (`zip` unless told otherwise); - a recognisable extension on `-o` picks the format on its own, and `-f` + a recognisable extension on `-o` picks the format on its own, and `-F` overrides it. A clashing output name by default gets a platform-native numbered sibling (`name 2.ext` on macOS, `name (2).ext` elsewhere), for an archive and a - single-file container alike; use `--overwrite`, `--skip-existing`, or - `--error-on-conflict` to change that. + single-file container alike; use `--on-conflict` or `-f` to change that. + + Each output gets one summary line; `-v` shows the plan and full stats + instead, and `-q` nothing at all. Args: inputs: The files and directories to compress. output: Where to write (default: named after the first input). format: Container to write (default: inferred, else .comp or zip). - algo: Algorithm for the Compresso container (default: auto). - strategy: Strategy for the Compresso container (default: balanced). + algo: Algorithm for the Compresso container (default: auto); any + other format refuses it. + strategy: Strategy for the Compresso container (default: balanced); + any other format refuses it, as does `-a`. level: Compression level (default: the format's own). - overwrite: If True, replace an existing output. - skip_existing: If True, leave an existing output untouched. - error_on_conflict: If True, fail instead of renaming a clashing output. + include: Globs an archive's entries must match (default: all); + a single-file output refuses it. + exclude: Globs an archive's entries must not match; a single-file + output refuses it. + on_conflict: What to do when the output exists (default: rename). + force: If True, replace an existing output. + remove: If True, delete the input once the output is written; an + archive deletes only what it stored, leaving anything else and + the directories holding it. + dereference: If True, an archive stores symlink targets, and `remove` + deletes the links, not targets; a dangling link, or one leading to + a directory above it, stays a link; single-file output always reads + through a link. threads: Worker threads for zstd and xz (default: one per core). + verbose: If True, show the plan and full stats. quiet: If True, suppress all output. """ - if sum([overwrite, skip_existing, error_on_conflict]) > 1: - fail( - "Error: --overwrite, --skip-existing and --error-on-conflict " - "are mutually exclusive", - EXIT_USAGE, - ) + overwrite_mode = conflict_mode(on_conflict, force, remove) + if verbose and quiet: + fail("Error: -v contradicts -q", EXIT_USAGE) - if overwrite: - overwrite_mode = OverwriteMode.OVERWRITE - elif skip_existing: - overwrite_mode = OverwriteMode.SKIP - elif error_on_conflict: - overwrite_mode = OverwriteMode.ERROR - else: - overwrite_mode = OverwriteMode.RENAME + if not inputs: + fail("Error: No input paths given", EXIT_USAGE) - try: - if not inputs: - fail("Error: No input paths given", EXIT_USAGE) - - chosen: str | None = format or infer_format(output) - many: bool = len(inputs) > 1 or any(p.is_dir() for p in inputs) - - if format_is_single_file(chosen) and many: - fail( - f"Error: {chosen} holds a single file; " - f"use an archive format such as tar.{chosen} for several", - EXIT_USAGE, - ) - - if is_archive_format(chosen) or (many and chosen is None): - _archive_many( - sources=inputs, - output=output, - fmt=chosen or DEFAULT_ARCHIVE_FORMAT, - level=level, - overwrite=overwrite_mode, - threads=threads, - quiet=quiet, - ) + algo_name: str | None = None if algo == "auto" else algo - else: - _compress_one_file( - source=inputs[0], - output=output, - fmt=chosen, - algo=algo, - strategy=strategy, - level=level, - overwrite=overwrite_mode, - threads=threads, - quiet=quiet, - ) + # -a names the algorithm that -s would otherwise pick + if algo_name and strategy: + fail("Error: -a contradicts -s", EXIT_USAGE) + + request = CompressRequest( + inputs=inputs, + output=output, + format=format, + algo=algo_name, + strategy=strategy, + level=level, + overwrite=overwrite_mode, + threads=threads, + include=include or (), + exclude=exclude or (), + remove_sources=remove, + dereference=dereference, + ) - except KeyboardInterrupt: - cancelled("Compression") + try: + job = plan_compress(request) + + except ValueError as e: + fail(f"Error: {e}", EXIT_USAGE) - except (Error, HeaderError, BackendError) as e: - fail(f"Compression error: {e}") + def run_one(job: CompressionJob | ArchiveJob) -> None: + if isinstance(job, ArchiveJob): + _archive_many(job, quiet, verbose) + else: + _compress_one_file(job, quiet, verbose, remove) - except Exception as e: - fail(f"Unexpected error: {e}") + sys.exit(run_all([job], run_one, "Compression")) def register() -> None: """Attach this command to the app.""" - app.command(aliases=["c", "comp", "archive", "a", "ar"])(compress) + app.command(aliases=["c"])(compress) diff --git a/src/compresso/cli/extract.py b/src/compresso/cli/extract.py index e16b9c7..1279bab 100644 --- a/src/compresso/cli/extract.py +++ b/src/compresso/cli/extract.py @@ -1,258 +1,254 @@ -"""The `extract` command, which also answers to `decompress`.""" +"""The `extract` command, unpacking archives and compressed files.""" from __future__ import annotations -import time +import re +import sys from pathlib import Path from typing import Annotated -from .._core import BackendError, Error, HeaderError from ..frontend.api import DecompressionJob from ..frontend.archive_api import ( - ArchiveEntry, ExtractJob, ExtractOptions, OverwriteMode, SymlinkMode, ) +from ..frontend.dispatch import ExtractRequest, plan_extract from ._app import app -from ._dispatch import looks_like_archive, resolve_single_output -from ._render import ( - BAR_THRESHOLD, - EXIT_USAGE, - cancelled, - exit_for_result, - fail, - format_size, - format_time, - progress_bar, - succeed, +from ._render import EXIT_USAGE, fail, handles_errors, report_warning +from ._render.jobs import ( + decompress_plan, + decompress_result, + extract_plan, + extract_result, + skipped, + summary, +) +from ._run import ( + InputFailed, + conflict_mode, + remove_source, + require_files, + run_all, + run_job, ) +_SIZE = re.compile(r"(\d+)(?:([kmgt])(?:i?b)?|b)?", re.IGNORECASE) +_UNITS = {"": 1, "k": 1024, "m": 1024**2, "g": 1024**3, "t": 1024**4} -def list_entries(entries: list[ArchiveEntry]) -> None: - """Print one line per entry: its size, then its path. - Directories and symlinks leave the size blank, a symlink is followed by - its target, and each entry is indented once for every directory above it - that was already listed. +def parse_size(value: str) -> int: + """Parse a byte count with an optional binary suffix, e.g. `500M` or `1KB`. Args: - entries: The archive's entries, in archive order. - """ - listed_dirs: set[str] = set() + value: The text given. - for entry in entries: - parts = entry.path.rstrip("/").split("/") - depth = sum("/".join(parts[:i]) in listed_dirs for i in range(1, len(parts))) + Returns: + The count in bytes. - if entry.is_dir: - listed_dirs.add("/".join(parts)) + Raises: + ValueError: If `value` isn't a size, which the parser reports as a + usage error. + """ + match = _SIZE.fullmatch(value.strip()) + if not match: + raise ValueError(value) - # A symlink's stored size is the length of its target; hardlink's have no data - size = ( - "" - if entry.is_dir or entry.is_symlink or entry.is_hardlink - else format_size(size_bytes=entry.size) - ) - target = f" -> {entry.link_target}" if entry.is_symlink else "" + count, unit = match.groups() - # "1023.99 KB" is the widest format_size gives - print(f"{size:>10} {' ' * depth}{entry.path}{target}") + return int(count) * _UNITS[(unit or "").lower()] -def _extract_archive( - source: Path, - output_dir: Path | None, - list_only: bool, - options: ExtractOptions, - quiet: bool, -) -> None: +def _extract_archive(job: ExtractJob, quiet: bool, verbose: bool, remove: bool) -> None: """Unpack one archive. Args: - source: The path to the archive to extract. - output_dir: The directory to extract to. - list_only: Whether to list entries only. - options: The extract options. - quiet: Whether to suppress output. + job: The planned job. + quiet: Whether to suppress all output. + verbose: Whether to print the plan and stats blocks. + remove: Whether to delete the archive once it is extracted. """ - job = ExtractJob.from_archive( - archive=source, output_dir=output_dir, options=options - ) plan = job.plan if not plan.can_run: - fail(f"Error: {plan.reason_if_unavailable}", EXIT_USAGE) - - if list_only: - list_entries(job.list_contents()) - return - - if not quiet: - print( - f"Extracting: {plan.archive}\n" - f"Output dir: {plan.output_dir}\n" - f"Entries: {len(plan.entries)}\n" - ) + raise InputFailed(f"{plan.archive}: {plan.reason_if_unavailable}") - start_time: float = time.time() + if verbose: + extract_plan(plan) # Progress covers the bytes written out; a compressed archive also reports # the pass that decompresses it, so the job's own total wins - extracted_size: int = sum(entry.size for entry in plan.entries) - - with progress_bar( + total: int = sum(entry.size for entry in plan.entries) + before = plan.archive.stat() + elapsed, _ = run_job( + job, + plan.archive, "Extracting", - extracted_size, - enabled=not quiet and extracted_size > BAR_THRESHOLD, - ) as on_progress: - result = job.run(progress=on_progress) + total, + quiet=quiet, + what="Extraction", + verbose=verbose, + ) - elapsed: float = time.time() - start_time + # Names and includes the filter dropped still count as found + if plan.options.exclude and not plan.entries and not quiet: + report_warning(f"{plan.archive}: nothing left to extract") - exit_for_result(result, "Extraction") + if verbose: + extract_result(plan, elapsed) - if not quiet: - print() - succeed("Extraction successful!") - print( - f" Entries: {len(plan.entries)}\n" - f" Time: {format_time(seconds=elapsed)}\n" - ) + elif not quiet: + summary(plan.archive, before.st_size, total, elapsed, None, ratio=False) + + if remove: + remove_source(plan.archive, before) def _decompress_one_file( - source: Path, - output: Path | None, - list_only: bool, - overwrite: OverwriteMode, - threads: int, - restore_name: bool, - quiet: bool, + job: DecompressionJob, quiet: bool, verbose: bool, remove: bool ) -> None: """Unpack one single-file container: a `.comp` or a standalone codec. Args: - source: The path to the single-file container. - output: The output path. - list_only: Whether to list entries only. - overwrite: What to do when the destination file already exists. - threads: Worker threads for xz, or 0 for one per core. - restore_name: Whether to name and date a gzip file's output as its - header does. - quiet: Whether to suppress output. + job: The planned job. + quiet: Whether to suppress all output. + verbose: Whether to print the plan and stats blocks. + remove: Whether to delete the input once the output is written. """ - if list_only: - fail( - f"Error: {source.name} holds a single file, so there is nothing to list", - EXIT_USAGE, - ) - - job = DecompressionJob.from_file( - src=source, - dest=None, - overwrite=overwrite, - threads=threads, - restore_name=restore_name, - ) - dest: Path = resolve_single_output(output, job.plan.dest) - if dest != job.plan.dest: - job = DecompressionJob.from_file( - src=source, - dest=dest, - overwrite=overwrite, - threads=threads, - restore_name=restore_name, - ) - plan = job.plan if not plan.can_run: - fail(f"Error: {plan.reason_if_unavailable}", EXIT_USAGE) - - insp = plan.inspection - - if not quiet: - lines: list[str] = [ - f"Decompressing: {plan.src}", - f"Output: {plan.dest}", - f"Format: {plan.format or insp.algo_name}", - ] - if insp.orig_size: - lines.append(f"Original size: {format_size(size_bytes=insp.orig_size)}") - print("\n".join(lines) + "\n") + raise InputFailed(f"{plan.src}: {plan.reason_if_unavailable}") - start_time: float = time.time() + if verbose: + decompress_plan(plan) # Progress counts the compressed bytes read, so the source's size is the # denominator rather than the size it expands to - compressed_size: int = plan.src.stat().st_size + before = plan.src.stat() + compressed_size: int = before.st_size + skips: bool = plan.overwrite is OverwriteMode.SKIP and plan.dest.exists() - with progress_bar( + elapsed, _ = run_job( + job, + plan.src, "Decompressing", compressed_size, - enabled=not quiet and compressed_size > BAR_THRESHOLD, - ) as on_progress: - result = job.run(progress=on_progress) - - elapsed: float = time.time() - start_time - - exit_for_result(result, "Decompression") - - # RENAME may have written somewhere other than `dest`; job.plan is updated - # in place on success to reflect the path actually written - actual_dest: Path = job.plan.dest - decompressed_size: int = actual_dest.stat().st_size - speed_mbs: int | float = ( - (decompressed_size / (1024 * 1024)) / elapsed if elapsed > 0 else 0 + quiet=quiet, + what="Decompression", + verbose=verbose, ) - if not quiet: - succeed("Decompression successful!\n") - if actual_dest != dest: - print(f" Renamed to: {actual_dest}") - print( - f" Compressed size: {format_size(size_bytes=compressed_size)}\n" - f" Decompressed size: {format_size(size_bytes=decompressed_size)}\n" - f" Time: {format_time(seconds=elapsed)}\n" - f" Speed: {speed_mbs:.2f} MB/s\n" + # RENAME may have written somewhere other than `plan.dest`; job.plan is + # replaced on success with the path actually written + written: Path = job.plan.dest + if verbose: + decompress_result(plan, written, compressed_size, elapsed) + + elif skips and not quiet: + skipped(plan.dest) + + elif not quiet: + renamed: Path | None = written if written != plan.dest else None + summary( + plan.src, + compressed_size, + written.stat().st_size, + elapsed, + renamed, + ratio=False, ) + if remove: + remove_source(plan.src, before) + +@handles_errors("Extraction") def extract( - inputs: Annotated[list[Path], app.Argument(help="Archives or compressed files")], + inputs: Annotated[ + list[Path], + app.Argument(help="Archives or compressed files", readable=False), + ], + directory: Annotated[ + Path | None, + app.Option( + "--directory", + "-C", + help="Directory to write into (default: beside each input)", + ), + ] = None, output: Annotated[ Path | None, app.Option( - "--output-dir", "--output", "-o", - help="Where to write (default: beside the input)", + help="File to write, for one compressed file that isn't an archive", ), ] = None, - list_only: Annotated[ - bool, app.Option("--list", help="List archive contents without extracting") - ] = False, - overwrite: Annotated[ - bool, app.Option("--overwrite", help="Replace files that already exist") - ] = False, - skip_existing: Annotated[ - bool, app.Option("--skip-existing", help="Leave files that already exist") + entry: Annotated[ + list[str] | None, + app.Option( + "--entry", + "-e", + help="Extract only this entry, or the directory and all below it; " + "repeatable", + ), + ] = None, + include: Annotated[ + list[str] | None, + app.Option( + "--include", + help="Extract entries matching this glob, as well as any --entry; " + "repeatable", + ), + ] = None, + exclude: Annotated[ + list[str] | None, + app.Option( + "--exclude", + help="Skip entries matching this glob, overriding --entry and " + "--include; repeatable", + ), + ] = None, + on_conflict: Annotated[ + OverwriteMode | None, + app.Option( + "--on-conflict", + case_sensitive=False, + help="What to do when a file exists", + show_default="rename", + ), + ] = None, + force: Annotated[ + bool, + app.Option("--force", "-f", help="Replace files that already exist"), ] = False, - error_on_conflict: Annotated[ + remove: Annotated[ bool, app.Option( - "--error-on-conflict", help="Fail instead of renaming a clashing file" + "--rm/--keep", + " /-k", + help="Delete the input once it is fully extracted, or keep it", ), ] = False, - max_total_size: Annotated[ + max_size: Annotated[ int | None, app.Option( - "--max-total-size", - help="Refuse archives extracting to more than this many bytes", + "--max-size", + metavar="SIZE", + parser=parse_size, + help="Refuse archives extracting to more than this, e.g. 500M or 10G", ), ] = None, + max_depth: Annotated[ + int, + app.Option( + "--max-depth", + min=0, + help="Refuse entries nested deeper than this, or 0 for no limit", + ), + ] = 32, same_permissions: Annotated[ bool, app.Option( @@ -261,14 +257,28 @@ def extract( help="Restore modes exactly, keeping setuid/setgid/sticky and ignoring the umask", ), ] = False, + no_same_permissions: Annotated[ + bool, + app.Option("--no-same-permissions", help="Don't restore modes"), + ] = False, + touch: Annotated[ + bool, + app.Option("--touch", "-m", help="Don't restore modification times"), + ] = False, + overwrite_dir: Annotated[ + bool, + app.Option( + "--overwrite-dir", + help="Also restore the mode and time of directories that already exist", + ), + ] = False, symlinks: Annotated[ SymlinkMode | None, app.Option( "--symlinks", case_sensitive=False, - help="Refuse symlinks, create those staying inside the output, " - "or write each as a file holding its target " - "(default: file for zip, deny otherwise)", + help="Refuse, create (non-escaping), or write as a file", + show_default="file for zip, deny otherwise", ), ] = None, threads: Annotated[ @@ -288,6 +298,14 @@ def extract( help="Name and date a .gz file's output as its header does", ), ] = False, + verbose: Annotated[ + bool, + app.Option( + "--verbose", + "-v", + help="Show the plan and full stats", + ), + ] = False, quiet: Annotated[ bool, app.Option("--quiet", "-q", help="Suppress all output") ] = False, @@ -300,78 +318,105 @@ def extract( Compresso `.comp`, or a standalone `.gz`/`.bz2`/`.xz`/`.zst`/`.lz4`/`.sz`, is decompressed to one file. - The result is written beside its input unless `-o` is provided. + The result is written beside its input unless `-C` or `-o` is provided. By default, a clashing name gets a platform-native numbered sibling - (`name 2.ext` on macOS, `name (2).ext` elsewhere); use `--overwrite`, - `--skip-existing`, or `--error-on-conflict` to change that. + (`name 2.ext` on macOS, `name (2).ext` elsewhere); use `--on-conflict` or + `-f` to change that. + + Each input gets one summary line; `-v` shows the plan and full stats + instead, and `-q` nothing at all. Args: inputs: The archives or compressed files to unpack. - output: Where to write (default: beside the input); a directory for - archives, and for one file either a directory to put it in or the - path to write. - list_only: If True, list archive contents without extracting. - overwrite: If True, replace files that already exist. - skip_existing: If True, leave files that already exist untouched. - error_on_conflict: If True, fail instead of renaming a clashing file. - max_total_size: Cap on total extracted bytes (default: no cap). + directory: The directory to write into (default: beside each input). + output: The file to write, for a single input that isn't an archive. + entry: Entry paths to extract from each archive (default: all); a + path matching no entry fails that archive after the rest is + extracted. + include: Globs selecting entries as well as any `entry` names; one + matching no entry fails that archive likewise. + exclude: Globs dropping entries, overriding `entry` and `include`; a + run left with nothing to extract warns, but still succeeds. + on_conflict: What to do when a file exists (default: rename). + force: If True, replace files that already exist. + remove: If True, delete each input once it is fully written out. + max_size: Cap on total extracted bytes (default: no cap). + max_depth: Cap on an entry path's depth, or 0 for none. same_permissions: If True, restore archive entries' modes exactly; single-file outputs are unaffected. + no_same_permissions: If True, don't restore archive entries' modes. + touch: If True, don't restore archive entries' modification times. + overwrite_dir: If True, directories that already exist also get the + archive's mode and modification time. symlinks: What to do with a symlink entry (default: write a zip's as files, refuse any other archive's). threads: Worker threads for xz (default: one per core). name: If True, name a gzip file's output after the name its header stores, and give it the stored modification time, if any. + verbose: If True, show the plan and full stats. quiet: If True, suppress all output. """ - if sum([overwrite, skip_existing, error_on_conflict]) > 1: - fail( - "Error: --overwrite, --skip-existing and --error-on-conflict " - "are mutually exclusive", - EXIT_USAGE, - ) + mode = conflict_mode(on_conflict, force, remove) + + # Whatever these leave out would go with the archive + if remove and (entry or include or exclude): + fail("Error: --rm contradicts --entry, --include and --exclude", EXIT_USAGE) + + if verbose and quiet: + fail("Error: -v contradicts -q", EXIT_USAGE) if not inputs: fail("Error: No input files given", EXIT_USAGE) - if overwrite: - mode = OverwriteMode.OVERWRITE - elif skip_existing: - mode = OverwriteMode.SKIP - elif error_on_conflict: - mode = OverwriteMode.ERROR - else: - mode = OverwriteMode.RENAME + if output and directory: + fail("Error: -o contradicts -C", EXIT_USAGE) + + if output and len(inputs) > 1: + fail("Error: -o names one file; use -C for several inputs", EXIT_USAGE) + + if same_permissions and no_same_permissions: + fail("Error: -p contradicts --no-same-permissions", EXIT_USAGE) + + require_files(inputs) options = ExtractOptions( overwrite=mode, - max_total_size=max_total_size or 0, + max_total_size=max_size or 0, + max_depth=max_depth, + preserve_permissions=not no_same_permissions, exact_permissions=same_permissions, + preserve_timestamps=not touch, + overwrite_dir_metadata=overwrite_dir, symlinks=symlinks, threads=threads, + include=include or (), + exclude=exclude or (), ) - try: - for source in inputs: - if looks_like_archive(source): - _extract_archive(source, output, list_only, options, quiet) + request = ExtractRequest( + inputs=inputs, + directory=directory, + output=output, + files=entry, + options=options, + restore_name=name, + ) - else: - _decompress_one_file( - source, output, list_only, mode, threads, name, quiet - ) + def run_one(job: ExtractJob | DecompressionJob) -> None: + if isinstance(job, ExtractJob): + _extract_archive(job, quiet, verbose, remove) - except KeyboardInterrupt: - cancelled("Extraction") + else: + # An archive creates a missing `-C` itself; a single file needs it made + if directory: + directory.mkdir(parents=True, exist_ok=True) - except (Error, HeaderError, BackendError) as e: - fail(f"Extraction error: {e}") + _decompress_one_file(job, quiet, verbose, remove) - except Exception as e: - fail(f"Unexpected error: {e}") + sys.exit(run_all(plan_extract(request), run_one, "Extraction")) def register() -> None: """Attach this command to the app.""" - app.command(aliases=["x", "ex", "decompress", "d", "decomp"])(extract) + app.command(aliases=["x"])(extract) diff --git a/src/compresso/cli/formats.py b/src/compresso/cli/formats.py new file mode 100644 index 0000000..3a93dc4 --- /dev/null +++ b/src/compresso/cli/formats.py @@ -0,0 +1,26 @@ +"""The `formats` command, showing the algorithms and formats available.""" + +from __future__ import annotations + +from .._core import standalone_formats +from ..frontend._formats import archive_formats +from ..introspect.capabilities import list_capabilities +from ._app import app +from ._render import fail, handles_errors +from ._render.formats import list_formats + + +@handles_errors("Format listing") +def formats() -> None: + """List the available compression algorithms and file formats.""" + caps = list_capabilities() + + if not caps: + fail("No compression backends available") + + list_formats(caps, standalone_formats(), archive_formats()) + + +def register() -> None: + """Attach this command to the app.""" + app.command(name="formats")(formats) diff --git a/src/compresso/cli/info.py b/src/compresso/cli/info.py new file mode 100644 index 0000000..4b257e0 --- /dev/null +++ b/src/compresso/cli/info.py @@ -0,0 +1,111 @@ +"""The `info` command, describing archives and compressed files.""" + +from __future__ import annotations + +import sys +from collections.abc import Callable +from functools import partial +from pathlib import Path +from typing import Annotated + +from .._core import detect_format, gzip_stored_name, stored_content_size +from ..frontend._formats import gzip_stored_mtime +from ..frontend.archive_api import plan_extraction as plan_archive_extraction +from ..frontend.dispatch import is_archive_format +from ..introspect.file_inspect import inspect as inspect_file +from ._app import app +from ._render import handles_errors, input_header +from ._render.info import archive_info, codec_info, file_info, print_json +from ._run import InputFailed, require_files, run_all + + +def _describer(path: Path) -> Callable[..., dict[str, object]]: + """Read one input, returning the call that describes it. + + Args: + path: The input. + + Returns: + A render call taking `as_json`, which prints the input unless JSON is + wanted and returns its JSON item. + + Raises: + InputFailed: If the input is unrecognised or can't be read. + """ + fmt = detect_format(path) + + if is_archive_format(fmt): + plan = plan_archive_extraction(archive=path) + if not plan.can_run: + raise InputFailed(f"{path}: {plan.reason_if_unavailable}") + + return partial(archive_info, path, fmt, plan.entries) + + if fmt == "compresso": + result = inspect_file(path=path) + if not result.header_ok: + raise InputFailed(f"{path}: {result.reason}") + + return partial(file_info, result) + + if fmt == "unknown": + raise InputFailed(f"{path}: not a recognised archive or compressed file") + + stored: bytes | None = gzip_stored_name(path) + + return partial( + codec_info, + path, + fmt, + # Only for display, so a name that isn't UTF-8 can stay approximate + stored_name=stored.decode(errors="replace") if stored else None, + mtime=gzip_stored_mtime(path) if fmt == "gzip" else None, + orig_size=stored_content_size(path), + ) + + +@handles_errors("Inspection") +def info( + inputs: Annotated[ + list[Path], app.Argument(help="Files to describe", readable=False) + ], + output_json: Annotated[ + bool, app.Option("--json", help="Output a JSON array, one item per input") + ] = False, +) -> None: + """Describe archives and compressed files. + \f + + An archive reports its format, entry count, and total size; `list` shows + its entries. A Compresso `.comp` reports its algorithm and decompression + info, and a standalone codec file its format and size, with whatever its + header stores: a gzip's name and mtime, an xz, zstd, lz4 or snappy file's + original size. + + Args: + inputs: The archives or compressed files to describe. + output_json: If True, print a JSON array, one item per input. + """ + require_files(inputs) + + items: list[dict[str, object]] = [] + + def run_one(path: Path) -> None: + describe = _describer(path) + + if len(inputs) > 1 and not output_json: + input_header(path, first=not items) + + items.append(describe(as_json=output_json)) + + code = run_all(inputs, run_one, "Inspection") + + if output_json: + print_json(items) + + sys.exit(code) + + +def register() -> None: + """Attach this command to the app.""" + app.command(aliases=["i"])(info) diff --git a/src/compresso/cli/inspect.py b/src/compresso/cli/inspect.py deleted file mode 100644 index 3d3bbce..0000000 --- a/src/compresso/cli/inspect.py +++ /dev/null @@ -1,244 +0,0 @@ -"""The `inspect` command.""" - -from __future__ import annotations - -import stat -import time -from pathlib import Path -from typing import Annotated - -import orjson - -from .._core import detect_format -from ..frontend.archive_api import ArchiveEntry -from ..frontend.archive_api import plan_extraction as plan_archive_extraction -from ..introspect.file_inspect import inspect as inspect_file -from ._app import app -from ._dispatch import looks_like_archive -from ._render import ( - EXIT_USAGE, - cancelled, - fail, - format_size, - format_time, - succeed, -) - - -def _entry_detail_line(entry: ArchiveEntry) -> str: - """Format one entry's full detail: mode, mtime, size, path, and target. - - Args: - entry: The archive entry to format. - - Returns: - A single formatted line for the entry. - """ - if entry.is_dir: - type_bit = stat.S_IFDIR - elif entry.is_symlink: - type_bit = stat.S_IFLNK - else: - type_bit = stat.S_IFREG - - mode = stat.filemode(type_bit | entry.mode) - mtime = time.strftime("%Y-%m-%d %H:%M", time.localtime(entry.mtime)) - has_no_data = entry.is_dir or entry.is_symlink or entry.is_hardlink - size = "" if has_no_data else format_size(size_bytes=entry.size) - if entry.is_symlink: - target = f" -> {entry.link_target}" - elif entry.is_hardlink: - target = f" link to {entry.link_target}" - else: - target = "" - - detail = "" - if entry.compressed_size is not None: - detail = f" (compressed {format_size(size_bytes=entry.compressed_size)}" - if entry.crc is not None: - detail += f", crc {entry.crc:08x}" - if entry.method is not None: - detail += f", method {entry.method}" - detail += ")" - - return f"{mode} {mtime} {size:>10} {entry.path}{target}{detail}" - - -def _inspect_archive(file: Path, output_json: bool, show_entries: bool) -> None: - """Inspect an archive: a container that can hold many entries. - - Args: - file: The path to the archive. - output_json: If True, output metadata in JSON format. - show_entries: If True, also report each entry's own detail. - """ - plan = plan_archive_extraction(archive=file) - - if not plan.can_run: - fail(f"Error: {plan.reason_if_unavailable}", EXIT_USAGE) - - fmt = detect_format(file) - total_size = sum(entry.size for entry in plan.entries) - archive_size: int = file.stat().st_size - - if output_json: - data: dict[str, object] = { - "path": str(file), - "format": fmt, - "is_archive": True, - "entry_count": len(plan.entries), - "total_size": total_size, - "archive_size": archive_size, - } - if show_entries: - data["entries"] = [ - { - "path": entry.path, - "size": entry.size, - "is_dir": entry.is_dir, - "is_symlink": entry.is_symlink, - "is_hardlink": entry.is_hardlink, - "mtime": entry.mtime, - "mode": entry.mode, - "link_target": entry.link_target, - "compressed_size": entry.compressed_size, - "crc": entry.crc, - "method": entry.method, - } - for entry in plan.entries - ] - print(orjson.dumps(data, option=orjson.OPT_INDENT_2).decode()) - return - - lines: list[str] = [ - f"File: {file}", - f"Format: {fmt}", - f"Entries: {len(plan.entries)}", - f"Total size: {format_size(size_bytes=total_size)}", - f"Archive size: {format_size(size_bytes=archive_size)}", - ] - - if total_size: - ratio: int | float = (archive_size / total_size) * 100 - lines.append(f"Compression: {ratio:.1f}% of original") - - print("\n".join(lines)) - - if show_entries: - print("\n" + "\n".join(_entry_detail_line(entry) for entry in plan.entries)) - - -def inspect( - file: Annotated[Path, app.Argument(help="File to inspect")], - output_json: Annotated[ - bool, app.Option("--json", help="Output in JSON format") - ] = False, - show_entries: Annotated[ - bool, - app.Option( - "--entries", "-e", help="For an archive, also list each entry's detail" - ), - ] = False, -) -> None: - """Inspect a compressed file or archive and show metadata. - \f - - An archive reports its format, entry count, and total size; pass - `--entries` to also list each entry's own mtime, permissions, and (for a - format like zip that tracks it) per-entry compressed size and CRC. A - single-file container, a Compresso `.comp` or a standalone codec, reports - its algorithm and decompression info as before. - - Args: - file: The path to the compressed file or archive. - output_json: If True, output metadata in JSON format. - show_entries: If True and the file is an archive, also list each - entry's own detail. - """ - try: - if looks_like_archive(file): - _inspect_archive(file, output_json, show_entries) - return - - result = inspect_file(path=file) - - if output_json: - data: dict[str, str | int | None | float] = { - "path": str(object=result.path), - "is_compresso": result.is_compresso, - "header_ok": result.header_ok, - "version": result.version, - "algo_id": result.algo_id, - "algo_name": result.algo_name, - "level": result.level, - "flags": result.flags, - "orig_size": result.orig_size, - "backend_available": result.backend_available, - "can_decompress": result.can_decompress, - "estimated_decomp_s": result.estimated_decomp_s, - "reason": result.reason, - } - print(orjson.dumps(data, option=orjson.OPT_INDENT_2).decode()) - return - - print(f"File: {result.path}\n") - - if not result.is_compresso: - reason = f"\n Reason: {result.reason}" if result.reason else "" - fail(f"Not a valid Compresso file{reason}", EXIT_USAGE) - - if not result.header_ok: - reason = f"\n Reason: {result.reason}" if result.reason else "" - fail(f"Invalid file header{reason}", EXIT_USAGE) - - succeed("Valid Compresso file\n") - - lines: list[str] = [ - f"Algorithm: {result.algo_name or 'Unknown'} (ID: {result.algo_id})", - f"Version: {result.version}", - "", - ] - - if result.level is not None: - lines.append(f"Level: {result.level}") - - else: - lines.append("Level: auto") - - if result.orig_size: - lines.append(f"Original size: {format_size(size_bytes=result.orig_size)}") - - compressed_size: int = file.stat().st_size - lines.append(f"Compressed size: {format_size(size_bytes=compressed_size)}") - - if result.orig_size: - ratio: int | float = (compressed_size / result.orig_size) * 100 - lines.append(f"Compression: {ratio:.1f}% of original") - - lines += [ - "", - f"Backend available: {'Yes' if result.backend_available else 'No'}", - f"Can decompress: {'Yes' if result.can_decompress else 'No'}", - ] - - if result.estimated_decomp_s: - lines.append( - f"Est. decomp time: {format_time(seconds=result.estimated_decomp_s)}" - ) - lines.append("") - - print("\n".join(lines)) - - if not result.can_decompress and result.reason: - app.echo(message=app.style(text=f"\n⚠ {result.reason}", fg="yellow")) - - except KeyboardInterrupt: - cancelled("Inspection") - - except Exception as e: - fail(f"Error inspecting file: {e}") - - -def register() -> None: - """Attach this command to the app.""" - app.command(aliases=["i", "info"])(inspect) diff --git a/src/compresso/cli/list.py b/src/compresso/cli/list.py new file mode 100644 index 0000000..b6e3c64 --- /dev/null +++ b/src/compresso/cli/list.py @@ -0,0 +1,97 @@ +"""The `list` command, showing an archive's entries.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Annotated + +from .._core import detect_format +from ..frontend.archive_api import ArchiveEntry +from ..frontend.archive_api import plan_extraction as plan_archive_extraction +from ..frontend.dispatch import is_archive_format +from ._app import app +from ._render import handles_errors, input_header +from ._render.info import list_entries, print_json +from ._run import InputFailed, require_files, run_all + + +def _read_entries(path: Path) -> tuple[str, list[ArchiveEntry]]: + """Read one archive's format and entries. + + Args: + path: The archive. + + Returns: + The detected format and the entries, in archive order. + + Raises: + InputFailed: If the input is not an archive or can't be read. + """ + fmt = detect_format(path) + + if not is_archive_format(fmt): + if fmt == "unknown": + raise InputFailed(f"{path}: not a recognised archive") + + raise InputFailed(f"{path}: holds a single file, so there is nothing to list") + + plan = plan_archive_extraction(archive=path) + + if not plan.can_run: + raise InputFailed(f"{path}: {plan.reason_if_unavailable}") + + return fmt, plan.entries + + +@handles_errors("Listing") +def list_archives( + inputs: Annotated[ + list[Path], app.Argument(help="Archives to list", readable=False) + ], + long: Annotated[ + bool, + app.Option("--long", "-l", help="Show each entry's mode, mtime and detail"), + ] = False, + output_json: Annotated[ + bool, app.Option("--json", help="Output a JSON array, one item per archive") + ] = False, +) -> None: + """List the entries of archives. + \f + + Each entry is shown with its size and path, indented under the directories + listed above it; `--long` adds its mode and mtime, and (for a format like + zip that tracks it) its compressed size and CRC. + + Args: + inputs: The archives to list. + long: If True, show each entry's full detail. + output_json: If True, print a JSON array of `{path, format, entries}`. + """ + require_files(inputs) + + listings: list[dict[str, object]] = [] + + def run_one(path: Path) -> None: + fmt, entries = _read_entries(path) + + if not output_json: + if len(inputs) > 1: + input_header(path, first=not listings) + + list_entries(entries, long=long) + + listings.append({"path": str(path), "format": fmt, "entries": entries}) + + code = run_all(inputs, run_one, "Listing") + + if output_json: + print_json(listings) + + sys.exit(code) + + +def register() -> None: + """Attach this command to the app.""" + app.command(name="list", aliases=["l"])(list_archives) diff --git a/src/compresso/cli/verify.py b/src/compresso/cli/verify.py new file mode 100644 index 0000000..4979207 --- /dev/null +++ b/src/compresso/cli/verify.py @@ -0,0 +1,82 @@ +"""The `test` command, checking the integrity of archives and compressed files.""" + +from __future__ import annotations + +import sys +import warnings +from pathlib import Path +from typing import Annotated + +from .._core import TrailingDataWarning +from ..frontend.api import VerificationJob +from ._app import app +from ._render import handles_errors, succeed +from ._run import InputFailed, require_files, run_all, run_job + + +def _verify_one(job: VerificationJob, quiet: bool) -> None: + """Check one input. + + Args: + job: The planned job. + quiet: Whether to suppress all output. + """ + plan = job.plan + + if not plan.can_run: + raise InputFailed(f"{plan.src}: {plan.reason_if_unavailable}") + + size: int = plan.src.stat().st_size + run_job(job, plan.src, "Testing", size, quiet=quiet, what="Test") + + if not quiet: + succeed(f"OK: {plan.src}") + + +@handles_errors("Test") +def verify( + inputs: Annotated[ + list[Path], + app.Argument(help="Archives or compressed files", readable=False), + ], + threads: Annotated[ + int, + app.Option( + "--threads", + "-T", + min=0, + help="Worker threads for xz, or 0 for one per core", + ), + ] = 0, + quiet: Annotated[ + bool, app.Option("--quiet", "-q", help="Suppress all output") + ] = False, +) -> None: + """Check archives and compressed files for damage, writing nothing. + \f + + Each input is decoded in full, as `extract` would, so this catches whatever + its format can detect: a checksum or length that doesn't match, or a file + cut short. Trailing data fails too, as `gzip -t` treats it, though `extract` + only warns. A plain `.tar` has no checksum over its file data, so only its + headers are checked, and that the data they declare is there. + + Args: + inputs: The archives or compressed files to check. + threads: Worker threads for xz (default: one per core). + quiet: If True, suppress all output. + """ + require_files(inputs) + + jobs = (VerificationJob.from_file(src=path, threads=threads) for path in inputs) + + with warnings.catch_warnings(): + warnings.simplefilter("error", TrailingDataWarning) + code = run_all(jobs, lambda job: _verify_one(job, quiet), "Test") + + sys.exit(code) + + +def register() -> None: + """Attach this command to the app.""" + app.command(name="test", aliases=["t"])(verify) diff --git a/src/compresso/csrc/_core.c b/src/compresso/csrc/_core.c index 26cc5a4..46aa857 100644 --- a/src/compresso/csrc/_core.c +++ b/src/compresso/csrc/_core.c @@ -1,5 +1,6 @@ #define PY_SSIZE_T_CLEAN #include +#include "archives/select.h" #include "codec/crc32c.h" #include "common.h" #include "fsutil.h" @@ -240,6 +241,54 @@ static int path_list_converter(PyObject *obj, void *out) { return 0; } +// Glob patterns for `include` and `exclude`, decoded as UTF-8 as archives store +// entry names; `keepalive` holds the str objects that own that UTF-8 +static int pattern_list_converter(PyObject *obj, void *out) { + PathList *list = out; + if (!obj) { + Py_CLEAR(list->keepalive); + free(list->paths); + list->paths = NULL; + return 1; + } + + if (PyUnicode_Check(obj) || PyBytes_Check(obj)) { + PyErr_Format(PyExc_TypeError, "patterns must be a sequence of str, not %s", + Py_TYPE(obj)->tp_name); + return 0; + } + + list->keepalive = PySequence_Tuple(obj); + if (!list->keepalive) { + return 0; + } + + Py_ssize_t n = PyTuple_GET_SIZE(list->keepalive); + list->paths = n > 0 ? safe_malloc((size_t)n * sizeof(char *)) : NULL; + list->count = (size_t)n; + if (n > 0 && !list->paths) { + goto fail; + } + + for (Py_ssize_t i = 0; i < n; i++) { + PyObject *item = PyTuple_GET_ITEM(list->keepalive, i); + if (!PyUnicode_Check(item)) { + PyErr_Format(PyExc_TypeError, "patterns must contain only str, not %s", + Py_TYPE(item)->tp_name); + goto fail; + } + list->paths[i] = PyUnicode_AsUTF8(item); + if (!list->paths[i]) { + goto fail; + } + } + return Py_CLEANUP_SUPPORTED; + +fail: + pattern_list_converter(NULL, list); + return 0; +} + // ---- Argument Parsing ---- // Raises ValueError for an unknown strategy name @@ -364,9 +413,11 @@ static PyObject *py_decompress_file(PyObject *self, PyObject *args, static PyObject *py_create_archive(PyObject *self, PyObject *args, PyObject *kwargs) { - static char *kwlist[] = {"output_path", "format", "input_paths", - "compression_level", "overwrite", "threads", - "progress", "cancel", NULL}; + static char *kwlist[] = { + "output_path", "format", "input_paths", "compression_level", + "overwrite", "threads", "progress", "cancel", + "include", "exclude", "remove_sources", "dereference", + NULL}; PyObject *output_path_bytes = NULL; const char *format_name = NULL; @@ -376,14 +427,21 @@ static PyObject *py_create_archive(PyObject *self, PyObject *args, int threads = 0; // One per core PyObject *progress = NULL; PyObject *cancel = NULL; + PathList include = {0}; + PathList exclude = {0}; + int remove_sources = 0; + int dereference = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&sO&|i$iiOO", kwlist, - PyUnicode_FSConverter, &output_path_bytes, - &format_name, path_list_converter, &inputs, - &compression_level, &overwrite_existing, - &threads, &progress, &cancel)) { + if (!PyArg_ParseTupleAndKeywords( + args, kwargs, "O&sO&|i$iiOOO&O&pp", kwlist, PyUnicode_FSConverter, + &output_path_bytes, &format_name, path_list_converter, &inputs, + &compression_level, &overwrite_existing, &threads, &progress, &cancel, + pattern_list_converter, &include, pattern_list_converter, &exclude, + &remove_sources, &dereference)) { return NULL; // Error already set } + PathFilter filter = {include.paths, include.count, exclude.paths, + exclude.count}; CompressionPipeline pipe = pipeline_from_name(format_name, compression_level); CoreContext ctx; @@ -404,13 +462,16 @@ static PyObject *py_create_archive(PyObject *self, PyObject *args, } char actual_path[FS_PATH_MAX]; - result = create_archive(PyBytes_AS_STRING(output_path_bytes), &pipe, - inputs.paths, inputs.count, overwrite_existing, - actual_path, sizeof(actual_path), &ctx); + result = + create_archive(PyBytes_AS_STRING(output_path_bytes), &pipe, inputs.paths, + inputs.count, &filter, overwrite_existing, remove_sources, + dereference, actual_path, sizeof(actual_path), &ctx); done: Py_DECREF(output_path_bytes); path_list_converter(NULL, &inputs); + pattern_list_converter(NULL, &include); + pattern_list_converter(NULL, &exclude); return finish_call(self, result, &ctx, actual_path); } @@ -430,6 +491,8 @@ static PyObject *py_extract_archive(PyObject *self, PyObject *args, "threads", "progress", "cancel", + "include", + "exclude", NULL}; PyObject *archive_path_bytes = NULL; @@ -438,6 +501,8 @@ static PyObject *py_extract_archive(PyObject *self, PyObject *args, int threads = 0; // One per core PyObject *progress = NULL; PyObject *cancel = NULL; + PathList include = {0}; + PathList exclude = {0}; // Anything the caller does not pass keeps its default ExtractionPolicy policy = extraction_policy_default(); @@ -448,14 +513,17 @@ static PyObject *py_extract_archive(PyObject *self, PyObject *args, unsigned int max_depth = (unsigned int)policy.max_depth; if (!PyArg_ParseTupleAndKeywords( - args, kwargs, "O&O&O|$iKIpppipiOO", kwlist, PyUnicode_FSConverter, + args, kwargs, "O&O&O|$iKIpppipiOOO&O&", kwlist, PyUnicode_FSConverter, &archive_path_bytes, PyUnicode_FSConverter, &output_dir_bytes, &files_obj, &policy.overwrite_existing, &max_total_size, &max_depth, &policy.preserve_permissions, &policy.exact_permissions, &policy.preserve_timestamps, &policy.allow_symlinks, - &policy.overwrite_dir_metadata, &threads, &progress, &cancel)) { + &policy.overwrite_dir_metadata, &threads, &progress, &cancel, + pattern_list_converter, &include, pattern_list_converter, &exclude)) { return NULL; // Error already set } + PathFilter filter = {include.paths, include.count, exclude.paths, + exclude.count}; policy.max_total_size = (uint64_t)max_total_size; policy.max_depth = (uint32_t)max_depth; @@ -521,11 +589,13 @@ static PyObject *py_extract_archive(PyObject *self, PyObject *args, result = extract_archive(PyBytes_AS_STRING(archive_path_bytes), PyBytes_AS_STRING(output_dir_bytes), files, - num_files, &policy, &ctx); + num_files, &filter, &policy, &ctx); done: Py_DECREF(archive_path_bytes); Py_DECREF(output_dir_bytes); + pattern_list_converter(NULL, &include); + pattern_list_converter(NULL, &exclude); free(files); Py_XDECREF(files_tuple); return finish_call(self, result, &ctx, NULL); @@ -695,6 +765,31 @@ static PyObject *py_decompress_standalone(PyObject *self, PyObject *args, return finish_call(self, rc, &ctx, actual_path); } +static PyObject *py_verify_file(PyObject *self, PyObject *args, + PyObject *kwargs) { + static char *kwlist[] = {"path", "threads", "progress", "cancel", NULL}; + + PyObject *path_bytes = NULL; + int threads = 0; // One per core + PyObject *progress = NULL; + PyObject *cancel = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&|$iOO", kwlist, + PyUnicode_FSConverter, &path_bytes, &threads, + &progress, &cancel)) { + return NULL; // Error already set + } + + CoreContext ctx; + int result = -1; + if (core_context_init(&ctx, self, progress, cancel, threads) == 0) { + result = verify_file(PyBytes_AS_STRING(path_bytes), &ctx); + } + + Py_DECREF(path_bytes); + return finish_call(self, result, &ctx, NULL); +} + // ---- Format Detection Methods ---- static PyObject *py_detect_format(PyObject *Py_UNUSED(self), PyObject *args) { @@ -740,6 +835,30 @@ static PyObject *py_gzip_stored_name(PyObject *Py_UNUSED(self), return result; } +static PyObject *py_stored_content_size(PyObject *Py_UNUSED(self), + PyObject *args) { + PyObject *file_path_bytes = NULL; + + if (!PyArg_ParseTuple(args, "O&", PyUnicode_FSConverter, &file_path_bytes)) { + return NULL; // Error already set + } + + Format format; + FILE *f = open_source(PyBytes_AS_STRING(file_path_bytes), &format); + Py_DECREF(file_path_bytes); + if (!f) { + return NULL; + } + + uint64_t size; + int stated = standalone_content_size(f, format, &size) == 0; + fclose(f); + if (!stated) { + Py_RETURN_NONE; + } + return PyLong_FromUnsignedLongLong(size); +} + static PyObject *py_format_is_archive(PyObject *Py_UNUSED(self), PyObject *args) { const char *format_name = NULL; @@ -754,6 +873,43 @@ static PyObject *py_format_is_archive(PyObject *Py_UNUSED(self), pipe.archive != ARCHIVE_NONE); } +static PyObject *py_format_stores_hardlinks(PyObject *Py_UNUSED(self), + PyObject *args) { + const char *format_name = NULL; + + if (!PyArg_ParseTuple(args, "s", &format_name)) { + return NULL; // Error already set + } + + CompressionPipeline pipe = pipeline_from_name(format_name, -1); + const CArchive *archive = + pipeline_is_valid(&pipe) ? find_archive_by_id(pipe.archive) : NULL; + + return PyBool_FromLong(archive && archive->stores_hardlinks); +} + +static PyObject *py_path_matches(PyObject *Py_UNUSED(self), PyObject *args) { + PyObject *path = NULL; + PathList patterns = {0}; + + if (!PyArg_ParseTuple(args, "UO&", &path, pattern_list_converter, + &patterns)) { + return NULL; // Error already set + } + + // The inverse of decode_entry_name, so a non-UTF-8 name keeps its bytes + PyObject *encoded = + PyUnicode_AsEncodedString(path, "utf-8", "surrogateescape"); + PyObject *result = + encoded ? PyBool_FromLong(path_matches_any( + PyBytes_AS_STRING(encoded), patterns.paths, patterns.count)) + : NULL; + + Py_XDECREF(encoded); + pattern_list_converter(NULL, &patterns); + return result; +} + // The set entries of a FormatDesc name slot, which fill from the front static PyObject *names_to_py(const char *const names[2]) { if (!names[0]) @@ -888,11 +1044,23 @@ static PyMethodDef CoreMethods[] = { {"decompress_standalone", (PyCFunction)(void (*)(void))py_decompress_standalone, METH_VARARGS | METH_KEYWORDS, "Decompress a standalone format file."}, + {"verify_file", (PyCFunction)(void (*)(void))py_verify_file, + METH_VARARGS | METH_KEYWORDS, + "Decode an archive or compressed file, writing nothing."}, {"detect_format", (PyCFunction)py_detect_format, METH_VARARGS, "Detect the format of a file."}, {"gzip_stored_name", (PyCFunction)py_gzip_stored_name, METH_VARARGS, "Return the name a gzip header stores (FNAME), or None."}, + {"stored_content_size", (PyCFunction)py_stored_content_size, METH_VARARGS, + "Return the uncompressed size an xz, zstd, lz4 or snappy file's own " + "metadata states, or None."}, + {"format_stores_hardlinks", (PyCFunction)py_format_stores_hardlinks, + METH_VARARGS, + "Return whether an archive format stores a file's later names as links."}, + {"path_matches", (PyCFunction)py_path_matches, METH_VARARGS, + "Return whether any glob matches an entry path, as the archive filters " + "match it."}, {"format_is_archive", (PyCFunction)py_format_is_archive, METH_VARARGS, "Check if a format is an archive format."}, {"standalone_formats", (PyCFunction)py_standalone_formats, METH_NOARGS, @@ -938,6 +1106,7 @@ static int core_exec(PyObject *module) { &errors->error}, {&errors->incomplete_extraction, "compresso.IncompleteExtractionError", &errors->error}, + {&errors->source_removal, "compresso.SourceRemovalError", &errors->error}, {&errors->trailing_data_warning, "compresso.TrailingDataWarning", &PyExc_UserWarning}, }; @@ -963,6 +1132,7 @@ static int core_traverse(PyObject *module, visitproc visit, void *arg) { Py_VISIT(errors->corrupt_data); Py_VISIT(errors->extraction_policy); Py_VISIT(errors->incomplete_extraction); + Py_VISIT(errors->source_removal); Py_VISIT(errors->cancelled); Py_VISIT(errors->trailing_data_warning); return 0; @@ -976,6 +1146,7 @@ static int core_clear(PyObject *module) { Py_CLEAR(errors->corrupt_data); Py_CLEAR(errors->extraction_policy); Py_CLEAR(errors->incomplete_extraction); + Py_CLEAR(errors->source_removal); Py_CLEAR(errors->cancelled); Py_CLEAR(errors->trailing_data_warning); return 0; diff --git a/src/compresso/csrc/archives.h b/src/compresso/csrc/archives.h index bacaa4d..7381570 100644 --- a/src/compresso/csrc/archives.h +++ b/src/compresso/csrc/archives.h @@ -157,19 +157,33 @@ int pipeline_is_valid(const CompressionPipeline *p); // ---- High-Level Operations ---- -// `overwrite_existing` follows the same 0-3 scheme as -// `ExtractionPolicy.overwrite_existing`; on success, the path actually written -// (which RENAME may have changed) is copied into `out_actual_path` +// Globs on entry names: an exclude wins, and with any includes an entry must +// match one, or be a directory holding one that does (when archiving) or a +// requested name (when extracting) +typedef struct { + const char **include; + size_t num_include; + const char **exclude; + size_t num_exclude; +} PathFilter; + +// `overwrite_existing` is as in `ExtractionPolicy`; `out_actual_path` receives +// the path written, which RENAME may change; `remove_sources` deletes only what +// was stored; with SKIP, a file appearing during the run stops the commit, but +// still returns success; `follow_links` stores symlink targets, and +// `remove_sources` then deletes the links and not targets int create_archive(const char *output_path, const CompressionPipeline *pipeline, const char **input_paths, size_t num_paths, - int overwrite_existing, char *out_actual_path, + const PathFilter *filter, int overwrite_existing, + int remove_sources, int follow_links, char *out_actual_path, size_t out_actual_path_size, CoreContext *ctx); // Entries are checked against `policy`; any failure but a cancel undoes what // the extraction wrote, restoring what OVERWRITE replaced int extract_archive(const char *archive_path, const char *output_dir, const char **files, size_t num_files, - const ExtractionPolicy *policy, CoreContext *ctx); + const PathFilter *filter, const ExtractionPolicy *policy, + CoreContext *ctx); // `ctx` carries signal checks and warnings; there is no progress to report PyObject *list_archive_contents(const char *archive_path, CoreContext *ctx); diff --git a/src/compresso/csrc/archives/archive_ops.c b/src/compresso/csrc/archives/archive_ops.c index 9326dbe..e8a433b 100644 --- a/src/compresso/csrc/archives/archive_ops.c +++ b/src/compresso/csrc/archives/archive_ops.c @@ -1,10 +1,10 @@ #define PY_SSIZE_T_CLEAN #include #include "archive_ops.h" -#include "paths.h" #include "../common.h" #include "../fsutil.h" #include "../standalone.h" +#include "paths.h" #include "walk.h" #include #include @@ -81,32 +81,9 @@ static int check_storable_entry_name(const char *name) { // A walked path's entry name, or NULL with an exception set static char *entry_name(const WalkItem *item) { - const char *rel_path = item->path + item->prefix_len; - while (FS_IS_SEP(*rel_path)) - rel_path++; - - char *name = strdup(rel_path); - if (!name) { - PyErr_NoMemory(); + char *name = walk_entry_name(item->path, item->prefix_len); + if (!name) return NULL; - } - - // Normalise to '/', collapsing runs and dropping a trailing one; writers add - // the trailing separator a directory needs - char *w = name; - for (const char *r = name; *r; r++) { - if (FS_IS_SEP(*r)) { - // Keep a leading separator so the absolute name is still rejected below - if (w > name && w[-1] == '/') - continue; - *w++ = '/'; - } else { - *w++ = *r; - } - } - while (w > name + 1 && w[-1] == '/') - w--; - *w = '\0'; if (check_storable_entry_name(name) != 0) { free(name); @@ -206,12 +183,67 @@ static int add_walked_entries(const CArchive *archive, void *writer, return 0; } +// Deepest first; a directory holding something unstored stays, and so does "."; +// a followed link goes, but not its target; a path that won't go lands in +// `failed`, and the rest still go +static int remove_walked(const WalkList *walked, PyObject *failed) { + for (size_t i = walked->count; i-- > 0;) { + const WalkItem *item = &walked->items[i]; + if (item->via_link) + continue; + int is_dir = item->st.type == FS_TYPE_DIR && !item->followed; + if ((is_dir ? fs_rmdir(item->path) : fs_unlink(item->path)) == 0) + continue; + if (is_dir && (errno == ENOTEMPTY || errno == EEXIST || errno == EINVAL)) + continue; + + const char *reason = strerror(errno); + PyObject *failure = + Py_BuildValue("(Ns)", PyUnicode_DecodeFSDefault(item->path), reason); + int rc = failure ? PyList_Append(failed, failure) : -1; + Py_XDECREF(failure); + if (rc != 0) + return -1; + } + return 0; +} + +// The archive is committed by now, so what's raised also says where it went +static int remove_stored(const WalkList *walked, const char *output, + const CompErrors *errors) { + PyObject *failed = PyList_New(0); + int ret = failed ? remove_walked(walked, failed) : -1; + + if (ret == 0 && PyList_GET_SIZE(failed) > 0) { + ret = -1; + PyObject *error = PyObject_CallFunction( + errors->source_removal, "s", + "Archive written, but some of its sources could not be deleted"); + PyObject *path = PyUnicode_DecodeFSDefault(output); + if (error && path && PyObject_SetAttrString(error, "output", path) == 0 && + PyObject_SetAttrString(error, "failures", failed) == 0) + PyErr_SetObject(errors->source_removal, error); + Py_XDECREF(path); + Py_XDECREF(error); + } + + Py_XDECREF(failed); + return ret; +} + // ---- Archive Operations ---- int create_archive(const char *output_path, const CompressionPipeline *pipeline, const char **input_paths, size_t num_paths, - int overwrite_existing, char *out_actual_path, + const PathFilter *filter, int overwrite_existing, + int remove_sources, int follow_links, char *out_actual_path, size_t out_actual_path_size, CoreContext *ctx) { + if (remove_sources && overwrite_existing == 1) { + PyErr_SetString(PyExc_ValueError, + "remove_sources can't be combined with skip"); + return -1; + } + if (!pipeline_is_valid(pipeline) || pipeline->archive == ARCHIVE_NONE) { char name[32]; pipeline_display_name(pipeline, name, sizeof(name)); @@ -276,13 +308,13 @@ int create_archive(const char *output_path, const CompressionPipeline *pipeline, own_files_add(&own, output_path); WalkList walked = {0}; - int ret = walk_sources(input_paths, num_paths, &own, archive->stores_specials, - archive->stores_hardlinks, ctx, &walked); + int ret = walk_sources(input_paths, num_paths, filter, &own, + archive->stores_specials, archive->stores_hardlinks, + follow_links, ctx, &walked); if (ret == 0) { ctx_begin_stage(ctx, walked.file_bytes); ret = add_walked_entries(archive, writer, &walked, ctx); } - walk_list_free(&walked); // libzip does all its compression inside zip_close, so that is where its // progress comes from; a run that already failed is discarded instead @@ -292,28 +324,33 @@ int create_archive(const char *output_path, const CompressionPipeline *pipeline, if (!stream) { if (ret == 0) - return output_commit(tmp_path, &out); - fs_unlink(tmp_path); - return ret; - } - - // libarchive only knows its write callback failed; failed streams return - // a failure to every write; empty ones swap in the cause (e.g. cancelled) - if (ret != 0) { - int stream_ret = codec_write(stream, NULL, 0); - if (stream_ret != 0) { - PyErr_Clear(); - ret = stream_ret; + ret = output_commit(tmp_path, &out); + else + fs_unlink(tmp_path); + } else { + // libarchive only knows its write callback failed; failed streams return + // a failure to every write; empty ones swap in the cause (e.g. cancelled) + if (ret != 0) { + int stream_ret = codec_write(stream, NULL, 0); + if (stream_ret != 0) { + PyErr_Clear(); + ret = stream_ret; + } } - } - int codec_ret = codec_writer_close(stream, ret != 0); - if (ret == 0) - ret = codec_ret; + int codec_ret = codec_writer_close(stream, ret != 0); + if (ret == 0) + ret = codec_ret; + + char failure[64]; + snprintf(failure, sizeof(failure), "%s compression failed", fmt->name); + ret = output_finish(ret, NULL, tmp, tmp_path, &out, ctx->errors, failure); + } - char failure[64]; - snprintf(failure, sizeof(failure), "%s compression failed", fmt->name); - return output_finish(ret, NULL, tmp, tmp_path, &out, ctx->errors, failure); + if (ret == 0 && remove_sources) + ret = remove_stored(&walked, out_actual_path, ctx->errors); + walk_list_free(&walked); + return ret; } FILE *open_archive_source(const char *archive_path, const CArchive **archive, @@ -468,12 +505,19 @@ static PyObject *read_archive_entries(const CArchive *archive, void *reader, return list; } -PyObject *list_archive_contents(const char *archive_path, CoreContext *ctx) { +// One read of an archive; `data_ctx` takes the progress of its entries' data, +// and is NULL when a codec stage already counts the input; returns 0 or a +// negative code with an exception set +typedef int (*ReaderPass)(const CArchive *archive, void *reader, void *state, + CoreContext *data_ctx, CoreContext *ctx); + +static int run_reader_pass(const char *archive_path, ReaderPass pass, + void *state, CoreContext *ctx) { const CArchive *archive; CompressionPipeline pipe; FILE *src = open_archive_source(archive_path, &archive, &pipe, ctx); if (!src) - return NULL; + return -1; // Decoded as libarchive reads CodecStream *stream = NULL; @@ -490,24 +534,111 @@ PyObject *list_archive_contents(const char *archive_path, CoreContext *ctx) { reader = archive->create_reader(archive_path, ctx); } - PyObject *list = reader ? read_archive_entries(archive, reader, ctx) : NULL; - int failed = list == NULL; - // A close failure after a full listing would otherwise return the list - if (reader && archive->close_reader(reader, failed) != 0) - failed = 1; + // A cancel while opening fails the open, which can only return NULL + int ret = reader ? pass(archive, reader, state, stream ? NULL : ctx, ctx) + : ctx_cancelled(ctx) ? COMP_CANCELLED + : -1; + + // A close failure after a full pass would otherwise go unreported + if (reader && archive->close_reader(reader, ret != 0) != 0 && ret == 0) + ret = -1; - if (stream && close_codec_source(stream, src, !failed) != 0) - failed = 1; - else if (!stream && src) + if (stream) { + int rc = close_codec_source(stream, src, ret == 0); + if (ret == 0) + ret = rc; + } else if (src) { fclose(src); + } + return ret; +} + +static int list_pass(const CArchive *archive, void *reader, void *state, + CoreContext *data_ctx, CoreContext *ctx) { + (void)data_ctx; + PyObject **list = state; + *list = read_archive_entries(archive, reader, ctx); + return *list ? 0 : -1; +} - if (failed) { +PyObject *list_archive_contents(const char *archive_path, CoreContext *ctx) { + PyObject *list = NULL; + if (run_reader_pass(archive_path, list_pass, &list, ctx) != 0) { Py_XDECREF(list); return NULL; } return list; } +typedef struct { + FILE *sink; // Takes each file's data; NULL to only add up `declared` + uint64_t declared; +} VerifyState; + +static int verify_pass(const CArchive *archive, void *reader, void *state, + CoreContext *data_ctx, CoreContext *ctx) { + VerifyState *verify = state; + ArchiveEntry entry = {0}; + int ret; + + // NULL: the data pass reads the same headers again, and warns once there + while ((ret = archive->get_next_entry(reader, &entry, + verify->sink ? ctx : NULL)) == 1) { + int rc = ctx_advance(ctx, 0); + if (rc == 0 && entry.type == ENTRY_FILE) { + uint64_t written; + if (verify->sink) + rc = archive->extract_entry_data(reader, verify->sink, UINT64_MAX, + &written, data_ctx); + else + verify->declared += entry.size; + } + entry_reset(&entry); + if (rc != 0) + return rc; + } + return ret; +} + +static int verify_archive(const char *archive_path, + const CompressionPipeline *pipeline, + CoreContext *ctx) { + FILE *sink = fs_fopen(FS_NULL_DEVICE, "wb"); + if (!sink) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, FS_NULL_DEVICE); + return -1; + } + + // A codec stream is decoded once, counting its compressed bytes; a plain file + // gets a headers pass first, for the total its data counts towards + VerifyState state = {NULL, 0}; + int ret = 0; + if (pipeline->codec == FORMAT_UNKNOWN) { + ret = run_reader_pass(archive_path, verify_pass, &state, ctx); + ctx_begin_stage(ctx, state.declared); + } + if (ret == 0) { + state.sink = sink; + ret = run_reader_pass(archive_path, verify_pass, &state, ctx); + } + + fclose(sink); + return ret; +} + +int verify_file(const char *path, CoreContext *ctx) { + Format format; + FILE *src = open_source(path, &format); + if (!src) + return -1; + fclose(src); + + CompressionPipeline pipe = pipeline_from_format(format, path); + if (pipe.archive != ARCHIVE_NONE) + return verify_archive(path, &pipe, ctx); + return decompress_file(path, NULL, ALGO_NONE, 0, NULL, 0, ctx); +} + // ---- Archive Capabilities ---- PyObject *get_archive_capabilities(void) { diff --git a/src/compresso/csrc/archives/extract.c b/src/compresso/csrc/archives/extract.c index 1975540..75ee161 100644 --- a/src/compresso/csrc/archives/extract.c +++ b/src/compresso/csrc/archives/extract.c @@ -302,14 +302,27 @@ static int make_node(const void *entry, const char *out_path) { return fs_mknod(out_path, e->special, e->mode, e->rdev); } +// Appends "path (reason)" to `skipped_nodes`; returns 0, or -1 with an +// exception set +static int skip_node(PyObject *skipped_nodes, const char *path, + const char *reason) { + PyObject *node = PyUnicode_FromFormat("%s (%s)", path, reason); + int rc = node ? PyList_Append(skipped_nodes, node) : -1; + Py_XDECREF(node); + return rc; +} + // Runs `make(source, out_path)` under the overwrite mode, as for a file; -// returns 0, or -1 with an exception set +// returns 0, 1 if symlinks can't be created here, or -1 with an exception set static int place_link(int (*make)(const void *, const char *), const void *source, const char *out_path, const char *entry_path, int overwrite_existing, int64_t mtime, Changes *changes) { if (make(source, out_path) == 0) return link_made(changes, out_path, mtime); + // Only here, as nothing has been moved aside yet + if (make == make_symlink && errno == EPERM) + return 1; if (errno != EEXIST) { PyErr_SetFromErrnoWithFilename(PyExc_OSError, out_path); return -1; @@ -421,9 +434,11 @@ static int extract_hardlink(const CompErrors *errors, const char *resolved_root, policy->overwrite_existing, 0, changes); } +// A symlink this user or filesystem can't create is appended to `skipped_nodes` static int extract_symlink(const CompErrors *errors, const char *resolved_root, const char *out_path, const ArchiveEntry *entry, - const ExtractionPolicy *policy, Changes *changes) { + const ExtractionPolicy *policy, + PyObject *skipped_nodes, Changes *changes) { char parent[FS_PATH_MAX]; if (prepare_parent(errors, resolved_root, out_path, entry->path, changes, parent) != 0) @@ -432,26 +447,29 @@ static int extract_symlink(const CompErrors *errors, const char *resolved_root, if (check_target_from(errors, resolved_root, parent, entry) != 0) return -1; - return place_link(make_symlink, entry->link_target, out_path, entry->path, - policy->overwrite_existing, - policy->preserve_timestamps ? (int64_t)entry->mtime : 0, - changes); + int rc = place_link(make_symlink, entry->link_target, out_path, entry->path, + policy->overwrite_existing, + policy->preserve_timestamps ? (int64_t)entry->mtime : 0, + changes); + if (rc != 1) + return rc; + return skip_node(skipped_nodes, entry->path, + "this user or filesystem can't create symlinks"); } -// A node this platform or user can't create is skipped with a warning; returns -// 0, or -1 with an exception set +// A node this platform or user can't create is appended to `skipped_nodes` as +// "path (reason)"; returns 0, or -1 with an exception set static int extract_special(const CompErrors *errors, const char *resolved_root, const char *out_path, const ArchiveEntry *entry, const ExtractionPolicy *policy, uint32_t mode_mask, - CoreContext *ctx, Changes *changes) { - if (!fs_can_mknod(entry->special)) { - ctx_log(ctx, CTX_LOG_WARNING, "Skipped %s: %s", entry->path, - !entry->special ? "sockets are not extracted" - : entry->special == AE_IFIFO - ? "FIFOs can't be created on this platform" - : "devices are only created by root, and not on Windows"); - return 0; - } + PyObject *skipped_nodes, Changes *changes) { + if (!fs_can_mknod(entry->special)) + return skip_node( + skipped_nodes, entry->path, + !entry->special ? "sockets are not extracted" + : entry->special == AE_IFIFO + ? "FIFOs can't be created on this platform" + : "devices are only created by root, and not on Windows"); char parent[FS_PATH_MAX]; if (prepare_parent(errors, resolved_root, out_path, entry->path, changes, @@ -621,13 +639,14 @@ static int extract_file(const CArchive *archive, void *reader, // Extract each entry from an open reader into `changes` and `deferred`; each // hardlink whose target is missing is appended to `skipped_links` as -// "link -> target"; `data_ctx` counts bytes written, NULL callers count input +// "link -> target", and each node that can't be created to `skipped_nodes`; +// `data_ctx` counts bytes written, NULL callers count input static int extract_entries(const CArchive *archive, void *reader, const char *output_dir, const char *resolved_root, Selection *sel, const ExtractionPolicy *policy, CoreContext *ctx, CoreContext *data_ctx, Changes *changes, DeferredDirs *deferred, - PyObject *skipped_links) { + PyObject *skipped_links, PyObject *skipped_nodes) { ArchiveEntry entry = {0}; uint64_t written_total = 0; Renames renames = {0}; @@ -691,7 +710,7 @@ static int extract_entries(const CArchive *archive, void *reader, policy, mode_mask, &renames, changes, deferred); else if (entry.type == ENTRY_SYMLINK && policy->allow_symlinks == 1) rc = extract_symlink(ctx->errors, resolved_root, out_path, &entry, policy, - changes); + skipped_nodes, changes); else if (entry.type == ENTRY_FILE || entry.type == ENTRY_SYMLINK) rc = extract_file(archive, reader, ctx->errors, resolved_root, out_path, &entry, policy, mode_mask, data_ctx, &written_total, @@ -708,7 +727,7 @@ static int extract_entries(const CArchive *archive, void *reader, } } else if (entry.type == ENTRY_SPECIAL) rc = extract_special(ctx->errors, resolved_root, out_path, &entry, policy, - mode_mask, ctx, changes); + mode_mask, skipped_nodes, changes); if (rc < 0) { result = rc; goto cleanup; @@ -727,7 +746,8 @@ static int extract_entries(const CArchive *archive, void *reader, int extract_archive(const char *archive_path, const char *output_dir, const char **files, size_t num_files, - const ExtractionPolicy *policy, CoreContext *ctx) { + const PathFilter *filter, const ExtractionPolicy *policy, + CoreContext *ctx) { const CArchive *archive; CompressionPipeline pipe; FILE *src = open_archive_source(archive_path, &archive, &pipe, ctx); @@ -763,8 +783,11 @@ int extract_archive(const char *archive_path, const char *output_dir, Selection sel; PyObject *skipped_links = PyList_New(0); - if (!skipped_links || selection_init(&sel, files, num_files) != 0) { + PyObject *skipped_nodes = PyList_New(0); + if (!skipped_links || !skipped_nodes || + selection_init(&sel, files, num_files, filter) != 0) { Py_XDECREF(skipped_links); + Py_XDECREF(skipped_nodes); if (src) fclose(src); return -1; @@ -791,7 +814,8 @@ int extract_archive(const char *archive_path, const char *output_dir, reader = archive->create_reader(archive_path, ctx); } if (!reader) { - ret = -1; + // A cancel while opening fails the open, which can only return NULL + ret = ctx_cancelled(ctx) ? COMP_CANCELLED : -1; break; } @@ -800,7 +824,7 @@ int extract_archive(const char *archive_path, const char *output_dir, &sel, policy, ctx, &declared_total) : extract_entries(archive, reader, output_dir, resolved_root, &sel, policy, ctx, src ? NULL : ctx, &changes, - &deferred, skipped_links); + &deferred, skipped_links, skipped_nodes); // A failed pass keeps its own error if (archive->close_reader(reader, ret != 0) != 0 && ret == 0) @@ -825,8 +849,10 @@ int extract_archive(const char *archive_path, const char *output_dir, // After everything else has been extracted if (ret == 0) - ret = report_incomplete(&sel, skipped_links, ctx); + ret = report_incomplete(&sel, skipped_links, skipped_nodes, ctx); Py_DECREF(skipped_links); + Py_DECREF(skipped_nodes); free(sel.items); + free(sel.include_found); return ret; } diff --git a/src/compresso/csrc/archives/select.c b/src/compresso/csrc/archives/select.c index 902dc97..6d6fb46 100644 --- a/src/compresso/csrc/archives/select.c +++ b/src/compresso/csrc/archives/select.c @@ -11,14 +11,23 @@ static int selected_name_cmp(const void *a, const void *b) { return c ? c : (x->len > y->len) - (x->len < y->len); } -int selection_init(Selection *sel, const char **files, size_t num_files) { - *sel = (Selection){0}; +int selection_init(Selection *sel, const char **files, size_t num_files, + const PathFilter *filter) { + *sel = (Selection){.filter = *filter}; + if (filter->num_include > 0) { + sel->include_found = safe_malloc(filter->num_include); + if (!sel->include_found) + return -1; + memset(sel->include_found, 0, filter->num_include); + } if (num_files == 0) return 0; sel->items = safe_malloc(num_files * sizeof(*sel->items)); - if (!sel->items) + if (!sel->items) { + free(sel->include_found); return -1; + } for (size_t i = 0; i < num_files; i++) { size_t len = strlen(files[i]); @@ -39,11 +48,9 @@ int selection_init(Selection *sel, const char **files, size_t num_files) { } int entry_is_selected(const ArchiveEntry *entry, Selection *sel) { - if (sel->count == 0) - return 1; - - int selected = 0; - for (const char *p = entry->path;; p++) { + const PathFilter *filter = &sel->filter; + int selected = sel->count == 0 && filter->num_include == 0; + for (const char *p = entry->path; sel->count > 0; p++) { if ((*p == '/' || *p == '\0') && p > entry->path) { SelectedName key = {entry->path, (size_t)(p - entry->path), 0}; SelectedName *hit = bsearch(&key, sel->items, sel->count, @@ -54,8 +61,99 @@ int entry_is_selected(const ArchiveEntry *entry, Selection *sel) { } } if (*p == '\0') - return selected; + break; } + + for (size_t i = 0; i < filter->num_include; i++) + if ((!selected || !sel->include_found[i]) && + path_matches(entry->path, filter->include[i])) { + sel->include_found[i] = 1; + selected = 1; + } + + // Excludes come last, so dropped entries still counts as found and aren't + // reported as missing from the archive + return selected && + !path_matches_any(entry->path, filter->exclude, filter->num_exclude); +} + +// The pattern after its first token if that token matches `c`, else NULL +static const char *match_token(const char *pat, unsigned char c) { + if (*pat == '?') + return pat + 1; + if (*pat == '\\' && pat[1]) + return (unsigned char)pat[1] == c ? pat + 2 : NULL; + + if (*pat == '[') { + const char *p = pat + 1; + int negate = *p == '!' || *p == '^'; + if (negate) + p++; + // A `]` straight after the opening is literal + int hit = 0; + do { + if (*p == '\0') + return c == '[' ? pat + 1 : NULL; // Unterminated, so `[` is literal + unsigned char lo = (unsigned char)*p, hi = lo; + if (p[1] == '-' && p[2] && p[2] != ']') { + hi = (unsigned char)p[2]; + p += 2; + } + hit |= lo <= c && c <= hi; + p++; + } while (*p != ']'); + return hit != negate ? p + 1 : NULL; + } + + return (unsigned char)*pat == c ? pat + 1 : NULL; +} + +// Whether `pat` matches `s` up to its end or a '/'; backtracks only to the +// last `*`, so the time stays linear: O(len * strlen(pat)) +static int glob_match(const char *pat, const char *s, size_t len) { + const char *star = NULL; + size_t retry = 0, i = 0; + while (i < len) { + if (*pat == '\0' && i > 0 && s[i] == '/') + return 1; + if (*pat == '*') { + star = ++pat; + retry = i; + continue; + } + const char *next = match_token(pat, (unsigned char)s[i]); + if (next) { + pat = next; + i++; + } else if (star) { + pat = star; + i = ++retry; + } else { + return 0; + } + } + while (*pat == '*') + pat++; + return *pat == '\0'; +} + +int path_matches(const char *path, const char *pat) { + size_t len = strlen(path); + while (len > 0 && path[len - 1] == '/') + len--; + + for (size_t start = 0; start < len; start++) + if ((start == 0 || path[start - 1] == '/') && + glob_match(pat, path + start, len - start)) + return 1; + return 0; +} + +int path_matches_any(const char *path, const char **pats, size_t count) { + for (size_t i = 0; i < count; i++) + if (path_matches(path, pats[i])) + return 1; + return 0; } static PyObject *join_str(const char *sep, PyObject *items) { @@ -79,7 +177,7 @@ static int append_part(PyObject *parts, const char *label, PyObject *items) { } int report_incomplete(const Selection *sel, PyObject *skipped_links, - CoreContext *ctx) { + PyObject *skipped_nodes, CoreContext *ctx) { PyObject *names = PyList_New(0); PyObject *parts = PyList_New(0); int ret = -1; @@ -95,8 +193,18 @@ int report_incomplete(const Selection *sel, PyObject *skipped_links, if (rc != 0) goto done; } + for (size_t i = 0; i < sel->filter.num_include; i++) { + if (sel->include_found[i]) + continue; + PyObject *pattern = PyUnicode_FromString(sel->filter.include[i]); + int rc = pattern ? PyList_Append(names, pattern) : -1; + Py_XDECREF(pattern); + if (rc != 0) + goto done; + } if (append_part(parts, "Not found in archive", names) != 0 || - append_part(parts, "Hardlink target missing", skipped_links) != 0) + append_part(parts, "Hardlink target missing", skipped_links) != 0 || + append_part(parts, "Skipped, the rest was extracted", skipped_nodes) != 0) goto done; if (PyList_GET_SIZE(parts) == 0) { diff --git a/src/compresso/csrc/archives/select.h b/src/compresso/csrc/archives/select.h index c6f34a1..b1d20c3 100644 --- a/src/compresso/csrc/archives/select.h +++ b/src/compresso/csrc/archives/select.h @@ -4,7 +4,8 @@ #include "../archives.h" #include -// The requested names, sorted for bsearch; none means everything +// The requested names, sorted for bsearch; with no includes, none means +// everything typedef struct { const char *name; size_t len; // Without trailing slashes @@ -14,17 +15,31 @@ typedef struct { typedef struct { SelectedName *items; size_t count; + PathFilter filter; + char *include_found; // One flag per include; NULL with none } Selection; -// Borrows `files`, which must outlive `sel`; free `sel->items` when done -int selection_init(Selection *sel, const char **files, size_t num_files); +// Borrows `files` and the filter's patterns, which must outlive `sel`; free +// `sel->items` and `sel->include_found` when done +int selection_init(Selection *sel, const char **files, size_t num_files, + const PathFilter *filter); -// Whether `entry` or a directory above it was requested, as tar matches names; -// marks every name that matches, so "d" and "d/x" both count as found +// Whether `entry` or a directory above it was requested, as tar matches names, +// or an include matches it, and no exclude does; marks every name and include +// that matches, so "d" and "d/x" both count as found int entry_is_selected(const ArchiveEntry *entry, Selection *sel); -// Raises an error listing unmatched requested names and skipped hardlinks +// Whether the glob `pat` matches any run of whole components in `path`: e.g. +// "*.log" matches "a/b.log", and "a" matches everything below "a/"; the syntax +// is fnmatch(3)'s with no flags, so `*` and `?` also match '/' +int path_matches(const char *path, const char *pat); + +// Whether any of `count` patterns matches `path` +int path_matches_any(const char *path, const char **pats, size_t count); + +// Raises an error listing unmatched names and includes, skipped hardlinks, and +// nodes that couldn't be created int report_incomplete(const Selection *sel, PyObject *skipped_links, - CoreContext *ctx); + PyObject *skipped_nodes, CoreContext *ctx); #endif // SELECT_H diff --git a/src/compresso/csrc/archives/walk.c b/src/compresso/csrc/archives/walk.c index 1409920..d043a38 100644 --- a/src/compresso/csrc/archives/walk.c +++ b/src/compresso/csrc/archives/walk.c @@ -2,6 +2,7 @@ #include #include "walk.h" #include "../common.h" +#include "select.h" #include #include @@ -50,10 +51,8 @@ static int push(WalkList *list, char *path, size_t prefix_len) { static int is_skipped(const WalkItem *item, const OwnFiles *own, int keep_specials, CoreContext *ctx) { - if (item->st.type == FS_TYPE_OTHER && - !(keep_specials && item->st.special)) { - ctx_log(ctx, CTX_LOG_WARNING, "Skipped %s: %s are not archived", - item->path, + if (item->st.type == FS_TYPE_OTHER && !(keep_specials && item->st.special)) { + ctx_log(ctx, CTX_LOG_WARNING, "Skipped %s: %s are not archived", item->path, keep_specials ? "sockets" : "FIFOs, sockets and devices"); return 1; } @@ -68,8 +67,36 @@ static int is_skipped(const WalkItem *item, const OwnFiles *own, return 0; } -// Pushes the children of `dir` so they pop in the order readdir gave them -static int push_children(WalkList *pending, const WalkItem *dir) { +static void follow_link(WalkItem *item, const WalkList *walked) { + fs_stat target; + if (fs_stat_target(item->path, &target) != 0) + return; + + for (size_t i = item->parent; i > 0; i = walked->items[i - 1].parent) + if (fs_same_file(&target, &walked->items[i - 1].st)) + return; + + item->st = target; + item->followed = 1; +} + +// Only where the link was meant to give data: left in a tree it's a faithful +// copy, and such links are routine +static void warn_if_dangling(const WalkItem *item, int follow_links, + CoreContext *ctx) { + fs_stat target; + if (item->st.type != FS_TYPE_SYMLINK || !(follow_links || item->parent == 0)) + return; + + if (fs_stat_target(item->path, &target) != 0) + ctx_log(ctx, CTX_LOG_WARNING, + "Stored %s as a link: its target can't be reached", item->path); +} + +// Pushes the children of `dir`, whose index is `parent`, so they pop in the +// order readdir gave them +static int push_children(WalkList *pending, const WalkItem *dir, + size_t parent) { fs_dir *d = fs_opendir(dir->path); if (!d) { PyErr_SetFromErrnoWithFilename(PyExc_OSError, dir->path); @@ -90,6 +117,9 @@ static int push_children(WalkList *pending, const WalkItem *dir) { ret = -1; break; } + pending->items[pending->count - 1].parent = parent; + pending->items[pending->count - 1].via_link = + dir->followed || dir->via_link; } // A conversion failure also ends fs_readdir; unchecked, files would silently @@ -141,8 +171,7 @@ static int find_hardlinks(WalkList *list) { for (size_t i = 1; i < n; i++) { if (!fs_same_file(&linked[i]->st, &linked[i - 1]->st)) continue; - linked[i]->link = - linked[i - 1]->link ? linked[i - 1]->link : linked[i - 1]; + linked[i]->link = linked[i - 1]->link ? linked[i - 1]->link : linked[i - 1]; if (linked[i]->st.type == FS_TYPE_FILE) list->file_bytes -= linked[i]->st.size; } @@ -150,8 +179,67 @@ static int find_hardlinks(WalkList *list) { return 0; } +char *walk_entry_name(const char *path, size_t prefix_len) { + const char *rel_path = path + prefix_len; + while (FS_IS_SEP(*rel_path)) + rel_path++; + + char *name = strdup(rel_path); + if (!name) { + PyErr_NoMemory(); + return NULL; + } + + // Normalise to '/', collapsing runs and dropping a trailing one; writers add + // the trailing separator a directory needs + char *w = name; + for (const char *r = name; *r; r++) { + if (FS_IS_SEP(*r)) { + // Keep a leading separator so the absolute name is still rejected + if (w > name && w[-1] == '/') + continue; + *w++ = '/'; + } else { + *w++ = *r; + } + } + while (w > name + 1 && w[-1] == '/') + w--; + *w = '\0'; + return name; +} + +// Whether `path` is below the directory `dir`, as fs_join builds child paths +static int is_below(const char *path, const char *dir) { + size_t n = strlen(dir); + return strncmp(path, dir, n) == 0 && + (FS_IS_SEP(path[n]) || (n > 0 && FS_IS_SEP(dir[n - 1]))); +} + +// Drops each provisional directory with nothing kept below it; depth-first, so +// the nearest kept item after a directory is below it if anything is +static void prune_provisional(WalkList *list) { + const char *next_kept = NULL; + for (size_t i = list->count; i-- > 0;) { + WalkItem *item = &list->items[i]; + if (item->provisional && !(next_kept && is_below(next_kept, item->path))) { + free(item->path); + item->path = NULL; + } else { + next_kept = item->path; + } + } + + size_t kept = 0; + for (size_t i = 0; i < list->count; i++) + if (list->items[i].path) + list->items[kept++] = list->items[i]; + list->count = kept; +} + int walk_sources(const char **input_paths, size_t num_paths, - const OwnFiles *own, int keep_specials, int link_hardlinks, + const PathFilter *filter, const OwnFiles *own, + int keep_specials, int link_hardlinks, int follow_links, CoreContext *ctx, WalkList *out) { // Depth-first like the archive's order, on a heap stack rather than the C one WalkList pending = {0}; @@ -172,29 +260,57 @@ int walk_sources(const char **input_paths, size_t num_paths, } WalkItem item = pending.items[--pending.count]; + + // Matched before the stat, so an excluded tree costs nothing to skip + int included = 1; + if (filter->num_include > 0 || filter->num_exclude > 0) { + char *name = walk_entry_name(item.path, item.prefix_len); + if (!name) { + free(item.path); + ret = -1; + break; + } + int excluded = + path_matches_any(name, filter->exclude, filter->num_exclude); + included = filter->num_include == 0 || + path_matches_any(name, filter->include, filter->num_include); + free(name); + if (excluded) { + free(item.path); + continue; + } + } + if (fs_stat_path(item.path, &item.st) != 0) { PyErr_SetFromErrnoWithFilename(PyExc_OSError, item.path); free(item.path); ret = -1; break; } - if (is_skipped(&item, own, keep_specials, ctx)) { + if (follow_links && item.st.type == FS_TYPE_SYMLINK) + follow_link(&item, out); + if ((!included && item.st.type != FS_TYPE_DIR) || + is_skipped(&item, own, keep_specials, ctx)) { free(item.path); continue; } + warn_if_dangling(&item, follow_links, ctx); ret = push(out, item.path, item.prefix_len); if (ret != 0) break; - out->items[out->count - 1].st = item.st; + item.provisional = !included; + out->items[out->count - 1] = item; if (item.st.type == FS_TYPE_DIR) - ret = push_children(&pending, &item); + ret = push_children(&pending, &item, out->count); else if (item.st.type == FS_TYPE_FILE) out->file_bytes += item.st.size; } walk_list_free(&pending); + if (ret == 0) + prune_provisional(out); if (ret == 0 && link_hardlinks) ret = find_hardlinks(out); return ret; diff --git a/src/compresso/csrc/archives/walk.h b/src/compresso/csrc/archives/walk.h index 08a3e5b..1d45860 100644 --- a/src/compresso/csrc/archives/walk.h +++ b/src/compresso/csrc/archives/walk.h @@ -1,6 +1,7 @@ #ifndef WALK_H #define WALK_H +#include "../archives.h" #include "../context.h" #include "../fsutil.h" #include @@ -20,6 +21,10 @@ typedef struct WalkItem { size_t prefix_len; // Its input's parent prefix, which entry names drop fs_stat st; const struct WalkItem *link; // The file's first name, if this is a later one + int provisional; // A directory kept only if something below it is + size_t parent; // 1-based index of its directory, 0 for an input + int followed; // A symlink stored as what it leads to + int via_link; // Reached through a followed symlink } WalkItem; typedef struct { @@ -28,13 +33,21 @@ typedef struct { uint64_t file_bytes; // The regular files' total, for progress } WalkList; -// Every input and everything below it, depth-first, symlinks not followed. -// Warns and skips `own` and sockets, and FIFOs and devices unless -// `keep_specials`; with `link_hardlinks`, a file's later names get `link` and -// aren't counted in `file_bytes`; returns COMP_CANCELLED, or -1 with an -// exception set +// The entry name of `path`, found below an input whose parent prefix is +// `prefix_len` long: '/'-separated, with runs collapsed and no trailing one; +// NULL with an exception set +char *walk_entry_name(const char *path, size_t prefix_len); + +// Every input and everything below it that `filter` keeps, depth-first, not +// entering excluded directories; warns and skips `own` and sockets, and FIFOs +// and devices unless `keep_specials`; `link_hardlinks` gives a file's later +// names `link` and leaves them out of `file_bytes`; `follow_links` walks +// symlinks as their targets, except dangling ones and those to an ancestor; +// dangling ones warn with this flag or as inputs; returns COMP_CANCELLED, or -1 +// with an exception set int walk_sources(const char **input_paths, size_t num_paths, - const OwnFiles *own, int keep_specials, int link_hardlinks, + const PathFilter *filter, const OwnFiles *own, + int keep_specials, int link_hardlinks, int follow_links, CoreContext *ctx, WalkList *out); void walk_list_free(WalkList *list); diff --git a/src/compresso/csrc/archives/zip.c b/src/compresso/csrc/archives/zip.c index 18116c0..d528bcf 100644 --- a/src/compresso/csrc/archives/zip.c +++ b/src/compresso/csrc/archives/zip.c @@ -524,9 +524,11 @@ static int zip_read_link_target(ZipReader *reader, zip_int64_t idx, return -1; } + // The CRC is only checked by a read that reaches the end char target[FS_PATH_MAX]; + char past_end; zip_int64_t n = zip_fread(zf, target, entry->size); - if (n < 0) { + if (n < 0 || zip_fread(zf, &past_end, 1) < 0) { set_zip_error(reader->errors, zip_file_get_error(zf), "Error reading from archive", reader->input_path, 1, 0); zip_fclose(zf); diff --git a/src/compresso/csrc/codec/lz4.c b/src/compresso/csrc/codec/lz4.c index 6d5b310..f39dc7d 100644 --- a/src/compresso/csrc/codec/lz4.c +++ b/src/compresso/csrc/codec/lz4.c @@ -6,7 +6,8 @@ #include // At the HC levels the output depends on how the input is split across -// LZ4F_compressUpdate calls, so it is fed a fixed amount whatever CODEC_CHUNK is +// LZ4F_compressUpdate calls, so it is fed a fixed amount whatever CODEC_CHUNK +// is #define LZ4_FEED 65536 // Room for every block one LZ4F_compressUpdate call can complete: with the @@ -45,6 +46,12 @@ static int lz4_begin(void *state, const CodecParams *params, int decompress, s->prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; } + // Lets `info` read the size back; liblz4 omits a zero, and fails at the end + // if the input isn't this long + if (params->exact_size) { + s->prefs.frameInfo.contentSize = params->orig_size; + } + return 0; } diff --git a/src/compresso/csrc/common.c b/src/compresso/csrc/common.c index ccf4156..f3e7914 100644 --- a/src/compresso/csrc/common.c +++ b/src/compresso/csrc/common.c @@ -198,6 +198,10 @@ static void report_actual(const OutputTarget *out, const char *path) { } int output_check(const char *src_path, const OutputTarget *out) { + if (!out->path) { + return 0; + } + char resolved[FS_PATH_MAX]; int rc = fs_resolve_conflict(out->path, out->overwrite, resolved, sizeof(resolved)); @@ -222,6 +226,14 @@ int output_check(const char *src_path, const OutputTarget *out) { } FILE *output_open(const OutputTarget *out, char *temp) { + if (!out->path) { + FILE *sink = fs_fopen(FS_NULL_DEVICE, "wb"); + if (!sink) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, FS_NULL_DEVICE); + } + return sink; + } + static const char SUFFIX[] = ".compresso-XXXXXX"; const char *slash = fs_last_sep(out->path); size_t dir_len = slash ? (size_t)(slash - out->path + 1) : 0; @@ -275,15 +287,16 @@ int output_finish(int err, FILE *input, FILE *output, const char *temp, // Flushed first, as a later write would move the mtime again; a failed flush // is left for codec_finish_file to report; best-effort, as in extraction fs_stat src; - if (err == 0 && out->keep_source_metadata && input && output && + int writes = output && out->path; + if (err == 0 && out->keep_source_metadata && input && writes && fflush(output) == 0 && fs_fstat(input, &src) == 0) { fs_futimens(output, src.mtime); fs_fchmod(output, src.mode); } - err = codec_finish_file(err, input, output, output ? temp : NULL, errors, + err = codec_finish_file(err, input, output, writes ? temp : NULL, errors, failure_message); - if (err != 0 || !output) { + if (err != 0 || !writes) { return err; } return output_commit(temp, out); diff --git a/src/compresso/csrc/common.h b/src/compresso/csrc/common.h index 0e2619b..b00c504 100644 --- a/src/compresso/csrc/common.h +++ b/src/compresso/csrc/common.h @@ -149,6 +149,7 @@ struct CompErrors { PyObject *corrupt_data; PyObject *extraction_policy; PyObject *incomplete_extraction; + PyObject *source_removal; PyObject *cancelled; PyObject *trailing_data_warning; }; @@ -190,4 +191,9 @@ int decompress_file(const char *src_path, const char *dst_path, AlgoID algo, int overwrite_existing, char *out_actual_path, size_t out_actual_path_size, CoreContext *ctx); +// Decode all of an archive or compressed file, writing nothing, so only what +// the format checks can fail it: a plain tar's data is proved to be there, but +// not to be right; returns 0, -1 with an exception set, or COMP_CANCELLED +int verify_file(const char *path, CoreContext *ctx); + #endif // COMMON_H diff --git a/src/compresso/csrc/fsutil.c b/src/compresso/csrc/fsutil.c index 2b05e51..2bf24ec 100644 --- a/src/compresso/csrc/fsutil.c +++ b/src/compresso/csrc/fsutil.c @@ -15,30 +15,21 @@ int64_t fs_ftell(FILE *stream) { #endif } -int64_t fs_stream_size(FILE *stream) { - int64_t original = fs_ftell(stream); - if (original < 0) - return -1; - +int fs_fseek(FILE *stream, int64_t offset, int whence) { #if defined(_WIN32) || defined(_WIN64) - if (_fseeki64(stream, 0, SEEK_END) != 0) - return -1; + return _fseeki64(stream, offset, whence); #else - if (fseeko(stream, 0, SEEK_END) != 0) - return -1; + return fseeko(stream, (off_t)offset, whence); #endif +} - int64_t size = fs_ftell(stream); - -#if defined(_WIN32) || defined(_WIN64) - if (_fseeki64(stream, original, SEEK_SET) != 0) - return -1; -#else - if (fseeko(stream, (off_t)original, SEEK_SET) != 0) +int64_t fs_stream_size(FILE *stream) { + int64_t original = fs_ftell(stream); + if (original < 0 || fs_fseek(stream, 0, SEEK_END) != 0) return -1; -#endif - return size; + int64_t size = fs_ftell(stream); + return fs_fseek(stream, original, SEEK_SET) == 0 ? size : -1; } int fs_is_absolute(const char *path) { @@ -79,7 +70,7 @@ int fs_join(char *out, size_t out_size, const char *dir, const char *name) { memcpy(out, dir, dir_len); if (sep_len) - out[dir_len] = '/'; + out[dir_len] = FS_SEP; memcpy(out + dir_len + sep_len, name, name_len + 1); return 0; } diff --git a/src/compresso/csrc/fsutil.h b/src/compresso/csrc/fsutil.h index 31c248a..cbd7a36 100644 --- a/src/compresso/csrc/fsutil.h +++ b/src/compresso/csrc/fsutil.h @@ -14,6 +14,8 @@ // so prefix comparisons must test both #if defined(_WIN32) || defined(_WIN64) #define FS_IS_SEP(c) ((c) == '/' || (c) == '\\') +#define FS_SEP '\\' +#define FS_NULL_DEVICE "NUL" #include @@ -22,6 +24,8 @@ int fs_widen(const char *utf8, wchar_t *out, int out_count); #else #define FS_IS_SEP(c) ((c) == '/') +#define FS_SEP '/' +#define FS_NULL_DEVICE "/dev/null" #endif typedef enum { @@ -42,8 +46,8 @@ typedef struct { uint32_t special; uint64_t rdev; - // Identity, for fs_same_file; on Windows only filled for regular files (the - // volume serial and file index), and 0 wherever it is unknown + // Identity, for fs_same_file; on Windows only filled for regular files and + // directories (volume serial and file index), and 0 wherever it is unknown uint64_t dev, ino; uint32_t nlink; // Its names' count; filled wherever the identity is } fs_stat; @@ -73,6 +77,9 @@ int fs_join(char *out, size_t out_size, const char *dir, const char *name); // Windows reparse point reports FS_TYPE_SYMLINK rather than its target's type int fs_stat_path(const char *path, fs_stat *out); +// Stat what `path` leads to, following symlinks; -1 if the link is dangling +int fs_stat_target(const char *path, fs_stat *out); + // Stat an open file, so a path reached through a symlink reports its target; // the identity is left 0 on Windows int fs_fstat(FILE *f, fs_stat *out); @@ -152,8 +159,8 @@ int fs_commit_temp(const char *temp, const char *dst, int overwrite_existing, // A file output, written to a temp beside `path` and committed on success typedef struct { - const char *path; - int overwrite; // The 0-3 scheme above + const char *path; // NULL discards the output, leaving only its failures + int overwrite; // The 0-3 scheme above char *actual; // Optional; receives the path written, or skipped size_t actual_size; int owner_only; // 0600, for an intermediate that holds a whole archive @@ -182,8 +189,8 @@ int fs_rename(const char *from, const char *to, int replace); int fs_link(const char *existing, const char *new_path); // Symlink `link_path` to `target`, stored unresolved; -1 with errno set -// (EEXIST if `link_path` exists, EPERM on Windows without Developer Mode/admin -// rights); on Windows, a target that is a directory gets a directory link +// (EEXIST if `link_path` exists, EPERM if this user or filesystem can't create +// symlinks; on Windows, a target that is a directory gets a directory link int fs_symlink(const char *target, const char *link_path); // Whether this process can create a node of `type` (fs_stat.special's values): @@ -198,6 +205,9 @@ int fs_mknod(const char *path, uint32_t type, uint32_t mode, uint64_t rdev); // platform (plain ftell() is 32-bit on Windows); returns -1 on failure int64_t fs_ftell(FILE *stream); +// fseek() with a 64-bit offset on every platform; returns 0 on success +int fs_fseek(FILE *stream, int64_t offset, int whence); + // Size of an open stream in bytes, leaving the offset where it found it; // returns -1 on failure int64_t fs_stream_size(FILE *stream); diff --git a/src/compresso/csrc/fsutil_posix.c b/src/compresso/csrc/fsutil_posix.c index bfaacea..18db2ad 100644 --- a/src/compresso/csrc/fsutil_posix.c +++ b/src/compresso/csrc/fsutil_posix.c @@ -47,6 +47,14 @@ int fs_stat_path(const char *path, fs_stat *out) { return 0; } +int fs_stat_target(const char *path, fs_stat *out) { + struct stat st; + if (stat(path, &st) != 0) + return -1; + fs_fill_stat(&st, out); + return 0; +} + int fs_fstat(FILE *f, fs_stat *out) { struct stat st; if (fstat(fileno(f), &st) != 0) diff --git a/src/compresso/csrc/fsutil_win32.c b/src/compresso/csrc/fsutil_win32.c index a1e084c..fff3e43 100644 --- a/src/compresso/csrc/fsutil_win32.c +++ b/src/compresso/csrc/fsutil_win32.c @@ -51,6 +51,8 @@ int fs_widen(const char *utf8, wchar_t *out, int out_count) { return 0; } +static void set_errno_from_win32(int fallback); + static int fs_narrow(const wchar_t *wide, char *out, int out_size) { if (WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wide, -1, out, out_size, NULL, NULL) == 0) { @@ -72,26 +74,10 @@ static HANDLE fs_open_for_metadata(const wchar_t *wpath) { NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); } -int fs_stat_path(const char *path, fs_stat *out) { - wchar_t wpath[FS_PATH_MAX]; - if (fs_widen(path, wpath, FS_PATH_MAX) != 0) - return -1; - - // _wstat64 follows reparse points and has no S_ISLNK equivalent, so without - // this archiver walks into any junction - DWORD attrs = GetFileAttributesW(wpath); - int is_reparse = attrs != INVALID_FILE_ATTRIBUTES && - (attrs & FILE_ATTRIBUTE_REPARSE_POINT) != 0; - +static int fs_stat_wide(const wchar_t *wpath, fs_stat *out) { struct __stat64 st; - if (_wstat64(wpath, &st) != 0) { - // A link is still a real entry when its target is missing - if (!is_reparse) - return -1; - memset(out, 0, sizeof(*out)); - out->type = FS_TYPE_SYMLINK; - return 0; - } + if (_wstat64(wpath, &st) != 0) + return -1; out->size = (uint64_t)st.st_size; out->mtime = (int64_t)st.st_mtime; @@ -102,28 +88,75 @@ int fs_stat_path(const char *path, fs_stat *out) { out->ino = 0; out->nlink = 0; - if (is_reparse) - out->type = FS_TYPE_SYMLINK; - else if (st.st_mode & _S_IFDIR) + if (st.st_mode & _S_IFDIR) out->type = FS_TYPE_DIR; else if (st.st_mode & _S_IFREG) out->type = FS_TYPE_FILE; else out->type = FS_TYPE_OTHER; - // _wstat64 leaves st_ino 0, so the identity needs a handle; only regular - // files get one, as nothing compares directories - if (out->type == FS_TYPE_FILE) { + // _wstat64 describes a symlink itself, and leaves st_ino 0; the handle + // follows links and supplies the rest; failing to open means dangling link + if (out->type == FS_TYPE_FILE || out->type == FS_TYPE_DIR) { HANDLE h = fs_open_for_metadata(wpath); + if (h == INVALID_HANDLE_VALUE) { + set_errno_from_win32(ENOENT); + return -1; + } BY_HANDLE_FILE_INFORMATION info; - if (h != INVALID_HANDLE_VALUE) { - if (GetFileInformationByHandle(h, &info)) { - out->dev = info.dwVolumeSerialNumber; - out->ino = ((uint64_t)info.nFileIndexHigh << 32) | info.nFileIndexLow; - out->nlink = info.nNumberOfLinks; - } - CloseHandle(h); + BOOL ok = GetFileInformationByHandle(h, &info); + CloseHandle(h); + if (!ok) { + set_errno_from_win32(EIO); + return -1; } + + int is_dir = (info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; + out->type = is_dir ? FS_TYPE_DIR : FS_TYPE_FILE; + out->size = + is_dir ? 0 : ((uint64_t)info.nFileSizeHigh << 32) | info.nFileSizeLow; + // FILETIME counts 100ns intervals since 1601-01-01 + ULONGLONG ticks = ((ULONGLONG)info.ftLastWriteTime.dwHighDateTime << 32) | + info.ftLastWriteTime.dwLowDateTime; + out->mtime = (int64_t)(ticks / 10000000ULL) - 11644473600LL; + out->dev = info.dwVolumeSerialNumber; + out->ino = ((uint64_t)info.nFileIndexHigh << 32) | info.nFileIndexLow; + out->nlink = info.nNumberOfLinks; + } + + return 0; +} + +int fs_stat_target(const char *path, fs_stat *out) { + wchar_t wpath[FS_PATH_MAX]; + if (fs_widen(path, wpath, FS_PATH_MAX) != 0) + return -1; + return fs_stat_wide(wpath, out); +} + +int fs_stat_path(const char *path, fs_stat *out) { + wchar_t wpath[FS_PATH_MAX]; + if (fs_widen(path, wpath, FS_PATH_MAX) != 0) + return -1; + + // _wstat64 follows reparse points and has no S_ISLNK equivalent, so without + // this archiver walks into any junction + DWORD attrs = GetFileAttributesW(wpath); + int is_reparse = attrs != INVALID_FILE_ATTRIBUTES && + (attrs & FILE_ATTRIBUTE_REPARSE_POINT) != 0; + + if (fs_stat_wide(wpath, out) != 0) { + // A link is still a real entry when its target is missing + if (!is_reparse) + return -1; + memset(out, 0, sizeof(*out)); + } + + if (is_reparse) { + out->type = FS_TYPE_SYMLINK; + out->dev = 0; + out->ino = 0; + out->nlink = 0; } return 0; @@ -436,6 +469,11 @@ static void set_errno_from_win32(int fallback) { case ERROR_ACCESS_DENIED: errno = EACCES; break; + case ERROR_PRIVILEGE_NOT_HELD: + case ERROR_NOT_SUPPORTED: + case ERROR_INVALID_FUNCTION: // e.g. a FAT volume + errno = EPERM; + break; default: errno = fallback; break; @@ -554,7 +592,8 @@ int fs_symlink(const char *target, const char *link_path) { if (ok) return 0; - set_errno_from_win32(EPERM); // e.g. ERROR_PRIVILEGE_NOT_HELD + // Not EPERM for anything else, e.g. a full disk, which must fail extraction + set_errno_from_win32(EIO); return -1; } diff --git a/src/compresso/csrc/standalone.c b/src/compresso/csrc/standalone.c index b4b26fe..8b39738 100644 --- a/src/compresso/csrc/standalone.c +++ b/src/compresso/csrc/standalone.c @@ -2,8 +2,12 @@ #include #include "standalone.h" #include "common.h" +#include +#include +#include #include #include +#include const FormatDesc *find_standalone_format(Format format) { const FormatDesc *desc = format_by_id(format); @@ -81,3 +85,251 @@ char *gzip_original_name(const char *compressed_path) { return c == 0 && i > 0 ? strdup(name_buf) : NULL; } + +static int xz_content_size(FILE *f, uint64_t *size) { + int64_t file_size = fs_stream_size(f); + lzma_stream strm = LZMA_STREAM_INIT; + lzma_index *index = NULL; + if (file_size < 0 || lzma_file_info_decoder(&strm, &index, UINT64_MAX, + (uint64_t)file_size) != LZMA_OK) { + return -1; + } + + uint8_t buf[BUFSIZ]; + lzma_ret ret = LZMA_OK; + while (ret == LZMA_OK) { + if (strm.avail_in == 0) { + strm.next_in = buf; + strm.avail_in = fread(buf, 1, sizeof(buf), f); + } + ret = lzma_code(&strm, LZMA_RUN); + if (ret == LZMA_SEEK_NEEDED) { + strm.avail_in = 0; + ret = fs_fseek(f, (int64_t)strm.seek_pos, SEEK_SET) == 0 ? LZMA_OK + : LZMA_DATA_ERROR; + } + } + + if (ret == LZMA_STREAM_END) { + *size = lzma_index_uncompressed_size(index); + } + lzma_index_end(index, NULL); + lzma_end(&strm); + return ret == LZMA_STREAM_END ? 0 : -1; +} + +// ZSTD_getFrameHeader needs static linking, so frames are walked by RFC 8878 +#define ZSTD_HEADER_MAX 18 // ZSTD_FRAMEHEADERSIZE_MAX, also static-only +#define ZSTD_BLOCK_HEADER 3 +#define ZSTD_SKIPPABLE_HEADER 8 +#define ZSTD_CHECKSUM 4 + +#define ZSTD_FHD_CHECKSUM 0x04 +#define ZSTD_FHD_SINGLE_SEGMENT 0x20 +#define ZSTD_BLOCK_RLE 1 + +// The offset past the blocks at `pos`, or -1 +static int64_t zstd_skip_blocks(FILE *f, int64_t pos, int64_t end) { + for (;;) { + unsigned char header[ZSTD_BLOCK_HEADER]; + if (fs_fseek(f, pos, SEEK_SET) != 0 || + fread(header, sizeof(header), 1, f) != 1) { + return -1; + } + + // Bit 0 ends the frame, 1-2 are the type, and the rest the size; an RLE + // block stores just the byte it repeats + uint32_t block = (uint32_t)get_le(header, ZSTD_BLOCK_HEADER); + pos += ZSTD_BLOCK_HEADER; + pos += ((block >> 1) & 3) == ZSTD_BLOCK_RLE ? 1 : block >> 3; + if (pos > end) { + return -1; + } + if (block & 1) { + return pos; + } + } +} + +// Adds the content size of the frame whose first `got` bytes are `header` to +// `size`, returning the offset past it, or -1 +typedef int64_t (*FrameWalker)(FILE *f, const unsigned char *header, + size_t got, int64_t pos, int64_t end, + uint64_t *size); + +static int64_t zstd_frame(FILE *f, const unsigned char *header, size_t got, + int64_t pos, int64_t end, uint64_t *size) { + static const uint8_t dict_id_size[] = {0, 1, 2, 4}; + static const uint8_t content_size_size[] = {0, 2, 4, 8}; + + // Both failures sit at the top of the range, and reject a bad magic + unsigned long long frame = ZSTD_getFrameContentSize(header, got); + if (frame >= ZSTD_CONTENTSIZE_ERROR || frame > UINT64_MAX - *size) { + return -1; + } + *size += frame; + + // Magic and descriptor, then the window byte unless the frame's single + // segment, whose content size is never absent + uint8_t fhd = header[4]; + int single = (fhd & ZSTD_FHD_SINGLE_SEGMENT) != 0; + int content_field = content_size_size[fhd >> 6]; + int header_size = 5 + !single + dict_id_size[fhd & 3] + + (single && !content_field ? 1 : content_field); + + pos = zstd_skip_blocks(f, pos + header_size, end); + return pos >= 0 && (fhd & ZSTD_FHD_CHECKSUM) ? pos + ZSTD_CHECKSUM : pos; +} + +#define LZ4_FLG_VERSION_MASK 0xC0 +#define LZ4_FLG_VERSION 0x40 +#define LZ4_FLG_BLOCK_CHECKSUM 0x10 +#define LZ4_FLG_CONTENT_SIZE 0x08 +#define LZ4_FLG_CONTENT_CHECKSUM 0x04 +#define LZ4_FLG_DICT_ID 0x01 +#define LZ4_BLOCK_SIZE_MASK 0x7FFFFFFF // The top bit marks a stored block + +static int64_t lz4_frame(FILE *f, const unsigned char *header, size_t got, + int64_t pos, int64_t end, uint64_t *size) { + // The minimum is magic, FLG, BD and the header checksum; the content size + // and dictionary ID sit between BD and the checksum + uint8_t flg = header[4]; + size_t header_size = + LZ4F_HEADER_SIZE_MIN + 8 + (flg & LZ4_FLG_DICT_ID ? 4 : 0); + + // A frame written from a stream, the lz4 CLI's default, states no size + if (get_le(header, 4) != LZ4F_MAGICNUMBER || + (flg & LZ4_FLG_VERSION_MASK) != LZ4_FLG_VERSION || + !(flg & LZ4_FLG_CONTENT_SIZE) || got < header_size) { + return -1; + } + uint64_t frame = get_le(header + 6, 8); + if (frame > UINT64_MAX - *size) { + return -1; + } + *size += frame; + + int64_t checksum = flg & LZ4_FLG_BLOCK_CHECKSUM ? LZ4F_BLOCK_CHECKSUM_SIZE : 0; + pos += (int64_t)header_size; + for (;;) { + unsigned char block[LZ4F_BLOCK_HEADER_SIZE]; + if (fs_fseek(f, pos, SEEK_SET) != 0 || + fread(block, sizeof(block), 1, f) != 1) { + return -1; + } + pos += LZ4F_BLOCK_HEADER_SIZE; + + // liblz4 ends the frame on a zero size, whatever the top bit + int64_t len = (int64_t)(get_le(block, 4) & LZ4_BLOCK_SIZE_MASK); + if (len == 0) { + break; + } + pos += len + checksum; + if (pos > end) { + return -1; + } + } + + return flg & LZ4_FLG_CONTENT_CHECKSUM ? pos + LZ4F_CONTENT_CHECKSUM_SIZE + : pos; +} + +// zstd and lz4 share skippable frames, so only the other frames differ +static int frames_content_size(FILE *f, FrameWalker frame, uint64_t *size) { + int64_t end = fs_stream_size(f); + int64_t pos = 0; + *size = 0; + + while (pos >= 0 && pos < end) { + unsigned char header[LZ4F_HEADER_SIZE_MAX]; // Over ZSTD_HEADER_MAX too + if (fs_fseek(f, pos, SEEK_SET) != 0) { + return -1; + } + size_t got = fread(header, 1, sizeof(header), f); + if (got < ZSTD_SKIPPABLE_HEADER) { + return -1; + } + + if ((get_le(header, 4) & ZSTD_MAGIC_SKIPPABLE_MASK) == + ZSTD_MAGIC_SKIPPABLE_START) { + pos += ZSTD_SKIPPABLE_HEADER + (int64_t)get_le(header + 4, 4); + } else { + pos = frame(f, header, got, pos, end, size); + } + } + + return pos == end && end > 0 ? 0 : -1; +} + +// Snappy's framing format, as codec/snappy.c reads it +#define SNAPPY_COMPRESSED 0x00 +#define SNAPPY_UNCOMPRESSED 0x01 +#define SNAPPY_SKIPPABLE 0x80 // 0x80-0xFE, then the stream identifier at 0xFF +#define SNAPPY_CHUNK 65536 // The most a chunk may decode to +#define SNAPPY_CHECKSUM 4 +#define SNAPPY_VARINT_MAX 3 // Enough to state SNAPPY_CHUNK + +// Every snappy block opens with its decoded length, so no chunk is decoded +static int snappy_content_size(FILE *f, uint64_t *size) { + int64_t end = fs_stream_size(f); + int64_t pos = 0; + *size = 0; + + while (pos < end) { + // Type and LE24 length, then a data chunk's checksum and the start of its + // body + unsigned char chunk[4 + SNAPPY_CHECKSUM + SNAPPY_VARINT_MAX]; + if (fs_fseek(f, pos, SEEK_SET) != 0) { + return -1; + } + size_t got = fread(chunk, 1, sizeof(chunk), f); + if (got < 4) { + return -1; + } + size_t len = (size_t)get_le(chunk + 1, 3); + size_t body = got - 4 < len ? got - 4 : len; + pos += 4 + (int64_t)len; + + size_t decoded = 0; + if (chunk[0] == SNAPPY_COMPRESSED) { + if (body < SNAPPY_CHECKSUM || + snappy_uncompressed_length((const char *)chunk + 4 + SNAPPY_CHECKSUM, + body - SNAPPY_CHECKSUM, + &decoded) != SNAPPY_OK) { + return -1; + } + } else if (chunk[0] == SNAPPY_UNCOMPRESSED) { + if (len < SNAPPY_CHECKSUM) { + return -1; + } + decoded = len - SNAPPY_CHECKSUM; + } else if (chunk[0] < SNAPPY_SKIPPABLE) { + return -1; // Reserved unskippable + } + + if (decoded > SNAPPY_CHUNK) { + return -1; + } + *size += decoded; + } + + return pos == end && end > 0 ? 0 : -1; +} + +int standalone_content_size(FILE *f, Format format, uint64_t *size) { + if (fs_fseek(f, 0, SEEK_SET) != 0) { + return -1; + } + switch (format) { + case FORMAT_XZ: + return xz_content_size(f, size); + case FORMAT_ZSTD: + return frames_content_size(f, zstd_frame, size); + case FORMAT_LZ4: + return frames_content_size(f, lz4_frame, size); + case FORMAT_SNAPPY: + return snappy_content_size(f, size); + default: + return -1; + } +} diff --git a/src/compresso/csrc/standalone.h b/src/compresso/csrc/standalone.h index 0bea8e6..b957b7a 100644 --- a/src/compresso/csrc/standalone.h +++ b/src/compresso/csrc/standalone.h @@ -23,4 +23,10 @@ int standalone_decompress(const FormatDesc *fmt, const char *input_path, // The original filename a gzip header stores, or NULL char *gzip_original_name(const char *compressed_path); +// The uncompressed size an xz index, the zstd or lz4 frame headers, or the +// snappy block headers of `f` state, without decoding; -1 if they don't state +// one, a zstd or lz4 frame written from a stream, or a file those structures +// don't fully describe +int standalone_content_size(FILE *f, Format format, uint64_t *size); + #endif // STANDALONE_H diff --git a/src/compresso/frontend/__init__.py b/src/compresso/frontend/__init__.py index dd08af8..ecaf28e 100644 --- a/src/compresso/frontend/__init__.py +++ b/src/compresso/frontend/__init__.py @@ -8,8 +8,12 @@ CompressionPlan, DecompressionJob, DecompressionPlan, + Strategy, + VerificationJob, + VerificationPlan, plan_compression, plan_decompression, + plan_verification, ) from .archive_api import ( ArchiveEntry, @@ -44,9 +48,13 @@ "JobResult", "OverwriteMode", "ProgressCallback", + "Strategy", "SymlinkMode", + "VerificationJob", + "VerificationPlan", "plan_archive", "plan_compression", "plan_decompression", "plan_extraction", + "plan_verification", ] diff --git a/src/compresso/frontend/_formats.py b/src/compresso/frontend/_formats.py new file mode 100644 index 0000000..6b6311c --- /dev/null +++ b/src/compresso/frontend/_formats.py @@ -0,0 +1,99 @@ +"""Format names and the rules for reading one from an output path.""" + +from __future__ import annotations + +from pathlib import Path + +from .._core import archive_capabilities, format_is_archive, standalone_formats + +# What an archive is written as when neither the caller nor its name says +DEFAULT_ARCHIVE_FORMAT = "zip" + +_STANDALONE = standalone_formats() + +# Single-file container formats, as opposed to the Compresso container or an +# archive, including aliases +STANDALONE_FORMATS: dict[str, str] = { + alias: f["name"] for f in _STANDALONE for alias in (f["name"], *f["aliases"]) +} + +# Suffix each standalone format conventionally uses, for naming an output the +# caller did not name themselves +STANDALONE_SUFFIXES: dict[str, str] = { + f["name"]: "." + f["exts"][0] for f in _STANDALONE +} + + +def archive_formats() -> dict[str, list[str]]: + """Return each available archive format with its compressed variants. + + Returns: + Each archive format (e.g. "tar") mapped to the variants the core + writes, named by the extension an output is given (e.g. "tar.gz"). + """ + # Asked of the core, which names a variant by any of the codec's names + return { + archive["name"]: [ + f"{archive['name']}.{fmt['exts'][0]}" + for fmt in _STANDALONE + if format_is_archive(f"{archive['name']}.{fmt['exts'][0]}") + ] + for archive in archive_capabilities() + } + + +def canonical_standalone_format(name: str | None) -> str | None: + """Return the canonical name for a single-file format, or None. + + Args: + name: A format as the user wrote it, e.g. "gz". + + Returns: + The canonical name (e.g. "gzip"), or None if it is not a single-file + format; includes None, archive formats and unknown names. + """ + if not name: + return None + + return STANDALONE_FORMATS.get(name.lower()) + + +def infer_format(output: Path | None) -> str | None: + """Infer a format from an output filename. + + Args: + output: The path the user asked to write, or None. + + Returns: + The format named by the suffix, "comp" for the Compresso container, + or None if it names none. + """ + if output is None: + return None + + # Mirrors the core (`x.tar.lz4` is tar in lz4); unwritable formats + # still match, so the default is never written under their suffix + suffix: str = output.suffix.lower().lstrip(".") + if Path(output.stem).suffix.lower() == ".tar" and format_is_archive( + f"tar.{suffix}" + ): + return f"tar.{suffix}" + + if suffix == "comp" or suffix in STANDALONE_FORMATS or format_is_archive(suffix): + return suffix + + return None + + +def gzip_stored_mtime(path: Path) -> int | None: + """Return the modification time a gzip header stores. + + Args: + path: A gzip file. + + Returns: + Its MTIME, in seconds since the epoch, or None if none is stored. + """ + with path.open("rb") as f: + # MTIME, bytes 4-7; zero means none was stored + return int.from_bytes(f.read(8)[4:], "little") or None diff --git a/src/compresso/frontend/_job.py b/src/compresso/frontend/_job.py index 847c585..ea725a6 100644 --- a/src/compresso/frontend/_job.py +++ b/src/compresso/frontend/_job.py @@ -2,7 +2,6 @@ from __future__ import annotations -import asyncio import threading from collections.abc import Callable from dataclasses import dataclass @@ -229,6 +228,9 @@ async def run_async( Returns: JobResult: The outcome of the job. """ + # Deferred so sync callers skip asyncio's import cost + import asyncio + token: CancelToken = cancel if cancel is not None else CancelToken() loop = asyncio.get_running_loop() future = loop.run_in_executor(None, self.run, progress, token) diff --git a/src/compresso/frontend/api.py b/src/compresso/frontend/api.py index 94e5ba1..0240c7e 100644 --- a/src/compresso/frontend/api.py +++ b/src/compresso/frontend/api.py @@ -4,6 +4,7 @@ import os from dataclasses import dataclass, replace +from enum import StrEnum from pathlib import Path from typing import Self @@ -18,46 +19,37 @@ detect_format, format_is_archive, gzip_stored_name, - standalone_formats, + verify_file, ) from .._core import get_default_backend_for_strategy as default_backend from .._levels import to_core_level from ..introspect.file_inspect import InspectResult from ..introspect.file_inspect import inspect as inspect_file from ..introspect.speeds import get_estimated_speeds +from ._formats import ( + STANDALONE_FORMATS, + STANDALONE_SUFFIXES, + canonical_standalone_format, + gzip_stored_mtime, +) from ._job import JobResult, ProgressCallback, ThreadedJob, to_core_progress from .archive_api import _OVERWRITE_CODES, OverwriteMode MB = 1024 * 1024 -_STANDALONE = standalone_formats() - -# Single-file container formats, as opposed to the Compresso container or an -# archive, including aliases -STANDALONE_FORMATS: dict[str, str] = { - alias: f["name"] for f in _STANDALONE for alias in (f["name"], *f["aliases"]) -} - -# Suffix each standalone format conventionally uses, for naming an output the -# caller did not name themselves -STANDALONE_SUFFIXES: dict[str, str] = { - f["name"]: "." + f["exts"][0] for f in _STANDALONE -} - -def canonical_standalone_format(name: str | None) -> str | None: - """Return the canonical name for a single-file format, or None. +class Strategy(StrEnum): + """Which backend the Compresso container picks when no algorithm is given. - Args: - name: A format as the user wrote it, e.g. "gz". - - Returns: - The canonical name (e.g. "gzip"), or None if it is not a single-file - format — which includes None, archive formats and unknown names. + Attributes: + FAST: Favour speed. + BALANCED: Trade speed against ratio. + MAX_RATIO: Favour the smallest output. """ - if not name: - return None - return STANDALONE_FORMATS.get(name.lower()) + + FAST = "fast" + BALANCED = "balanced" + MAX_RATIO = "max_ratio" @dataclass(frozen=True) @@ -66,7 +58,7 @@ class CompressionOptions: Attributes: algo: Compression algorithm name, or None for auto. - strategy: Compression strategy - "fast", "balanced", or "max_ratio". + strategy: Compression strategy; ignored when `algo` is given. level: Compression level, or None for the backend's default; the range depends on the backend (see `BackendCapabilities`). format: Single-file container to write, e.g. "gz". None writes the @@ -79,7 +71,7 @@ class CompressionOptions: """ algo: str | None = None - strategy: str = "balanced" + strategy: Strategy = Strategy.BALANCED level: int | None = None format: str | None = None overwrite: OverwriteMode = OverwriteMode.RENAME @@ -145,6 +137,23 @@ class DecompressionPlan: mtime: int | None = None +@dataclass(frozen=True) +class VerificationPlan: + """Holds an integrity check of an archive or compressed file. + + Attributes: + src: Source file path. + can_run: Whether the check can proceed. + reason_if_unavailable: Reason it cannot, if applicable. + threads: Worker threads for xz/lzma, or 0 for one per core. + """ + + src: Path + can_run: bool = True + reason_if_unavailable: str | None = None + threads: int = 0 + + def plan_compression( src: str | Path, dest: str | Path | None = None, @@ -258,25 +267,26 @@ def plan_compression( reason_if_unavailable="No suitable backend found for the selected strategy", ) - if options.level is not None: - try: - # The core's own check, so each backend's range lives in one place - if standalone: - check_level(options.level, format=standalone) - else: - check_level(options.level, algo=backend_name) + try: + # The core's own check, so each backend's range lives in one place; + # with no level it still refuses an unknown algorithm + level = to_core_level(options.level) + if standalone: + check_level(level, format=standalone) + else: + check_level(level, algo=backend_name) - except ValueError as e: - return CompressionPlan( - src=src_path, - dest=dest_path, - options=options, - input_size=input_size, - backend_name=backend_name, - estimated_seconds=None, - can_compress=False, - reason_if_unavailable=str(e), - ) + except ValueError as e: + return CompressionPlan( + src=src_path, + dest=dest_path, + options=options, + input_size=input_size, + backend_name=backend_name, + estimated_seconds=None, + can_compress=False, + reason_if_unavailable=str(e), + ) mb_s: int | float = get_estimated_speeds(algo=backend_name, operation="compress") estimated_seconds: int | float = (input_size / MB) / mb_s if input_size > 0 else 0.0 @@ -368,6 +378,10 @@ def plan( try: detected: str = detect_format(src_path) + # The caller names the file with the error + except OSError as e: + return plan(None, False, e.strerror or str(e)) + except Exception as e: # noqa: BLE001 - surface as an unavailable plan return plan(None, False, f"Cannot read file: {e}") @@ -383,11 +397,8 @@ def plan( if name not in ("", ".."): dest_path = src_path.parent / name - mtime: int | None = None - if restore_name and detected == "gzip": - with src_path.open("rb") as f: - # MTIME, bytes 4-7; zero means none was stored - mtime = int.from_bytes(f.read(8)[4:], "little") or None + restores_mtime: bool = restore_name and detected == "gzip" + mtime: int | None = gzip_stored_mtime(src_path) if restores_mtime else None return replace(plan(detected, True), mtime=mtime) @@ -416,6 +427,32 @@ def plan( return plan(None, True) +def plan_verification(src: str | Path, threads: int = 0) -> VerificationPlan: + """Plan an integrity check of an archive or compressed file. + + Args: + src: Source file path. + threads: Worker threads for xz/lzma, or 0 for one per core. + + Returns: + VerificationPlan: The resulting plan. + """ + src_path = Path(src) + + try: + recognised: bool = detect_format(src_path) != "unknown" + + # The caller names the file with the error + except OSError as e: + return VerificationPlan(src_path, False, e.strerror or str(e), threads) + + if not recognised: + reason = "not a recognised archive or compressed file" + return VerificationPlan(src_path, False, reason, threads) + + return VerificationPlan(src_path, threads=threads) + + class CompressionJob(ThreadedJob[CompressionPlan]): """Compression job high-level wrapper.""" @@ -669,3 +706,70 @@ def run( error=e, plan=self.plan, ) + + +class VerificationJob(ThreadedJob[VerificationPlan]): + """Integrity check high-level wrapper.""" + + def __init__(self, plan: VerificationPlan) -> None: + """Initialise the verification job. + + Args: + plan: The verification plan to execute. + """ + self.plan: VerificationPlan = plan + + @classmethod + def from_file(cls, src: str | Path, threads: int = 0) -> Self: + """Create a VerificationJob from a file path. + + Args: + src: Source file path. + threads: Worker threads for xz/lzma, or 0 for one per core. + + Returns: + VerificationJob: The created job. + """ + return cls(plan=plan_verification(src, threads)) + + def run( + self, + progress: ProgressCallback | None = None, + cancel: CancelToken | None = None, + ) -> JobResult: + """Decode the whole file, writing nothing. + + Args: + progress: Optional progress callback, invoked with + `(fraction, done_bytes, total_bytes)`; counts the bytes a plain + archive's entries hold, or else the compressed bytes consumed. + cancel: Optional `CancelToken`. + + Returns: + JobResult: The result, whose `error` says what is damaged. + """ + if not self.plan.can_run: + return JobResult( + ok=False, + error=RuntimeError(self.plan.reason_if_unavailable or "Cannot test"), + plan=self.plan, + ) + + try: + verify_file( + self.plan.src, + threads=self.plan.threads, + progress=to_core_progress(progress, 0), + cancel=cancel, + ) + + return JobResult(ok=True, error=None, plan=self.plan) + + # Cancellation is a deliberate stop, not a failure; see + # `CompressionJob.run`. + except Cancelled: + return JobResult(ok=False, error=None, plan=self.plan, cancelled=True) + + # `run` reports failure through JobResult rather than raising + except Exception as e: # noqa: BLE001 + return JobResult(ok=False, error=e, plan=self.plan) diff --git a/src/compresso/frontend/archive_api.py b/src/compresso/frontend/archive_api.py index 77e47e4..6bade6c 100644 --- a/src/compresso/frontend/archive_api.py +++ b/src/compresso/frontend/archive_api.py @@ -2,7 +2,9 @@ from __future__ import annotations -from collections.abc import Sequence +import os +import stat +from collections.abc import Iterator, Sequence from dataclasses import dataclass, replace from enum import StrEnum from pathlib import Path @@ -11,13 +13,17 @@ from .._core import ( Cancelled, CancelToken, + SourceRemovalError, check_level, create_archive, extract_archive, format_is_archive, + format_stores_hardlinks, list_archive_contents, + path_matches, ) from .._levels import to_core_level +from ._formats import DEFAULT_ARCHIVE_FORMAT, infer_format from ._job import JobResult, ProgressCallback, ThreadedJob, to_core_progress @@ -59,22 +65,22 @@ class OverwriteMode(StrEnum): """What archiving or extraction does when a destination already exists. Attributes: - ERROR: Fail the operation. - SKIP: Leave the existing file untouched. - OVERWRITE: Replace the existing file. RENAME: Write to a platform-native numbered sibling instead (e.g. "name 2.ext" on macOS, "name (2).ext" elsewhere), leaving the existing file untouched. + OVERWRITE: Replace the existing file. + SKIP: Leave the existing file untouched. + ERROR: Fail the operation. """ - ERROR = "error" - SKIP = "skip" - OVERWRITE = "overwrite" RENAME = "rename" + OVERWRITE = "overwrite" + SKIP = "skip" + ERROR = "error" -# The C policy uses ints; see `ExtractionPolicy` in csrc/archives.h. Archive -# creation applies the same codes to its single destination file. +# The C policy uses ints; see `ExtractionPolicy` in csrc/archives.h; archive +# creation applies the same codes to its single destination file _OVERWRITE_CODES: dict[OverwriteMode, int] = { OverwriteMode.ERROR: 0, OverwriteMode.SKIP: 1, @@ -88,13 +94,13 @@ class SymlinkMode(StrEnum): Attributes: DENY: Refuse the archive. - ALLOW: Create the link, if its target stays inside the output directory. FILE: Write a regular file holding the link's target. + ALLOW: Create the link, if its target stays inside the output directory. """ DENY = "deny" - ALLOW = "allow" FILE = "file" + ALLOW = "allow" _SYMLINK_CODES: dict[SymlinkMode, int] = { @@ -109,18 +115,33 @@ class ArchiveOptions: """Represents the options for an archive operation. Attributes: - format: The archive/compression format to write. + format: The archive/compression format to write, or None to take it + from the output's suffix, falling back to zip. compression_level: The compression level to use, or None for the format's default. overwrite: What to do when the destination archive already exists. threads: Worker threads for tar.zst and tar.xz, or 0 for one per core; the output is the same for any count. + include: Globs on entry names; with any, only what matches is stored, + plus the directories holding it. + exclude: Globs on entry names that aren't stored or descended into, + overriding `include`. + remove_sources: Whether to delete what was stored once the archive is + written; anything else stays, including parent directories; running + with `OverwriteMode.SKIP` fails, as a skipped archive holds nothing. + dereference: Whether to store what symlinks lead to; one that's + dangling, or that leads to a directory above it, stays a link; + `remove_sources` then deletes the links, not what they lead to. """ - format: str = "tar.zst" # Default to tar with zstd + format: str | None = None compression_level: int | None = None overwrite: OverwriteMode = OverwriteMode.RENAME threads: int = 0 + include: Sequence[str] = () + exclude: Sequence[str] = () + remove_sources: bool = False + dereference: bool = False @dataclass(frozen=True) @@ -166,6 +187,9 @@ class ExtractOptions: symlinks: What to do with a symlink entry; by default a zip's are written as files and any other archive's are refused. threads: Worker threads for tar.xz, or 0 for one per core. + include: Globs selecting entries as well as any named by `files`; one + that matches nothing fails the run as a missing name does. + exclude: Globs dropping entries, overriding `files` and `include`. """ overwrite: OverwriteMode = OverwriteMode.RENAME @@ -177,6 +201,8 @@ class ExtractOptions: overwrite_dir_metadata: bool = False symlinks: SymlinkMode | None = None threads: int = 0 + include: Sequence[str] = () + exclude: Sequence[str] = () @dataclass(frozen=True) @@ -188,7 +214,8 @@ class ExtractPlan: output_dir: Directory to extract into. files: Optional subset of entry paths to extract (None = all). options: Extraction policy to apply. - entries: Entries discovered in the archive. + entries: The archive's entries that `files` and the options' globs + select. can_run: Whether the extract operation can proceed. reason_if_unavailable: Reason the operation cannot proceed, if any. """ @@ -224,6 +251,9 @@ def plan_archive( source_paths: list[Path] = [Path(s) for s in sources] output_path = Path(output) + fmt: str = options.format or infer_format(output_path) or DEFAULT_ARCHIVE_FORMAT + options = replace(options, format=fmt) + try: # Plain strings still work at runtime options = replace(options, overwrite=OverwriteMode(options.overwrite)) @@ -253,7 +283,7 @@ def plan_archive( reason_if_unavailable="No source paths provided", ) - missing: list[Path] = [p for p in source_paths if not p.exists()] + missing: list[Path] = [p for p in source_paths if not os.path.lexists(p)] if missing: return ArchivePlan( sources=source_paths, @@ -265,7 +295,7 @@ def plan_archive( reason_if_unavailable=f"Source path does not exist: {missing[0]}", ) - if not format_is_archive(options.format): + if not format_is_archive(fmt): return ArchivePlan( sources=source_paths, output=output_path, @@ -273,13 +303,13 @@ def plan_archive( total_input_size=0, entry_count=len(source_paths), can_run=False, - reason_if_unavailable=f"Format does not support archives: {options.format}", + reason_if_unavailable=f"Format does not support archives: {fmt}", ) if options.compression_level is not None: try: # The level belongs to the codec stage, or to the container itself - check_level(options.compression_level, format=options.format) + check_level(options.compression_level, format=fmt) except ValueError as e: return ArchivePlan( @@ -292,9 +322,7 @@ def plan_archive( reason_if_unavailable=str(e), ) - total_input_size: int = sum( - f.stat().st_size for f in _iter_files(source_paths) if f.is_file() - ) + total_input_size: int = _stored_size(source_paths, options) return ArchivePlan( sources=source_paths, @@ -307,21 +335,118 @@ def plan_archive( ) -def _iter_files(paths: list[Path]): - """Yield every file under the given paths, recursing into directories. +def _stored_size(sources: list[Path], options: ArchiveOptions) -> int: + """Return the bytes the C walk would store from the sources. Args: - paths: List of paths to iterate over. + sources: The source paths; hardlinks can span them. + options: Archive options, with the format set. + + Returns: + The total size of the regular files kept, a hardlinked file counting + once if the format stores its later names as links. + """ + links_once = format_stores_hardlinks(options.format or DEFAULT_ARCHIVE_FORMAT) + seen: set[tuple[int, int]] = set() + total = 0 + + for info in (i for s in sources for i in _stored_files(s, options)): + if links_once and info.st_nlink > 1 and info.st_ino: + identity = (info.st_dev, info.st_ino) + if identity in seen: + continue + seen.add(identity) + + total += info.st_size + + return total + + +def _stored_files(source: Path, options: ArchiveOptions) -> Iterator[os.stat_result]: + """Yield the stat of each regular file the C walk keeps from a source. + + Only `exclude` prunes: a directory no include matches can still hold files + that one does. + + Args: + source: The source path; a symlink stays a link, not its target, + unless `options.dereference` is set. + options: Archive options. Yields: - Each file under the given paths. + Each kept regular file's stat. """ - for p in paths: - if p.is_dir(): - yield from (f for f in p.rglob("*")) + follow = options.dereference + name = source.relative_to(source.parent).as_posix() + + if not source.is_dir() or (source.is_symlink() and not follow): + candidates = [(source, name)] - else: - yield p + elif path_matches(name, options.exclude): + return + + else: + candidates = _walk_files(source, options) + + for path, entry in candidates: + if path_matches(entry, options.exclude): + continue + + if options.include and not path_matches(entry, options.include): + continue + + # A dangling link stays a link, which holds no bytes + if follow and not os.path.exists(path): + continue + + info = os.stat(path, follow_symlinks=follow) + if stat.S_ISREG(info.st_mode): + yield info + + +def _leads_above(path: Path, source: Path) -> bool: + """Return whether a symlink leads to a directory between it and a source. + + Args: + path: The path found below `source`. + source: The directory being walked. + + Returns: + True if `path` is a link the C walk wouldn't follow. + """ + if not path.is_symlink(): + return False + + above = path.parent + while not os.path.samefile(path, above): + if above == source: + return False + above = above.parent + + return True + + +def _walk_files(source: Path, options: ArchiveOptions) -> Iterator[tuple[str, str]]: + """Yield each non-directory under a source, pruning excluded directories. + + Args: + source: The directory to walk. + options: Archive options. + + Yields: + Each path on disk with its entry name. + """ + follow = options.dereference + for top, dirs, files in os.walk(source, followlinks=follow): + rel = Path(top).relative_to(source.parent).as_posix() + dirs[:] = [ + d + for d in dirs + if not path_matches(f"{rel}/{d}", options.exclude) + and not (follow and _leads_above(Path(top, d), source)) + ] + for f in files: + yield os.path.join(top, f), f"{rel}/{f}" def plan_extraction( @@ -411,6 +536,22 @@ def plan_extraction( reason_if_unavailable=f"Cannot read archive: {e}", ) + # As the extractor matches: a name selects that entry and anything below it, + # an include adds to those, and an exclude overrides both + names: set[str] = {name.rstrip("/") for name in files or ()} - {""} + + def selected(path: str) -> bool: + if path_matches(path, opts.exclude): + return False + + return ( + not (files or opts.include) + or any(path == n or path.startswith(f"{n}/") for n in names) + or path_matches(path, opts.include) + ) + + entries = [e for e in entries if selected(e.path)] + return ExtractPlan( archive=archive_path, output_dir=out_dir, @@ -481,6 +622,9 @@ def run( plan=self.plan, ) + # plan_archive resolves an unset format before a plan can run + assert self.plan.options.format is not None + total: int = self.plan.total_input_size try: actual_path = create_archive( @@ -490,6 +634,10 @@ def run( to_core_level(self.plan.options.compression_level), overwrite=_OVERWRITE_CODES[self.plan.options.overwrite], threads=self.plan.options.threads, + include=self.plan.options.include, + exclude=self.plan.options.exclude, + remove_sources=self.plan.options.remove_sources, + dereference=self.plan.options.dereference, progress=to_core_progress(progress, total), cancel=cancel, ) @@ -500,6 +648,10 @@ def run( self.plan = replace(self.plan, output=Path(actual_path)) return JobResult(ok=True, error=None, plan=self.plan) + except SourceRemovalError as e: + self.plan = replace(self.plan, output=Path(e.output)) + return JobResult(ok=False, error=e, plan=self.plan) + # Cancellation is a deliberate stop, not a failure; see # `CompressionJob.run`. except Cancelled: @@ -543,14 +695,6 @@ def from_archive( """ return cls(plan=plan_extraction(archive, output_dir, files, options)) - def list_contents(self) -> list[ArchiveEntry]: - """List archive entry paths without extracting. - - Returns: - List of ArchiveEntry instances. - """ - return list(self.plan.entries) - def run( self, progress: ProgressCallback | None = None, @@ -595,6 +739,8 @@ def run( -1 if options.symlinks is None else _SYMLINK_CODES[options.symlinks] ), threads=options.threads, + include=options.include, + exclude=options.exclude, progress=to_core_progress(progress, total), cancel=cancel, ) diff --git a/src/compresso/frontend/dispatch.py b/src/compresso/frontend/dispatch.py new file mode 100644 index 0000000..f7d8594 --- /dev/null +++ b/src/compresso/frontend/dispatch.py @@ -0,0 +1,305 @@ +"""Dispatch rules for the smart `compress` and `extract` commands.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass, field, replace +from functools import partial +from pathlib import Path + +from .._core import detect_format, format_is_archive +from ._formats import ( + DEFAULT_ARCHIVE_FORMAT, + STANDALONE_SUFFIXES, + archive_formats, + canonical_standalone_format, + infer_format, +) +from .api import CompressionJob, CompressionOptions, DecompressionJob, Strategy +from .archive_api import ( + ArchiveJob, + ArchiveOptions, + ExtractJob, + ExtractOptions, + OverwriteMode, +) + + +def looks_like_archive(path: Path) -> bool: + """Return whether `path`'s own bytes say it holds many entries. + + Args: + path: The path to check. + + Returns: + True if the path looks like an archive, False otherwise. + """ + try: + return bool(format_is_archive(detect_format(path))) + + except Exception: # noqa: BLE001 - unreadable means not an archive + return False + + +def is_archive_format(fmt: str | None) -> bool: + """Return whether `fmt` names a container that holds many entries. + + Args: + fmt: The format string to check. + + Returns: + True if `fmt` names an archive format, False otherwise. + """ + # Includes currently unwritable formats (e.g. 7z) + return fmt is not None and format_is_archive(fmt.lower()) + + +def default_archive_output(first_input: Path, fmt: str) -> Path: + """Name an archive the caller did not name. + + Args: + first_input: The first source path, which the archive is named after. + fmt: The archive format, used as the suffix. + + Returns: + A path beside the input, e.g. `src/` in tar.zst becomes `src.tar.zst`. + """ + # A trailing separator would otherwise make the name empty + stem: str = first_input.resolve().name + + return first_input.resolve().parent / f"{stem}.{fmt}" + + +def format_is_single_file(fmt: str | None) -> bool: + """Return whether `fmt` names a container holding exactly one file. + + Args: + fmt: The format string to check. + + Returns: + True if `fmt` names a single-file format, False otherwise. + """ + return canonical_standalone_format(fmt) is not None + + +@dataclass(frozen=True) +class CompressRequest: + """What `compress` was asked to do, already parsed. + + Attributes: + inputs: The files and directories to compress. + output: The `-o` value, or None to name the output after the first input. + format: The container asked for, or None to infer it. + algo: The `.comp` algorithm, or None for the strategy's default. + strategy: The `.comp` strategy, or None for the default. + level: The compression level, or None for the format's own. + overwrite: What to do when the output already exists. + threads: Worker threads, or 0 for one per core. + include: Globs an archive's entries must match; see `ArchiveOptions`. + exclude: Globs an archive's entries must not match. + remove_sources: Whether an archive deletes what it stored once written; + see `ArchiveOptions`. + dereference: Whether an archive stores what symlinks lead to; see + `ArchiveOptions`. + """ + + inputs: list[Path] + output: Path | None = None + format: str | None = None + algo: str | None = None + strategy: Strategy | None = None + level: int | None = None + overwrite: OverwriteMode = OverwriteMode.RENAME + threads: int = 0 + include: Sequence[str] = () + exclude: Sequence[str] = () + remove_sources: bool = False + dereference: bool = False + + +@dataclass(frozen=True) +class ExtractRequest: + """What `extract` was asked to do, already parsed. + + Attributes: + inputs: The archives and single-file containers to unpack. + directory: The `-C` value, or None to write beside each input. + output: The `-o` file, for a single non-archive input. + files: The `-e` entry paths to extract from each archive, or + None for all. + options: The extraction policy; single-file inputs use only its + `overwrite` and `threads`. + restore_name: Whether to name and date a gzip file's output as its + header does. + """ + + inputs: list[Path] + directory: Path | None = None + output: Path | None = None + files: list[str] | None = None + options: ExtractOptions = field(default_factory=ExtractOptions) + restore_name: bool = False + + +def plan_compress(request: CompressRequest) -> CompressionJob | ArchiveJob: + """Turn a compress request into a planned job. + + One file becomes a single-file container; a directory, several inputs, or + an archive format becomes an archive. + + Args: + request: The parsed request. + + Returns: + The planned job, which may still be unable to run (see its plan), + as when `algo` or `strategy` is given for anything but a `.comp`. + + Raises: + ValueError: If a single-file format was asked for with several inputs + or a directory, or globs were given for a single-file output. + """ + fmt: str | None = request.format or infer_format(request.output) + many: bool = len(request.inputs) > 1 or any(p.is_dir() for p in request.inputs) + + # Only the .comp container records its codec; the others fix their own + comp_flag: str | None = "-a" if request.algo else "-s" if request.strategy else None + + # The Compresso container is what no format writes, so naming it is the same + if fmt and fmt.lower() == "comp": + if many: + raise ValueError( + "comp holds a single file; " + f"use an archive format such as {DEFAULT_ARCHIVE_FORMAT} for several" + ) + + fmt = None + + if format_is_single_file(fmt) and many: + raise ValueError( + f"{fmt} holds a single file; " + f"use an archive format such as tar.{fmt} for several" + ) + + if is_archive_format(fmt) or (many and fmt is None): + fmt = fmt or DEFAULT_ARCHIVE_FORMAT + output: Path = request.output or default_archive_output(request.inputs[0], fmt) + archive_options = ArchiveOptions( + format=fmt.lower(), + compression_level=request.level, + overwrite=request.overwrite, + threads=request.threads, + include=request.include, + exclude=request.exclude, + remove_sources=request.remove_sources, + dereference=request.dereference, + ) + + archive_job = ArchiveJob.from_paths( + sources=request.inputs, output=output, options=archive_options + ) + if comp_flag: + archive_job.plan = replace( + archive_job.plan, + can_run=False, + reason_if_unavailable=f"{comp_flag} applies only to .comp, not {fmt}", + ) + + return archive_job + + if request.include or request.exclude: + raise ValueError("--include and --exclude apply only to archives") + + source: Path = request.inputs[0] + options = CompressionOptions( + algo=request.algo, + strategy=request.strategy or CompressionOptions.strategy, + level=request.level, + format=fmt, + overwrite=request.overwrite, + threads=request.threads, + ) + + job = CompressionJob.from_file(src=source, dest=request.output, options=options) + + if comp_flag and format_is_single_file(fmt): + job.plan = replace( + job.plan, + can_compress=False, + reason_if_unavailable=f"{comp_flag} applies only to .comp, not {fmt}", + ) + + # Archive formats returned above, so this is one the core doesn't know; + # the plan's own message lists only the single-file formats + elif fmt and not format_is_single_file(fmt): + known = ["comp", *(s.lstrip(".") for s in STANDALONE_SUFFIXES.values())] + for archive, variants in archive_formats().items(): + known += [archive, *variants] + + job.plan = replace( + job.plan, + can_compress=False, + reason_if_unavailable=( + f"Unknown format: {fmt} (expected one of {', '.join(known)})" + ), + ) + + return job + + +def plan_extract(request: ExtractRequest) -> list[ExtractJob | DecompressionJob]: + """Turn an extract request into one planned job per input. + + Each input is dispatched on its own bytes, not its name: an archive is + unpacked into a directory, anything else is decompressed to one file. + + Args: + request: The parsed request. + + Returns: + The planned jobs, in input order; each may still be unable to run. + """ + jobs: list[ExtractJob | DecompressionJob] = [] + for source in request.inputs: + if looks_like_archive(source): + archive_job = ExtractJob.from_archive( + archive=source, + output_dir=request.directory, + files=request.files, + options=request.options, + ) + if request.output: + archive_job.plan = replace( + archive_job.plan, + can_run=False, + reason_if_unavailable="-o names one file; use -C for an archive", + ) + + jobs.append(archive_job) + continue + + plan = partial( + DecompressionJob.from_file, + src=source, + overwrite=request.options.overwrite, + threads=request.options.threads, + restore_name=request.restore_name, + ) + if request.output or not request.directory: + job: DecompressionJob = plan(dest=request.output) + + else: + # Planned once to learn the default name, then moved into `-C` + job = plan(dest=request.directory / plan(dest=None).plan.dest.name) + + if request.files or request.options.include or request.options.exclude: + job.plan = replace( + job.plan, + can_run=False, + reason_if_unavailable=( + "--entry, --include and --exclude apply only to archives" + ), + ) + + jobs.append(job) + + return jobs diff --git a/src/compresso/introspect/__init__.py b/src/compresso/introspect/__init__.py index c13908e..25f2d9a 100644 --- a/src/compresso/introspect/__init__.py +++ b/src/compresso/introspect/__init__.py @@ -4,7 +4,7 @@ The compression backends themselves are the C extension's, under csrc/. """ -from .benchmark import benchmark_file, print_results +from .benchmark import benchmark_file from .capabilities import list_capabilities from .file_inspect import InspectResult, inspect from .speeds import get_estimated_speeds @@ -15,5 +15,4 @@ "get_estimated_speeds", "inspect", "list_capabilities", - "print_results", ] diff --git a/src/compresso/introspect/benchmark.py b/src/compresso/introspect/benchmark.py index 2af603a..ad7bd82 100644 --- a/src/compresso/introspect/benchmark.py +++ b/src/compresso/introspect/benchmark.py @@ -8,8 +8,6 @@ from dataclasses import dataclass from pathlib import Path -from tabulate import tabulate - from .._core import compress_file, decompress_file from .._levels import to_core_level from .capabilities import get_by_name @@ -223,17 +221,7 @@ def benchmark_file( level=level, ) comp_times.append(time.perf_counter() - start_time) - - if compressed_size is None: - compressed_size: int = comp_path.stat().st_size - - else: - sz: int = comp_path.stat().st_size - if sz != compressed_size: - print( - f"Warning: Compressed size changed between runs: {compressed_size} vs {sz}" - ) - compressed_size: int = sz + compressed_size = comp_path.stat().st_size # Decompression start_time: float = time.perf_counter() @@ -243,15 +231,6 @@ def benchmark_file( ) decomp_times.append(time.perf_counter() - start_time) - # Verify - if ( - not decomp_path.is_file() - or decomp_path.stat().st_size != input_size - ): - print( - f"Warning: Decompressed file size mismatch for {decomp_path}" - ) - finally: _safe_unlink(path=comp_path) _safe_unlink(path=decomp_path) @@ -275,41 +254,3 @@ def benchmark_file( update_from_benchmarks(results) return results - - -def print_results(results: list[BenchmarkResult]) -> None: - """Print benchmark results in a tabular format - - Args: - results: List of BenchmarkResult objects to print - """ - if not results: - print("No results to display.") - return - - headers: list[str] = [ - "Algo", - "Strategy", - "Level", - "Comp Time (s)", - "Decomp Time (s)", - "MB/s (Comp)", - "MB/s (Decomp)", - "Ratio (comp/orig)", - ] - - table_data: list[list[str | int]] = [] - for r in results: - row: list[str | int] = [ - r.algo, - r.strategy, - r.level if r.level is not None else "auto", - f"{r.compress_time:.4f}", - f"{r.decompress_time:.4f}", - f"{r.comp_mb_s:.2f}", - f"{r.decomp_mb_s:.2f}", - f"{r.ratio:.3f}", - ] - table_data.append(row) - - print(tabulate(tabular_data=table_data, headers=headers, tablefmt="grid")) diff --git a/src/compresso/meson.build b/src/compresso/meson.build index 283792f..1bd25bd 100644 --- a/src/compresso/meson.build +++ b/src/compresso/meson.build @@ -52,21 +52,33 @@ py.install_sources( 'cli/__init__.py', 'cli/__main__.py', 'cli/_app.py', - 'cli/_dispatch.py', - 'cli/_render.py', - 'cli/algos.py', + 'cli/_run.py', 'cli/benchmark.py', 'cli/compress.py', 'cli/extract.py', - 'cli/inspect.py', + 'cli/formats.py', + 'cli/info.py', + 'cli/list.py', + 'cli/verify.py', subdir: 'compresso/cli', ) +py.install_sources( + 'cli/_render/__init__.py', + 'cli/_render/benchmark.py', + 'cli/_render/formats.py', + 'cli/_render/info.py', + 'cli/_render/jobs.py', + subdir: 'compresso/cli/_render', +) + py.install_sources( 'frontend/__init__.py', + 'frontend/_formats.py', 'frontend/_job.py', 'frontend/api.py', 'frontend/archive_api.py', + 'frontend/dispatch.py', subdir: 'compresso/frontend', ) diff --git a/tests/c/fuzz/gzip_name.c b/tests/c/fuzz/gzip_name.c index 339b8e2..502611b 100644 --- a/tests/c/fuzz/gzip_name.c +++ b/tests/c/fuzz/gzip_name.c @@ -1,4 +1,4 @@ -// libFuzzer target for the gzip header's stored name, which is read by path +// libFuzzer target for what `info` reads from a header without decoding #include "standalone.h" #include @@ -31,5 +31,15 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { fwrite(data, 1, size, f); fclose(f); free(gzip_original_name(path)); + + f = fopen(path, "rb"); + if (f) { + uint64_t content_size; + standalone_content_size(f, FORMAT_XZ, &content_size); + standalone_content_size(f, FORMAT_ZSTD, &content_size); + standalone_content_size(f, FORMAT_LZ4, &content_size); + standalone_content_size(f, FORMAT_SNAPPY, &content_size); + fclose(f); + } return 0; } diff --git a/tests/c/test_select.c b/tests/c/test_select.c index b9e90ee..828bbe5 100644 --- a/tests/c/test_select.c +++ b/tests/c/test_select.c @@ -40,8 +40,8 @@ TEST_RANGE([ 0, 6, 1 ]) void test_names_select_whole_components(int index) { const SelectCase *c = &SELECT_CASES[index]; Selection sel; - TEST_ASSERT_EQUAL_INT( - 0, selection_init(&sel, (const char **)c->files, c->num_files)); + TEST_ASSERT_EQUAL_INT(0, selection_init(&sel, (const char **)c->files, + c->num_files, &(PathFilter){0})); char selected[128] = ""; for (size_t i = 0; i < sizeof(NAMES) / sizeof(NAMES[0]); i++) { @@ -66,3 +66,39 @@ void test_names_select_whole_components(int index) { TEST_ASSERT_EQUAL_STRING(c->selected, selected); TEST_ASSERT_EQUAL_STRING(c->unmatched, unmatched); } + +typedef struct { + const char *pat; + const char *path; + int matches; +} MatchCase; + +static const MatchCase MATCH_CASES[] = { + {"*.log", "a/b.log", 1}, + {"*.log", "b.log.gz", 0}, + {"node_modules", "x/node_modules_old/y.js", 0}, + {"a/b", "x/a/b/c", 1}, + {"b", "ab", 0}, + {"a*c", "a/b/c", 1}, // `*` crosses '/' + {"a*b*c", "aXbYbZc", 1}, + {"?.txt", "a.txt", 1}, + {"[!ab].txt", "a.txt", 0}, + {"[a-c]x", "bx", 1}, + {"[]]", "]", 1}, + {"[a", "[a", 1}, // Unterminated, so literal + {"\\*", "*", 1}, + {"\\*", "a", 0}, + {"d", "d/", 1}, // A zip directory's trailing slash + {"", "/a", 0}, // A tar's absolute path has no empty component to match +}; + +// The TEST_RANGE below must span exactly these rows +_Static_assert(sizeof(MATCH_CASES) / sizeof(MATCH_CASES[0]) == 16, + "update TEST_RANGE"); + +TEST_RANGE([ 0, 15, 1 ]) +void test_path_matches(int index) { + const MatchCase *c = &MATCH_CASES[index]; + TEST_ASSERT_EQUAL_INT_MESSAGE(c->matches, path_matches(c->path, c->pat), + c->pat); +} diff --git a/tests/fixtures/codec_manifest.json b/tests/fixtures/codec_manifest.json index 76c8721..e3f043e 100644 --- a/tests/fixtures/codec_manifest.json +++ b/tests/fixtures/codec_manifest.json @@ -26,26 +26,26 @@ "comp/bzip2/9/one_byte": "d9b98e06af4dd4815871946591eb980386839d9efa98ced727efc17b9422032a", "comp/bzip2/9/random": "170882d521c2b4c4ead01c3b9dc52a43721b4df544305127bf329ec2616e049c", "comp/bzip2/9/xargs": "82d5e670f90516791b7685288fa5d1fcdc6829ed27ae86453fa9f7d2a59c27c7", - "comp/lz4/-1/alice": "dadf4529b2493f5af03f2bbffe38a2ae8012f320d549f748690ffbf33ccc5f03", + "comp/lz4/-1/alice": "4752183bd94c55c284404b6b906002fbf176aa29d98a84faa9e6b64be636d6ce", "comp/lz4/-1/empty": "87c4277150bed1c418e64b0009a2922f7dafa818005a110f37dbeafe8df4af4d", - "comp/lz4/-1/one_byte": "7f388b5a3d640aa9e835dc174eb587a2c97a35be9f9f08c9668acb619e7089bb", - "comp/lz4/-1/random": "0756ec5722605a41013e956f7bbd42999fe20b6328ba5a74cb2befe99769fcbc", - "comp/lz4/-1/xargs": "5feaea7f0a22bf57d6587f38822504510dfee7289d29df1cd3f5faf9290749fb", - "comp/lz4/1/alice": "35c40b750ed0535f5ddf0d5175141bceaee03b8dc0851411bb2de9992e9e1e38", + "comp/lz4/-1/one_byte": "10d50f1be4f7c3b56ab4ac1a91e210b267ff9c825ce4a7126afb979135a31f12", + "comp/lz4/-1/random": "de5e823868af8ba53eab8d85fd2ae99783925aef690a387234b2d211dfdb9ffe", + "comp/lz4/-1/xargs": "ac765b6e5d85abda7e6b14aacd03ad5303b8cd0e3cf251d176c8baafc6aec271", + "comp/lz4/1/alice": "c822b3c34f8427501632e0e33c5c08252148de5ae3f10903f70207d95258e49c", "comp/lz4/1/empty": "59294d19c0874a3de85777b90fc4280488cac9fedfd1d4d7060262e933e5ae9b", - "comp/lz4/1/one_byte": "aae68f72724a5fcdb7c2a41835ae258750f011873ff9e6161ad5f3b61d34b503", - "comp/lz4/1/random": "35fa40e904baaf2a54a6782aa411b1b63edb95a0d026ffaa0770ee146775b7b9", - "comp/lz4/1/xargs": "28caf84c9ce033dd178eab509faab6aeac1cb45ee9ca3af554fa2421672b9ab0", - "comp/lz4/6/alice": "9bf7f8d9ee787edd4164131de727dcb0cc67077e4a0ebbc94139a196ac2b09fd", + "comp/lz4/1/one_byte": "7a0fea797622add91d202ae632c03ec9394a6402e7e0190c024d7aa68d1008ce", + "comp/lz4/1/random": "f0963e22cf45c7262d6219942aae482cbb968531de081da94446046f7091c266", + "comp/lz4/1/xargs": "d2198f98e911e4094f414758e146c96af3fb411df5a5657710e46bb5449b4464", + "comp/lz4/6/alice": "9269642f00ec505d38775186659876b1bc44e5ee0dc53c1a5a464ed240e7d575", "comp/lz4/6/empty": "4d33412556dd207e7095d4abd663b8a5fd30197f067edd75c081f3cbd53d00f6", - "comp/lz4/6/one_byte": "2eaff8da9cbe1124e3d8246b96f1267466fed99460121f36bddfad5a2bab7ae6", - "comp/lz4/6/random": "c12ce5e372d69def88e168895e8cc80cbf520123c88a66f10f91e716e3908349", - "comp/lz4/6/xargs": "ade7d1cf51b36670ca238ff76674a774f517e505ab329f7606c8f879f304ba82", - "comp/lz4/9/alice": "dac4b3721e86d24c3e42117295b17270733a2682d35791b1c75da7f1a6e583b1", + "comp/lz4/6/one_byte": "99ab50b6f1011730f61df8a8cd825e9bd624e1b0e6c54026a26a0aa2bfaa2b67", + "comp/lz4/6/random": "8566754262e336d1df411ba9b42a2fd3621449543b8c11509f975eeeefa8b326", + "comp/lz4/6/xargs": "49b715cc01ddd18fe56a8878463dc11879655efc8111e4c0ade479b8ac00d544", + "comp/lz4/9/alice": "e1aa1b0bb9216d14480fecf8d736632bc4b85aeeeda7c0d926a10f38189f36ad", "comp/lz4/9/empty": "e2571af42b428be1440e850df924fac6f5fef61862153692148a6a1ffef77759", - "comp/lz4/9/one_byte": "943242246d2988a4856ae59ab4911046731866b07f7168fd4a343bbe2f026428", - "comp/lz4/9/random": "9015cae80d4521bcf67822e12153984aefe188cbe5d7315cc1c824c5fb3bc954", - "comp/lz4/9/xargs": "3ed465a5f86a4c1040d9d37cc89c60c246225872a277ecda2d7b74a44df99777", + "comp/lz4/9/one_byte": "2ea5fc615396c4eaa1253d7e522c1e2e88a1af52c5f87d0b8b25a24938f8ad6d", + "comp/lz4/9/random": "047b03ed7762ea8600dd4d4483da70b17765a6bd9121e028a68206713980ba58", + "comp/lz4/9/xargs": "22da1815391fc4da766a8b83aabe10eed224cea1fab10bda051d4258b5e98fbd", "comp/lzma/-1/alice": "a56ad264fdfb57a4b1cd7175b12493a7f290ab891f191fe9cd471e062bc613c4", "comp/lzma/-1/empty": "a51c97d296ef49be2dc972aa578f088a10b8dc4fbe9409040f1bb61402a1532a", "comp/lzma/-1/one_byte": "29bd38e9b06cc5798d4c116d8930b7580858c2ec766a39e4fbc558563252740b", @@ -166,26 +166,26 @@ "standalone/gzip/9/one_byte": "676a53aff05d0332d03ae91ae54e0ab34a8a260e2b1840d71fb9d39980648883", "standalone/gzip/9/random": "a36344d3edbf8e9fa14c70239ad107a85049072706d92b89e66180d0025ef3b4", "standalone/gzip/9/xargs": "f6e6121a7577021494e0569d8bef58fc1065727afa714f863957b3b191ae17a3", - "standalone/lz4/-1/alice": "f7c4a5ac4e8cf94f855921922ab4ef3c2b93527674c8ddf3a3bcc6908289fa39", + "standalone/lz4/-1/alice": "36fca6c16ada62893ffa7ccce735e71266acc51e6cb7d10e0beadd33d309a77c", "standalone/lz4/-1/empty": "ff2d4fa25b032b9e88f09b73d8b58bf9b139945bbbe1735d10732636a36844be", - "standalone/lz4/-1/one_byte": "019638231ad533b09ce864640f065779885d89cb8896e7f1d4c828e6d06d8cb1", - "standalone/lz4/-1/random": "29f800ff0713579c3480cea009be421c4bc2ad6f512f8b8ed9f139d764bd5437", - "standalone/lz4/-1/xargs": "5f2484fb3e58f3d617f57908742043360f1eaf6f8b0393cda046695521728dce", - "standalone/lz4/1/alice": "f7c4a5ac4e8cf94f855921922ab4ef3c2b93527674c8ddf3a3bcc6908289fa39", + "standalone/lz4/-1/one_byte": "9c81485a8465e3a8436c50b2f576fabd179682e12fb1129f15b8f025604dfbba", + "standalone/lz4/-1/random": "3688f304f6edf20fa5b94d063b2a3f570ee228fab1b0c68d8c7782f9f548e00a", + "standalone/lz4/-1/xargs": "011b5c117af1061ba6be06e846727a112356df56404b2f41a04a3d24d5ea6e99", + "standalone/lz4/1/alice": "36fca6c16ada62893ffa7ccce735e71266acc51e6cb7d10e0beadd33d309a77c", "standalone/lz4/1/empty": "ff2d4fa25b032b9e88f09b73d8b58bf9b139945bbbe1735d10732636a36844be", - "standalone/lz4/1/one_byte": "019638231ad533b09ce864640f065779885d89cb8896e7f1d4c828e6d06d8cb1", - "standalone/lz4/1/random": "29f800ff0713579c3480cea009be421c4bc2ad6f512f8b8ed9f139d764bd5437", - "standalone/lz4/1/xargs": "5f2484fb3e58f3d617f57908742043360f1eaf6f8b0393cda046695521728dce", - "standalone/lz4/6/alice": "2c551f7b5e65e8214644da8c5bf0b7a4601b137906f64d4661cac95d43fbf801", + "standalone/lz4/1/one_byte": "9c81485a8465e3a8436c50b2f576fabd179682e12fb1129f15b8f025604dfbba", + "standalone/lz4/1/random": "3688f304f6edf20fa5b94d063b2a3f570ee228fab1b0c68d8c7782f9f548e00a", + "standalone/lz4/1/xargs": "011b5c117af1061ba6be06e846727a112356df56404b2f41a04a3d24d5ea6e99", + "standalone/lz4/6/alice": "015e6d6cb674c907220d90bf4f9bc10968ea85c85dc0ce5d653dd00676120898", "standalone/lz4/6/empty": "ff2d4fa25b032b9e88f09b73d8b58bf9b139945bbbe1735d10732636a36844be", - "standalone/lz4/6/one_byte": "019638231ad533b09ce864640f065779885d89cb8896e7f1d4c828e6d06d8cb1", - "standalone/lz4/6/random": "29f800ff0713579c3480cea009be421c4bc2ad6f512f8b8ed9f139d764bd5437", - "standalone/lz4/6/xargs": "46475505866608d6425f137ddb07033864e3d96c915925a100d3cacada5659d1", - "standalone/lz4/9/alice": "32db841e8169ce678805c7a52af8fb3bbb0ee480aa0097e2759c94c5bf44355c", + "standalone/lz4/6/one_byte": "9c81485a8465e3a8436c50b2f576fabd179682e12fb1129f15b8f025604dfbba", + "standalone/lz4/6/random": "3688f304f6edf20fa5b94d063b2a3f570ee228fab1b0c68d8c7782f9f548e00a", + "standalone/lz4/6/xargs": "f9b062b20d56e42aeb8626b50453fb0c7bfac0e53ef6434dc16ee3ee99472b32", + "standalone/lz4/9/alice": "b7a1c0cd137bf2ca55f19758b2f1460264cdef1f3588fc959942e33a7520cc7a", "standalone/lz4/9/empty": "ff2d4fa25b032b9e88f09b73d8b58bf9b139945bbbe1735d10732636a36844be", - "standalone/lz4/9/one_byte": "019638231ad533b09ce864640f065779885d89cb8896e7f1d4c828e6d06d8cb1", - "standalone/lz4/9/random": "29f800ff0713579c3480cea009be421c4bc2ad6f512f8b8ed9f139d764bd5437", - "standalone/lz4/9/xargs": "46475505866608d6425f137ddb07033864e3d96c915925a100d3cacada5659d1", + "standalone/lz4/9/one_byte": "9c81485a8465e3a8436c50b2f576fabd179682e12fb1129f15b8f025604dfbba", + "standalone/lz4/9/random": "3688f304f6edf20fa5b94d063b2a3f570ee228fab1b0c68d8c7782f9f548e00a", + "standalone/lz4/9/xargs": "f9b062b20d56e42aeb8626b50453fb0c7bfac0e53ef6434dc16ee3ee99472b32", "standalone/snappy/-1/alice": "9166e19e9ae120ab62af436e99ad4a59eeb25f1bc57cd1746c64fe24afb6fb48", "standalone/snappy/-1/empty": "75605fd35293ed52e316629e8d3971affee9fee08518e6657188c9e51450a66f", "standalone/snappy/-1/one_byte": "c4b4d76f5c545fa9c0ff12a3315caa3122a7e3222fd05d28a34b3c6969d281a2", diff --git a/tests/python/cli/__init__.py b/tests/python/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/python/cli/conftest.py b/tests/python/cli/conftest.py new file mode 100644 index 0000000..6c03761 --- /dev/null +++ b/tests/python/cli/conftest.py @@ -0,0 +1,166 @@ +"""Shared helpers and fixtures for the CLI tests.""" + +from __future__ import annotations + +import os +from collections.abc import Callable, Generator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path + +import pytest +from typer.testing import CliRunner, Result + +import compresso.cli._run as run_mod +from compresso.cli import app +from compresso.cli._render import ( + BAR_THRESHOLD, + EXIT_FAILED, + EXIT_OK, + progress_bar, +) +from compresso.frontend._job import ProgressCallback +from compresso.frontend.archive_api import ( + ArchiveJob, + ArchiveOptions, +) + +runner = CliRunner() + +# Incompressible throughout, so compressed sizes stay close to these +PAYLOAD_SIZE = 64 * 1024 +TREE_FILE_SIZE = 16 * 1024 + +# Past the progress bar threshold, for the tests that cover the bar +LARGE_PAYLOAD_SIZE = 2 * BAR_THRESHOLD + + +def invoke(*args: str | Path, expect: int | None = EXIT_OK) -> Result: + """Run the CLI, asserting it exits with `expect`. + + Args: + *args: The command line, without the program name. + expect: The exit code to assert, or None to leave it unchecked. + + Returns: + The runner's result. + """ + result = runner.invoke(app, [str(arg) for arg in args]) + if expect is not None: + assert result.exit_code == expect, result.output + + return result + + +@pytest.fixture +def payload(tmp_path: Path) -> Path: + """A small incompressible file, below the progress bar threshold.""" + path = tmp_path / "payload.bin" + path.write_bytes(os.urandom(PAYLOAD_SIZE)) + + return path + + +@pytest.fixture +def source_tree(tmp_path: Path) -> Path: + """A small directory tree to archive, function-scoped because some tests delete it.""" + root = tmp_path / "tree" + root.mkdir() + for i in range(3): + (root / f"f{i}.bin").write_bytes(os.urandom(TREE_FILE_SIZE)) + + return root + + +@pytest.fixture +def make_archive(source_tree: Path, tmp_path: Path) -> Callable[[str], Path]: + """Build an archive of `source_tree` as input for a test. + + Built through the API rather than the CLI, and checked, so a broken setup + cannot surface as a confusing failure later in the test. + """ + + def make(fmt: str) -> Path: + archive = tmp_path / f"out.{fmt}" + options = ArchiveOptions(format=fmt) + result = ArchiveJob.from_paths([source_tree], archive, options).run() + assert result.ok, result.error + return archive + + return make + + +@dataclass +class CompressTarget: + """What one `compress` call reads and writes.""" + + source: Path + dest: Path + args: tuple[str, ...] + + +@pytest.fixture(params=["file", "tree"]) +def compress_target( + request: pytest.FixtureRequest, payload: Path, source_tree: Path, tmp_path: Path +) -> CompressTarget: + """A single file into a .comp container, or a tree into a tar archive. + + The two take different paths through `compress`, but share its + overwrite-mode handling. + """ + if request.param == "file": + return CompressTarget(payload, tmp_path / "out.comp", ()) + + return CompressTarget(source_tree, tmp_path / "out.tar", ("-F", "tar")) + + +@dataclass +class Bar: + """One progress bar a command opened.""" + + enabled: bool + updates: int = 0 + + +@pytest.fixture +def bars(monkeypatch: pytest.MonkeyPatch) -> list[Bar]: + """Record each progress bar a command opens, in order, still drawing the real one.""" + opened: list[Bar] = [] + + @contextmanager + def spy( + label: str, total: int, *, enabled: bool + ) -> Generator[ProgressCallback | None]: + bar = Bar(enabled) + opened.append(bar) + with progress_bar(label, total, enabled=enabled) as on_progress: + if on_progress is None: + yield None + return + + def counting(fraction: float, done: int, reported_total: int) -> None: + bar.updates += 1 + on_progress(fraction, done, reported_total) + + yield counting + + monkeypatch.setattr(run_mod, "progress_bar", spy) + + return opened + + +# Each overwrite mode against an existing output: whether the old output +# survives, and whether the new one is written beside it under a new name +CONFLICT_MODES = pytest.mark.parametrize( + "flags,expect,kept,renames", + [ + ((), EXIT_OK, True, True), + (("--on-conflict", "rename"), EXIT_OK, True, True), + (("--on-conflict", "error"), EXIT_FAILED, True, False), + (("--on-conflict", "skip"), EXIT_OK, True, False), + (("--on-conflict", "overwrite"), EXIT_OK, False, False), + (("-f",), EXIT_OK, False, False), + (("-f", "--on-conflict", "overwrite"), EXIT_OK, False, False), + ], + ids=["default", "rename", "error", "skip", "overwrite", "force", "force+overwrite"], +) diff --git a/tests/python/cli/test_app.py b/tests/python/cli/test_app.py new file mode 100644 index 0000000..c7e0b9c --- /dev/null +++ b/tests/python/cli/test_app.py @@ -0,0 +1,525 @@ +"""Tests for the CLI app as a whole, and what every command shares.""" + +from __future__ import annotations + +import gzip +import io +import os +import re +import subprocess +import sys +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +import compresso.cli.formats as formats_cmd +import compresso.cli.info as info_cmd +import compresso.cli.list as list_cmd +from compresso import cli +from compresso._core import Error +from compresso.cli._render import ( + EXIT_CANCELLED, + EXIT_FAILED, + EXIT_USAGE, + format_size, + format_time, +) +from compresso.frontend.api import CompressionJob, DecompressionJob, VerificationJob +from compresso.frontend.archive_api import ( + ArchiveJob, + ExtractJob, +) + +from .conftest import LARGE_PAYLOAD_SIZE, PAYLOAD_SIZE, Bar, CompressTarget, invoke + + +class TestFormatters: + """Test utility formatting functions.""" + + @pytest.mark.parametrize( + "size,expected", + [ + (0, "0.00 B"), + (500, "500.00 B"), + (1024, "1.00 KB"), + (1536, "1.50 KB"), + (1048576, "1.00 MB"), + (1073741824, "1.00 GB"), + (1099511627776, "1.00 TB"), + ], + ) + def test_format_size(self, size: int, expected: str): + """Test size formatting with various values.""" + result = format_size(size) + assert result == expected + + @pytest.mark.parametrize( + "seconds,expected", + [ + (0.0005, "0ms"), + (0.5, "500ms"), + (1.0, "1.00s"), + (5.5, "5.50s"), + (59.9, "59.90s"), + (60.0, "1m 0.0s"), + (90.5, "1m 30.5s"), + (125.0, "2m 5.0s"), + ], + ) + def test_format_time(self, seconds: float, expected: str): + """Test time formatting with various values.""" + assert format_time(seconds) == expected + + +class TestAppStructure: + """Test the package assembles into one app with every command registered.""" + + def test_commands_are_listed_in_workflow_order(self) -> None: + """Test that the order is chosen in `__init__`, not inherited from import sorting. + + Alphabetical imports would put `benchmark` first. + """ + output = invoke("--help").output + + # Matched on the alias column, which is unique per row, or the + # description where there is none; a bare command name also appears + # inside other commands' descriptions + positions = [ + output.index(aliases) + for aliases in [ + "(c)", + "(x)", + "(t)", + "(l)", + "(i)", + "List the available", + "(b)", + ] + ] + assert positions == sorted(positions) + + @pytest.mark.parametrize( + "alias,command", + [ + ("c", "compress"), + ("x", "extract"), + ("t", "test"), + ("l", "list"), + ("i", "info"), + ("b", "benchmark"), + ], + ) + def test_aliases_still_resolve(self, alias: str, command: str) -> None: + """Test that each short name reaches the command it stands for. + + Help is compared without its usage line, which echoes the name typed. + """ + + def help_body(name: str) -> list[str]: + lines = invoke(name, "--help").output.splitlines() + return [line for line in lines if "Usage:" not in line] + + assert help_body(alias) == help_body(command) + + +class TestProgressBar: + """Test the bar is drawn only for a large job that is not asked to be quiet.""" + + @pytest.mark.parametrize( + "size,quiet,expected", + [ + (LARGE_PAYLOAD_SIZE, False, True), + (LARGE_PAYLOAD_SIZE, True, False), + (PAYLOAD_SIZE, False, False), + ], + ids=["large", "large-quiet", "small"], + ) + def test_compress_bar_follows_size_and_quiet( + self, tmp_path: Path, bars: list[Bar], size: int, quiet: bool, expected: bool + ) -> None: + """Test that the bar needs both a job past the threshold and no --quiet.""" + source = tmp_path / "source.bin" + source.write_bytes(os.urandom(size)) + + invoke( + "compress", source, "-o", tmp_path / "out.comp", *(["-q"] if quiet else []) + ) + + assert [bar.enabled for bar in bars] == [expected] + + def test_file_round_trip_draws_both_bars( + self, tmp_path: Path, bars: list[Bar] + ) -> None: + """Test that compress and extract each drive a bar with real progress.""" + source = tmp_path / "source.bin" + source.write_bytes(os.urandom(LARGE_PAYLOAD_SIZE)) + packed = tmp_path / "out.comp" + restored = tmp_path / "restored.bin" + + invoke("compress", source, "-o", packed) + invoke("extract", packed, "-o", restored) + + assert [bar.enabled for bar in bars] == [True, True] + assert all(bar.updates > 0 for bar in bars) + assert restored.read_bytes() == source.read_bytes() + + def test_archive_round_trip_draws_both_bars( + self, tmp_path: Path, bars: list[Bar] + ) -> None: + """Test that archiving and extracting a tree each drive a bar too.""" + tree = tmp_path / "big_tree" + tree.mkdir() + for i in range(2): + (tree / f"f{i}.bin").write_bytes(os.urandom(LARGE_PAYLOAD_SIZE // 2)) + archive = tmp_path / "out.tar" + dest = tmp_path / "dest" + + invoke("compress", tree, "-o", archive, "-F", "tar") + invoke("extract", archive, "-C", dest) + + assert [bar.enabled for bar in bars] == [True, True] + assert all(bar.updates > 0 for bar in bars) + for original in tree.iterdir(): + assert (dest / tree.name / original.name).read_bytes() == ( + original.read_bytes() + ) + + +@pytest.fixture(params=["compress", "archive", "decompress", "extract"]) +def job_args( + request: pytest.FixtureRequest, + payload: Path, + source_tree: Path, + tmp_path: Path, + make_archive: Callable[[str], Path], +) -> tuple[list[str | Path], Path]: + """One command line for each kind of job, and the path its summary names.""" + packed = tmp_path / "out.comp" + match request.param: + case "compress": + return ["compress", payload, "-o", packed], packed + + case "archive": + return ["compress", source_tree, "-o", tmp_path / "out.tar"], ( + tmp_path / "out.tar" + ) + + case "decompress": + invoke("compress", payload, "-o", packed, "-q") + return ["extract", packed, "-o", tmp_path / "restored.bin"], packed + + case _: + archive = make_archive("tar") + return ["extract", archive, "-C", tmp_path / "dest"], archive + + +class TestOutput: + """Test one summary line per input by default, the blocks with -v, and + nothing with -q.""" + + def test_default_is_one_summary_line( + self, job_args: tuple[list[str | Path], Path] + ) -> None: + """Test that a job prints `name before → after (n%) in time` and no + more, without the ratio on extraction.""" + args, name = job_args + output = invoke(*args).output + + ratio = r"\(\d+%\) " if args[0] == "compress" else "" + + assert re.fullmatch( + rf"{re.escape(str(name))} \S+ \S+ → \S+ \S+ {ratio}in \S+\n\n", output + ), output + + def test_inputs_are_one_blank_line_apart( + self, payload: Path, tmp_path: Path + ) -> None: + """Test that several inputs' lines have one blank line between and + after them.""" + inputs = [tmp_path / "a.gz", tmp_path / "b.gz"] + for path in inputs: + path.write_bytes(gzip.compress(payload.read_bytes())) + + output = invoke("extract", *inputs, "-C", tmp_path / "dest").output + + assert re.fullmatch( + "".join(rf"{re.escape(str(path))} [^\n]+\n\n" for path in inputs), + output, + ), output + + def test_verbose_prints_the_blocks( + self, job_args: tuple[list[str | Path], Path] + ) -> None: + """Test that -v brings back the plan before and the stats after.""" + args, _ = job_args + output = invoke(*args, "-v").output + + assert "Output" in output + assert "Time:" in output + + def test_quiet_prints_nothing( + self, job_args: tuple[list[str | Path], Path] + ) -> None: + """Test that -q prints nothing on success.""" + args, _ = job_args + assert invoke(*args, "-q").output == "" + + def test_verbose_with_quiet_is_usage( + self, job_args: tuple[list[str | Path], Path] + ) -> None: + """Test that -v and -q together are refused before any work starts.""" + args, _ = job_args + result = invoke(*args, "-v", "-q", expect=EXIT_USAGE) + assert "-v contradicts -q" in result.output + + def test_an_error_lands_after_the_inputs_before_it( + self, payload: Path, tmp_path: Path + ) -> None: + """Test that a failed input's error follows the earlier inputs' lines + when stdout is a pipe, which block-buffers it.""" + good = tmp_path / "good.gz" + good.write_bytes(gzip.compress(payload.read_bytes())) + bad = tmp_path / "bad.bin" + bad.write_bytes(payload.read_bytes()) + + result = subprocess.run( + [sys.executable, "-m", "compresso.cli", "extract", good, bad], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + check=False, + ) + + assert result.returncode == EXIT_FAILED + assert result.stdout.index(str(good)) < result.stdout.index(str(bad)) + + @pytest.mark.parametrize( + "argv", + [["formats"], ["--help"], ["info", "missing"]], + ids=["unpadded", "already-padded", "stderr"], + ) + def test_a_terminal_gets_one_blank_line_around_output( + self, monkeypatch: pytest.MonkeyPatch, argv: list[str] + ) -> None: + """Test that main() pads terminal output to exactly one blank line + before and after, whether or not it already had one.""" + + class Terminal(io.StringIO): + def isatty(self) -> bool: + return True + + # One stream for both, so the order they were written in shows + terminal = Terminal() + monkeypatch.setattr(sys, "stdout", terminal) + monkeypatch.setattr(sys, "stderr", terminal) + monkeypatch.setattr(sys, "argv", ["compresso", *argv]) + monkeypatch.setattr(cli._Padded, "tail", None) + monkeypatch.setattr(cli._Padded, "last", None) + + with pytest.raises(SystemExit): + cli.main() + + visible = re.sub(r"\x1b\[[0-9;]*m| ", "", terminal.getvalue()) + assert re.fullmatch(r"\n[^\n].*[^\n]\n\n", visible, re.DOTALL), visible + + +class TestExitCodes: + """Test 0 success, 1 the operation failed, 2 the request could not be made.""" + + @pytest.mark.parametrize( + "command,name", + [ + ("compress", "nope.bin"), + ("info", "nope.comp"), + ("extract", "nope.gz"), + ("list", "nope.zip"), + ("test", "nope.gz"), + ], + ) + def test_bad_input_is_usage(self, payload: Path, command: str, name: str) -> None: + """Test that a missing input is wrong, rather than the operation having + failed.""" + invoke(command, payload.parent / name, expect=EXIT_USAGE) + + @pytest.mark.skipif( + sys.platform == "win32" or os.geteuid() == 0, reason="needs POSIX, not root" + ) + @pytest.mark.parametrize( + "command,shown", + [("list", "f0.bin"), ("info", "Entries:"), ("extract", "out.zip ")], + ) + def test_unreadable_input_fails_alone( + self, + tmp_path: Path, + make_archive: Callable[[str], Path], + command: str, + shown: str, + ) -> None: + """Test that an OSError from one input is reported against it, as gzip + does, and the next input still runs.""" + locked = tmp_path / "locked.zip" + locked.write_bytes(b"") + locked.chmod(0) + + result = invoke(command, locked, make_archive("zip"), expect=EXIT_FAILED) + + assert f"{locked}: Permission denied" in result.output + assert shown in result.stdout + + @pytest.mark.parametrize("command", ["compress", "extract"]) + @pytest.mark.parametrize("mode", ["rename", "skip", "error"]) + def test_force_with_another_conflict_mode_is_usage( + self, + source_tree: Path, + tmp_path: Path, + make_archive: Callable[[str], Path], + command: str, + mode: str, + ) -> None: + """Test that -f, meaning overwrite, contradicts any other --on-conflict.""" + if command == "compress": + args = [source_tree, "-o", tmp_path / "out.tar", "-F", "tar"] + else: + args = [make_archive("tar"), "-C", tmp_path / "d"] + + result = invoke(command, *args, "-f", "--on-conflict", mode, expect=EXIT_USAGE) + assert f"-f contradicts --on-conflict {mode}" in result.output + + # An archive's level is checked against the stage that compresses + @pytest.mark.parametrize( + "compress_target,args,message", + [ + ( + "file", + ["-a", "zlib", "-l", "15"], + "zlib compression level 15 out of range (0-9", + ), + ("tree", ["-l", "5"], "tar has no compression levels"), + ], + indirect=["compress_target"], + ) + def test_out_of_range_level_is_usage( + self, compress_target: CompressTarget, args: list[str], message: str + ) -> None: + """Test that a level the backend rejects is refused up front, naming the range.""" + t = compress_target + result = invoke( + "compress", t.source, "-o", t.dest, *t.args, *args, expect=EXIT_USAGE + ) + assert message in result.output + assert not t.dest.exists() + + @pytest.mark.parametrize("flags", [(), ("-F", "7z")], ids=["suffix", "flag"]) + def test_recognised_but_unsupported_archive_names_itself( + self, source_tree: Path, tmp_path: Path, flags: tuple[str, ...] + ) -> None: + """Test that 7z, asked for by -F or by `-o x.7z` alone, reports its own name + rather than being written as a zip.""" + dest = tmp_path / "o.7z" + result = invoke("compress", source_tree, "-o", dest, *flags, expect=EXIT_FAILED) + assert "7z archives are recognised but not supported yet" in result.output + assert not dest.exists() + + +class TestInterruption: + """Test Ctrl-C reports and exits 130. + + ExtendedTyper catches KeyboardInterrupt around the command callback and turns + it into a silent `Exit(130)`, so each command catches it first to say what + was interrupted. + """ + + @pytest.mark.parametrize( + "command,expected", + [ + (["compress"], "Compression cancelled"), + (["info"], "Inspection cancelled"), + (["list"], "Listing cancelled"), + (["test"], "Test cancelled"), + (["formats"], "Format listing cancelled"), + ], + ) + def test_interrupt_is_reported( + self, + payload: Path, + monkeypatch: pytest.MonkeyPatch, + command: list[str], + expected: str, + ) -> None: + """Test that each command names itself rather than dying silently.""" + + # Interrupt the first real call each command makes + def interrupt(*args: object, **kwargs: object) -> None: + raise KeyboardInterrupt + + monkeypatch.setattr(CompressionJob, "from_file", interrupt) + monkeypatch.setattr(VerificationJob, "from_file", interrupt) + monkeypatch.setattr(info_cmd, "detect_format", interrupt) + monkeypatch.setattr(list_cmd, "detect_format", interrupt) + monkeypatch.setattr(formats_cmd, "list_capabilities", interrupt) + + args: list[str | Path] = [*command] + if command[0] != "formats": + args.append(payload) + + result = invoke(*args, expect=EXIT_CANCELLED) + + assert expected in result.output + + +def test_core_error_names_the_operation(monkeypatch: pytest.MonkeyPatch) -> None: + """Test that an error from the core is reported as the command's, not unexpected.""" + + def boom() -> None: + raise Error("boom") + + monkeypatch.setattr(formats_cmd, "list_capabilities", boom) + + result = invoke("formats", expect=EXIT_FAILED) + + assert "Format listing error: boom" in result.output + + +class TestThreads: + """Test that -T reaches each kind of job, and refuses a negative count.""" + + def test_threads_reach_every_job( + self, + payload: Path, + source_tree: Path, + make_archive: Callable[[str], Path], + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Test that compress, extract and test pass -T on, one file or many.""" + archive = make_archive("tar") + seen: list[int] = [] + + # Record the count each job was planned with, then stop before it runs + def record(**kwargs: Any) -> None: + options = kwargs.get("options") + seen.append(kwargs["threads"] if options is None else options.threads) + raise KeyboardInterrupt + + monkeypatch.setattr(CompressionJob, "from_file", record) + monkeypatch.setattr(ArchiveJob, "from_paths", record) + monkeypatch.setattr(DecompressionJob, "from_file", record) + monkeypatch.setattr(ExtractJob, "from_archive", record) + monkeypatch.setattr(VerificationJob, "from_file", record) + + for command, source in [ + ("compress", payload), + ("compress", source_tree), + ("extract", payload), + ("extract", archive), + ("test", archive), + ]: + invoke(command, source, "-T", "3", expect=EXIT_CANCELLED) + + assert seen == [3, 3, 3, 3, 3] + + @pytest.mark.parametrize("command", ["compress", "extract", "test"]) + def test_negative_threads_is_usage(self, payload: Path, command: str) -> None: + """Test that a count below 0 is refused before any work starts.""" + invoke(command, payload, "-T", "-1", expect=EXIT_USAGE) diff --git a/tests/python/cli/test_benchmark.py b/tests/python/cli/test_benchmark.py new file mode 100644 index 0000000..d4e6627 --- /dev/null +++ b/tests/python/cli/test_benchmark.py @@ -0,0 +1,99 @@ +"""Tests for `compresso benchmark`.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +import compresso.cli.benchmark as benchmark_cmd +from compresso.cli._render import EXIT_USAGE +from compresso.introspect.benchmark import BenchmarkResult + +from .conftest import invoke + + +@pytest.fixture +def calls(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, Any]]: + """Record each `benchmark_file` call, answering with one canned result.""" + seen: list[dict[str, Any]] = [] + + def record(**kwargs: Any) -> list[BenchmarkResult]: + seen.append(kwargs) + return [BenchmarkResult("zstd", "fast", None, 0.5, 0.25, 1000, 400)] + + monkeypatch.setattr(benchmark_cmd, "benchmark_file", record) + return seen + + +class TestBenchmark: + """Test how `benchmark` turns its flags into a run, and prints it.""" + + @pytest.mark.parametrize( + "args,algos,strategies,levels,repeats", + [ + ((), None, None, [None], 1), + ( + ( + "-a", + "zstd", + "--algo", + "lz4", + "-s", + "fast", + "--strategy", + "max_ratio", + ), + ["zstd", "lz4"], + ["fast", "max_ratio"], + [None], + 1, + ), + (("-l", "3", "--level", "AUTO", "-l", "9"), None, None, [3, None, 9], 1), + (("-r", "3"), None, None, [None], 3), + ], + ids=["defaults", "algo-strategy", "level", "repeats"], + ) + def test_flags_reach_benchmark_file( + self, + calls: list[dict[str, Any]], + payload: Path, + args: tuple[str, ...], + algos: list[str] | None, + strategies: list[str] | None, + levels: list[int | None], + repeats: int, + ) -> None: + """Test that each repeated flag arrives as a list, and an omitted -a or -s + as None, meaning all.""" + invoke("benchmark", payload, *args) + + (call,) = calls + assert call["algos"] == algos + assert call["strategies"] == strategies + assert call["levels"] == levels + assert call["repeats"] == repeats + + def test_prints_settings_then_table( + self, calls: list[dict[str, Any]], payload: Path + ) -> None: + """Test the header lines and that the table names the auto level.""" + output = invoke( + "benchmark", payload, "-a", "zstd", "-l", "auto", "-l", "3" + ).output + + assert output.startswith( + f"Running benchmarks on: {payload}\nRepeats: 1\n" + "Algorithms: zstd\nLevels: auto, 3\n\n" + ) + assert "| zstd | fast | auto |" in output + + @pytest.mark.parametrize("level", ["nope", "1.5", ""]) + def test_bad_level_is_usage( + self, calls: list[dict[str, Any]], payload: Path, level: str + ) -> None: + """Test that a level that is neither a number nor auto is a usage error.""" + invoke("benchmark", payload, "-l", level, expect=EXIT_USAGE) + + assert not calls diff --git a/tests/python/cli/test_compress.py b/tests/python/cli/test_compress.py new file mode 100644 index 0000000..e157875 --- /dev/null +++ b/tests/python/cli/test_compress.py @@ -0,0 +1,333 @@ +"""Tests for `compresso compress`.""" + +from __future__ import annotations + +import tarfile +from pathlib import Path + +import pytest + +from compresso.cli._render import ( + EXIT_FAILED, + EXIT_USAGE, +) + +from ..helpers import renamed +from .conftest import CONFLICT_MODES, CompressTarget, invoke + + +class TestCompressRoundTrip: + """Test compress and decompress, end to end.""" + + def test_compress_then_decompress(self, payload: Path, tmp_path: Path) -> None: + """Test that a file survives a round trip through the CLI unchanged.""" + archive = tmp_path / "out.comp" + restored = tmp_path / "restored.bin" + + invoke("compress", payload, "-o", archive, "-q") + invoke("extract", archive, "-o", restored, "-q") + + assert restored.read_bytes() == payload.read_bytes() + + # The backends are covered in core and C tests; this checks the flags + @pytest.mark.parametrize( + "args", + [ + ("-a", "bzip2"), + ("-a", "zstd", "-l", "19"), + ("-s", "fast"), + ("-s", "MAX_RATIO"), + ], + ids=["algo", "level", "strategy", "strategy-any-case"], + ) + def test_algorithms_round_trip( + self, payload: Path, tmp_path: Path, args: tuple[str, ...] + ) -> None: + """Test that -a, -l and -s reach the backend.""" + archive = tmp_path / "out.comp" + restored = tmp_path / "restored.bin" + + invoke("compress", payload, "-o", archive, *args, "-q") + invoke("extract", archive, "-o", restored, "-q") + + assert restored.read_bytes() == payload.read_bytes() + + +class TestCompOnlyFlags: + """Test that -a and -s, which pick the .comp codec, are refused elsewhere.""" + + def test_algo_with_strategy(self, payload: Path) -> None: + """Test that -a and -s together are a usage error.""" + result = invoke( + "compress", payload, "-a", "zstd", "-s", "fast", expect=EXIT_USAGE + ) + + assert "-a contradicts -s" in result.output + + @pytest.mark.parametrize( + ("args", "choices"), + [(("-s", "fsat"), ("fast", "max_ratio")), (("-a", "zsdt"), ("auto", "zstd"))], + ids=["strategy", "algo"], + ) + def test_unknown_choice( + self, payload: Path, args: tuple[str, ...], choices: tuple[str, ...] + ) -> None: + """Test that a strategy or algorithm that isn't one is refused while parsing.""" + result = invoke("compress", payload, *args, expect=EXIT_USAGE) + + assert all(choice in result.output for choice in choices) + + @pytest.mark.parametrize( + ("args", "message"), + [ + (("-a", "zstd", "-F", "gz"), "-a applies only to .comp, not gz"), + (("-s", "fast", "-o", "x.zip"), "-s applies only to .comp, not zip"), + ], + ids=["algo-standalone", "strategy-archive"], + ) + def test_refused_outside_comp( + self, + payload: Path, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + args: tuple[str, ...], + message: str, + ) -> None: + """Test that a format with its own codec refuses -a and -s, writing nothing.""" + monkeypatch.chdir(tmp_path) + before = set(tmp_path.iterdir()) + + result = invoke("compress", payload, *args, expect=EXIT_USAGE) + + assert message in result.output + assert set(tmp_path.iterdir()) == before + + +class TestCompressConflicts: + """Test what `compress` does when its output already exists.""" + + @CONFLICT_MODES + def test_existing_output( + self, + compress_target: CompressTarget, + flags: tuple[str, ...], + expect: int, + kept: bool, + renames: bool, + ) -> None: + """Test that each overwrite mode keeps, replaces or renames around the + output, and the summary line says which.""" + t = compress_target + t.dest.write_bytes(b"stale") + + result = invoke( + "compress", t.source, "-o", t.dest, *t.args, *flags, expect=expect + ) + + sibling = t.dest.parent / renamed(t.dest.name) + assert (t.dest.read_bytes() == b"stale") == kept + assert sibling.is_file() == renames + assert (f"output renamed to {sibling}" in result.output) == renames + assert (f"{t.dest} skipped" in result.output) == ("skip" in flags) + + +class TestDereference: + """Test `-h`.""" + + @pytest.mark.usefixtures("needs_symlinks") + @pytest.mark.parametrize("flag", ["-h", "--dereference"]) + def test_archive_holds_the_target(self, tmp_path: Path, flag: str) -> None: + """Test that the flag reaches the archive.""" + tree = tmp_path / "tree" + tree.mkdir() + (tree / "a").write_text("hello") + (tree / "link").symlink_to("a") + output = tmp_path / "out.tar" + + invoke("compress", tree, "-o", output, flag, "-q") + + with tarfile.open(output) as tar: + assert tar.getmember("tree/link").isfile() + + +class TestRemoveInput: + """Test what `--rm` and `-k` do to the input.""" + + @pytest.mark.parametrize( + "flags,removes", + [ + ((), False), + (("--rm",), True), + (("-k",), False), + (("--rm", "--keep"), False), + (("-k", "--rm"), True), + ], + ids=["default", "rm", "keep", "rm-then-keep", "keep-then-rm"], + ) + def test_flags( + self, + compress_target: CompressTarget, + flags: tuple[str, ...], + removes: bool, + ) -> None: + """Test that the input goes only with --rm, the last flag winning.""" + t = compress_target + + invoke("compress", t.source, "-o", t.dest, *t.args, *flags, "-q") + + assert t.dest.is_file() + assert t.source.exists() != removes + + def test_a_failed_run_keeps_the_input( + self, compress_target: CompressTarget + ) -> None: + """Test that an output refused by --on-conflict leaves the input.""" + t = compress_target + t.dest.write_bytes(b"stale") + + invoke( + "compress", + t.source, + "-o", + t.dest, + *t.args, + "--on-conflict", + "error", + "--rm", + expect=EXIT_FAILED, + ) + + assert t.source.exists() + + def test_skip_is_refused(self, payload: Path) -> None: + """Test that --rm can't lose an input to an output that was skipped.""" + result = invoke( + "compress", payload, "--on-conflict", "skip", "--rm", expect=EXIT_USAGE + ) + + assert "--rm contradicts --on-conflict skip" in result.output + assert payload.exists() + + def test_undeletable_file_follows_the_summary( + self, locked: Path, tmp_path: Path + ) -> None: + """Test that an input that won't go is a warning after a normal summary.""" + source = locked / "a.txt" + source.write_text("a") + locked.chmod(0o555) + out = tmp_path / "a.gz" + + result = invoke("compress", source, "-o", out, "--rm", expect=EXIT_FAILED) + + assert out.is_file() + assert source.exists() + warning = f"! {source}: not removed: Permission denied" + assert result.output.index("→") < result.output.index(warning) + + @pytest.mark.parametrize("flags", [(), ("-q",)], ids=["default", "quiet"]) + def test_undeletable_archive_source_names_what_was_written( + self, + locked: Path, + source_tree: Path, + tmp_path: Path, + flags: tuple[str, ...], + ) -> None: + """Test that the archive isn't called failed, and a rename is reported.""" + stuck = locked / "a.txt" + stuck.write_text("a") + locked.chmod(0o555) + out = tmp_path / "out.tar" + out.write_bytes(b"stale") + + result = invoke( + "compress", + source_tree, + locked, + "-o", + out, + "--rm", + *flags, + expect=EXIT_FAILED, + ) + + written = tmp_path / renamed("out.tar") + assert tarfile.is_tarfile(written) + assert out.read_bytes() == b"stale" + assert not source_tree.exists() + assert stuck.exists() + assert f"! {stuck}: not removed: Permission denied" in result.output + assert "Archive failed" not in result.output + assert (written.name in result.output) != bool(flags) + + +class TestSmartCompress: + """Test `compress` runs the job dispatch plans; the rules are in frontend/test_dispatch.py.""" + + def test_a_directory_round_trips_as_an_archive( + self, source_tree: Path, tmp_path: Path + ) -> None: + """Test that a directory is written as the default archive and comes back.""" + invoke("compress", source_tree, "-q") + + dest = tmp_path / "back" + invoke( + "extract", + source_tree.with_name(f"{source_tree.name}.zip"), + "-C", + dest, + "-q", + ) + + for original in source_tree.iterdir(): + restored = dest / source_tree.name / original.name + assert restored.read_bytes() == original.read_bytes() + + def test_single_file_format_refuses_several_inputs( + self, source_tree: Path, tmp_path: Path + ) -> None: + """Test that a .gz holds one file, so a directory cannot go into one.""" + result = invoke( + "compress", + source_tree, + "-F", + "gz", + "-o", + tmp_path / "o.gz", + expect=EXIT_USAGE, + ) + + assert "single file" in result.output + + def test_filters_narrow_an_archive(self, source_tree: Path, tmp_path: Path) -> None: + """Test that repeated --include and --exclude reach the archive.""" + output = tmp_path / "o.tar" + + invoke( + "compress", + source_tree, + "-o", + output, + "--include", + "f0.bin", + "--include", + "f1.bin", + "--exclude", + "f1.bin", + "-q", + ) + + with tarfile.open(output) as tar: + assert sorted(tar.getnames()) == ["tree", "tree/f0.bin"] + + @pytest.mark.parametrize("flag", ["--include", "--exclude"]) + def test_filters_are_refused_for_a_single_file( + self, payload: Path, flag: str + ) -> None: + """Test that a glob has nothing to select in a one-file container.""" + result = invoke("compress", payload, flag, "x", expect=EXIT_USAGE) + + assert "apply only to archives" in result.output + + def test_unknown_format_is_refused(self, payload: Path) -> None: + """Test that an unrecognised -F names neither container.""" + invoke("compress", payload, "-F", "bogus", expect=EXIT_USAGE) diff --git a/tests/python/cli/test_extract.py b/tests/python/cli/test_extract.py new file mode 100644 index 0000000..ceb9dcf --- /dev/null +++ b/tests/python/cli/test_extract.py @@ -0,0 +1,600 @@ +"""Tests for `compresso extract`.""" + +from __future__ import annotations + +import gzip +import os +import sys +import tarfile +from collections.abc import Callable +from pathlib import Path + +import pytest + +import compresso.cli.extract as extract_cmd +from compresso.cli._render import ( + EXIT_CANCELLED, + EXIT_FAILED, + EXIT_OK, + EXIT_USAGE, +) +from compresso.frontend.dispatch import ExtractRequest + +from ..helpers import file_entry, link_entry, make_tar, node_entry, renamed, umask +from .conftest import CONFLICT_MODES, invoke + + +class TestArchiveRoundTrip: + """Test archive and extract, end to end.""" + + @pytest.mark.parametrize("fmt", ["tar", "tar.gz", "tar.zst", "zip"]) + def test_archive_then_extract( + self, source_tree: Path, tmp_path: Path, fmt: str + ) -> None: + """Test that a tree survives a round trip through each archive format.""" + archive = tmp_path / f"out.{fmt}" + dest = tmp_path / f"dest.{fmt}" + + invoke("compress", source_tree, "-o", archive, "-F", fmt, "-q") + invoke("extract", archive, "-C", dest, "-q") + + for original in source_tree.iterdir(): + restored = dest / source_tree.name / original.name + assert restored.read_bytes() == original.read_bytes() + + @pytest.mark.parametrize( + "flags,expect,renames", + [ + ((), EXIT_OK, True), + (("--on-conflict", "error"), EXIT_FAILED, False), + (("--on-conflict", "skip"), EXIT_OK, False), + (("--on-conflict", "overwrite"), EXIT_OK, False), + (("-f",), EXIT_OK, False), + ], + ids=["rename", "error", "skip", "overwrite", "force"], + ) + def test_extract_over_an_existing_tree( + self, + source_tree: Path, + tmp_path: Path, + make_archive: Callable[[str], Path], + flags: tuple[str, ...], + expect: int, + renames: bool, + ) -> None: + """Test that a repeat extraction renames the clashing top-level directory + by default, and follows the overwrite mode otherwise.""" + archive = make_archive("tar") + dest = tmp_path / "dest" + + invoke("extract", archive, "-C", dest, "-q") + invoke("extract", archive, "-C", dest, *flags, "-q", expect=expect) + + top_level = [source_tree.name, *([renamed(source_tree.name)] * renames)] + assert sorted(p.name for p in dest.iterdir()) == sorted(top_level) + # The whole directory is renamed, never a file inside it + names = sorted(p.name for p in source_tree.iterdir()) + for top in top_level: + assert sorted(p.name for p in (dest / top).iterdir()) == names + + @pytest.mark.skipif( + sys.platform == "win32", reason="Windows has no POSIX permission bits" + ) + @pytest.mark.parametrize("exact", [False, True], ids=["umask", "same-perms"]) + def test_same_permissions_skips_the_umask( + self, tmp_path: Path, exact: bool + ) -> None: + """Test that -p restores an entry's mode without the umask applied.""" + archive = tmp_path / "modes.tar" + make_tar(archive, file_entry("a.txt", mode=0o777)) + dest = tmp_path / "dest" + + invoke("extract", archive, "-C", dest, *(["-p"] if exact else []), "-q") + + expected = 0o777 if exact else 0o777 & ~umask() + assert (dest / "a.txt").stat().st_mode & 0o7777 == expected + + def test_symlinks_flag_reaches_the_policy(self, tmp_path: Path) -> None: + """Test that a symlink refused by default extracts with `--symlinks file`.""" + archive = tmp_path / "link.tar" + make_tar(archive, link_entry("link", "target", kind=tarfile.SYMTYPE)) + dest = tmp_path / "dest" + + invoke("extract", archive, "-C", dest, "-q", expect=EXIT_FAILED) + invoke("extract", archive, "-C", dest, "--symlinks", "file", "-q") + + assert (dest / "link").read_bytes() == b"target" + + +class TestDecompressConflicts: + """Test what decompressing one file does when its output already exists.""" + + @CONFLICT_MODES + def test_existing_output( + self, + payload: Path, + tmp_path: Path, + flags: tuple[str, ...], + expect: int, + kept: bool, + renames: bool, + ) -> None: + """Test that each overwrite mode keeps, replaces or renames around the + output, and the summary line says which.""" + archive = tmp_path / "out.comp" + restored = tmp_path / "restored.bin" + invoke("compress", payload, "-o", archive, "-q") + restored.write_bytes(b"stale") + + result = invoke("extract", archive, "-o", restored, *flags, expect=expect) + + sibling = tmp_path / renamed(restored.name) + assert (f"output renamed to {sibling}" in result.output) == renames + assert (f"{restored} skipped" in result.output) == ("skip" in flags) + assert restored.read_bytes() == (b"stale" if kept else payload.read_bytes()) + assert sibling.exists() == renames + + +class TestDecompressName: + """Test that -N names a .gz file's output as its header does, as gzip -N does.""" + + @pytest.mark.parametrize( + ("flags", "expected"), [((), "renamed"), (("-N",), "orig.txt")] + ) + def test_name_flag( + self, tmp_path: Path, flags: tuple[str, ...], expected: str + ) -> None: + """Test that the stored name is used only with -N.""" + src = tmp_path / "renamed.gz" + with src.open("wb") as f, gzip.GzipFile("orig.txt", "wb", fileobj=f) as gz: + gz.write(b"payload") + + invoke("extract", src, *flags, "-q") + + assert (tmp_path / expected).read_bytes() == b"payload" + + +class TestSmartExtract: + """Test `extract` runs the jobs dispatch plans; the rules are in frontend/test_dispatch.py.""" + + def test_several_inputs_in_one_go(self, payload: Path, tmp_path: Path) -> None: + """Test that each input is dispatched on its own.""" + first = tmp_path / "a.gz" + second = tmp_path / "b.zst" + invoke("compress", payload, "-o", first, "-q") + invoke("compress", payload, "-o", second, "-q") + + invoke("extract", first, second, "-q") + + assert (tmp_path / "a").read_bytes() == payload.read_bytes() + assert (tmp_path / "b").read_bytes() == payload.read_bytes() + + # Unrecognised fails while planning, truncated while running; random bytes + # named .gz would be tried as gzip, so the unrecognised one has no codec suffix + @pytest.mark.parametrize("bad", ["unrecognised", "truncated"]) + def test_a_failed_input_does_not_stop_the_rest( + self, + payload: Path, + source_tree: Path, + tmp_path: Path, + make_archive: Callable[[str], Path], + bad: str, + ) -> None: + """Test that the inputs either side of a bad one are still extracted, and + the bad one is named and exits 1.""" + good = tmp_path / "good.gz" + good.write_bytes(gzip.compress(payload.read_bytes())) + + if bad == "unrecognised": + broken = tmp_path / "bad.bin" + broken.write_bytes(payload.read_bytes()) + else: + broken = tmp_path / "bad.gz" + broken.write_bytes(good.read_bytes()[: good.stat().st_size // 2]) + + dest = tmp_path / "dest" + dest.mkdir() + + result = invoke( + "extract", + good, + broken, + make_archive("zip"), + "-C", + dest, + "-q", + expect=EXIT_FAILED, + ) + + assert str(broken) in result.output + assert (dest / "good").read_bytes() == payload.read_bytes() + assert not (dest / "bad").exists() + for original in source_tree.iterdir(): + restored = dest / source_tree.name / original.name + assert restored.read_bytes() == original.read_bytes() + + +class TestOutputFlags: + """Test that -o names one file and -C the directory for everything else.""" + + def test_output_with_directory(self, payload: Path, tmp_path: Path) -> None: + """Test that -o contradicts -C.""" + packed = tmp_path / "a.gz" + packed.write_bytes(gzip.compress(payload.read_bytes())) + + result = invoke( + "extract", packed, "-o", tmp_path / "x", "-C", tmp_path, expect=EXIT_USAGE + ) + assert "-o contradicts -C" in result.output + + def test_output_with_several_inputs(self, payload: Path, tmp_path: Path) -> None: + """Test that -o refuses more than one input, pointing to -C.""" + inputs = [tmp_path / "a.gz", tmp_path / "b.gz"] + for packed in inputs: + packed.write_bytes(gzip.compress(payload.read_bytes())) + + result = invoke("extract", *inputs, "-o", tmp_path / "x", expect=EXIT_USAGE) + assert "use -C" in result.output + + def test_output_on_an_archive( + self, tmp_path: Path, make_archive: Callable[[str], Path] + ) -> None: + """Test that -o on an input that turns out to be an archive fails it.""" + archive = make_archive("zip") + out = tmp_path / "out" + + result = invoke("extract", archive, "-o", out, expect=EXIT_FAILED) + assert f"{archive}: -o names one file; use -C for an archive" in result.output + assert not out.exists() + + +class TestEntry: + """Test that --entry extracts only the entries named.""" + + @pytest.mark.parametrize("fmt", ["tar", "zip"]) + def test_only_the_chosen_entries( + self, + source_tree: Path, + tmp_path: Path, + make_archive: Callable[[str], Path], + fmt: str, + ) -> None: + """Test that the named entries are written and nothing else.""" + archive = make_archive(fmt) + dest = tmp_path / "dest" + top = source_tree.name + + invoke( + "extract", + archive, + "-C", + dest, + "--entry", + f"{top}/f0.bin", + "-e", + f"{top}/f2.bin", + "-q", + ) + + assert sorted(p.name for p in (dest / top).iterdir()) == ["f0.bin", "f2.bin"] + for name in ("f0.bin", "f2.bin"): + assert (dest / top / name).read_bytes() == (source_tree / name).read_bytes() + + def test_a_missing_entry_fails_after_the_rest( + self, source_tree: Path, tmp_path: Path, make_archive: Callable[[str], Path] + ) -> None: + """Test that a name matching nothing exits 1, naming it, once the rest is + written.""" + dest = tmp_path / "dest" + top = source_tree.name + + result = invoke( + "extract", + make_archive("tar"), + "-C", + dest, + "--entry", + f"{top}/f1.bin", + "-e", + "nope", + expect=EXIT_FAILED, + ) + + assert "nope" in result.output + assert sorted(p.name for p in (dest / top).iterdir()) == ["f1.bin"] + + def test_a_single_file_input_fails(self, payload: Path, tmp_path: Path) -> None: + """Test that --entry on an input that isn't an archive fails it.""" + packed = tmp_path / "a.gz" + packed.write_bytes(gzip.compress(payload.read_bytes())) + + result = invoke("extract", packed, "--entry", "x", expect=EXIT_FAILED) + + assert ( + f"{packed}: --entry, --include and --exclude apply only to archives" + in result.output + ) + assert not (tmp_path / "a").exists() + + +class TestFilter: + """Test that --include and --exclude select entries.""" + + def test_include_adds_to_entry_and_exclude_overrides( + self, source_tree: Path, tmp_path: Path, make_archive: Callable[[str], Path] + ) -> None: + """Test bsdtar's union, with the exclude winning over both.""" + dest = tmp_path / "dest" + top = source_tree.name + + invoke( + "extract", + make_archive("tar"), + "-C", + dest, + "-e", + f"{top}/f0.bin", + "--include", + "f[12].bin", + "--exclude", + "f2.bin", + "-q", + ) + + assert sorted(p.name for p in (dest / top).iterdir()) == ["f0.bin", "f1.bin"] + + @pytest.mark.parametrize("flags", [(), ("-e", "tree/f0.bin")]) + def test_excluding_everything_warns_and_succeeds( + self, + tmp_path: Path, + make_archive: Callable[[str], Path], + flags: tuple[str, ...], + ) -> None: + """Test that a filter leaving nothing exits 0 as tar does, but says so.""" + archive = make_archive("tar") + + result = invoke( + "extract", archive, "-C", tmp_path / "d", *flags, "--exclude", "*" + ) + + assert f"{archive}: nothing left to extract" in result.output + assert not list((tmp_path / "d").iterdir()) + + def test_quiet_silences_the_warning( + self, make_archive: Callable[[str], Path] + ) -> None: + """Test that -q suppresses even the warning.""" + result = invoke("extract", make_archive("tar"), "--exclude", "*", "-q") + + assert result.output == "" + + def test_a_partly_excluded_run_does_not_warn( + self, tmp_path: Path, make_archive: Callable[[str], Path] + ) -> None: + """Test that the warning needs every entry gone.""" + result = invoke( + "extract", make_archive("tar"), "-C", tmp_path / "d", "--exclude", "f0.bin" + ) + + assert "nothing left" not in result.output + + @pytest.mark.parametrize("flag", ["--include", "--exclude"]) + def test_a_single_file_input_fails( + self, payload: Path, tmp_path: Path, flag: str + ) -> None: + """Test that a glob on an input that isn't an archive fails it.""" + packed = tmp_path / "a.gz" + packed.write_bytes(gzip.compress(payload.read_bytes())) + + result = invoke("extract", packed, flag, "x", expect=EXIT_FAILED) + + assert "apply only to archives" in result.output + assert not (tmp_path / "a").exists() + + +class TestExtractOptions: + """Test the flags that set `ExtractOptions`, named as tar names them.""" + + @pytest.mark.parametrize( + "flags,field,value", + [ + (("--no-same-permissions",), "preserve_permissions", False), + (("-m",), "preserve_timestamps", False), + (("--overwrite-dir",), "overwrite_dir_metadata", True), + (("--max-depth", "5"), "max_depth", 5), + (("--max-depth", "0"), "max_depth", 0), + (("--max-size", "500M"), "max_total_size", 500 * 1024**2), + (("--max-size", "10g"), "max_total_size", 10 * 1024**3), + (("--max-size", "1KB"), "max_total_size", 1024), + (("--max-size", "2048"), "max_total_size", 2048), + (("--max-size", "0"), "max_total_size", 0), + ], + ) + def test_flag_reaches_its_field( + self, + payload: Path, + monkeypatch: pytest.MonkeyPatch, + flags: tuple[str, ...], + field: str, + value: object, + ) -> None: + """Test that each flag sets its field, recorded before anything runs.""" + seen: list[ExtractRequest] = [] + + def record(request: ExtractRequest) -> None: + seen.append(request) + raise KeyboardInterrupt + + monkeypatch.setattr(extract_cmd, "plan_extract", record) + + invoke("extract", payload, *flags, expect=EXIT_CANCELLED) + + assert getattr(seen[0].options, field) == value + + @pytest.mark.parametrize("size", ["5X", "-1", "1.5G", "M", "10MBB"]) + def test_bad_max_size_is_usage(self, payload: Path, size: str) -> None: + """Test that a size that doesn't parse is refused before any work.""" + invoke("extract", payload, "--max-size", size, expect=EXIT_USAGE) + + def test_same_permissions_contradicts_no_same_permissions( + self, payload: Path + ) -> None: + """Test that -p with --no-same-permissions exits 2.""" + result = invoke( + "extract", payload, "-p", "--no-same-permissions", expect=EXIT_USAGE + ) + assert "-p contradicts --no-same-permissions" in result.output + + @pytest.mark.parametrize("touch", [False, True], ids=["mtime", "touch"]) + def test_touch_skips_the_stored_time(self, tmp_path: Path, touch: bool) -> None: + """Test that -m leaves an entry with the time it was extracted.""" + archive = tmp_path / "old.tar" + make_tar(archive, file_entry("a.txt", mtime=1_000_000)) + dest = tmp_path / "dest" + + invoke("extract", archive, "-C", dest, *(["-m"] if touch else []), "-q") + + assert ((dest / "a.txt").stat().st_mtime == 1_000_000) != touch + + @pytest.mark.parametrize("depth,expect", [("2", EXIT_FAILED), ("3", EXIT_OK)]) + def test_max_depth_bounds_entry_paths( + self, tmp_path: Path, depth: str, expect: int + ) -> None: + """Test that an entry nested deeper than --max-depth fails the archive.""" + archive = tmp_path / "deep.tar" + make_tar(archive, file_entry("a/b/c.txt")) + dest = tmp_path / "dest" + + invoke("extract", archive, "-C", dest, "--max-depth", depth, expect=expect) + + assert (dest / "a/b/c.txt").exists() == (expect == EXIT_OK) + + def test_missing_directory_is_created( + self, payload: Path, tmp_path: Path, make_archive: Callable[[str], Path] + ) -> None: + """Test that a `-C` that doesn't exist is made for a single file, as it + is for an archive.""" + packed = tmp_path / "a.gz" + packed.write_bytes(gzip.compress(payload.read_bytes())) + dest = tmp_path / "new" / "dir" + + invoke("extract", packed, make_archive("zip"), "-C", dest, "-q") + + assert (dest / "a").read_bytes() == payload.read_bytes() + assert (dest / "tree").is_dir() + + +class TestRemoveInput: + """Test what `--rm` and `-k` do to the input.""" + + @pytest.mark.parametrize( + "flags,removes", + [((), False), (("--rm",), True), (("--rm", "-k"), False)], + ids=["default", "rm", "rm-then-keep"], + ) + @pytest.mark.parametrize("kind", ["file", "archive"]) + def test_flags( + self, + tmp_path: Path, + make_archive: Callable[[str], Path], + kind: str, + flags: tuple[str, ...], + removes: bool, + ) -> None: + """Test that a fully unpacked input goes only with --rm.""" + if kind == "file": + source = tmp_path / "a.gz" + source.write_bytes(gzip.compress(b"data")) + else: + source = make_archive("tar") + dest = tmp_path / "dest" + + invoke("extract", source, "-C", dest, *flags, "-q") + + assert source.exists() != removes + assert any(dest.iterdir()) + + def test_a_failed_decompression_keeps_the_input(self, tmp_path: Path) -> None: + """Test that a corrupt single-file input isn't deleted.""" + source = tmp_path / "a.gz" + source.write_bytes(gzip.compress(b"data")[:-4]) + + invoke( + "extract", source, "-C", tmp_path / "out", "--rm", "-q", expect=EXIT_FAILED + ) + + assert source.exists() + + @pytest.mark.skipif( + sys.platform == "win32", + reason="Windows can't replace the archive while libarchive has it open", + ) + def test_entry_written_over_the_archive_stays(self, tmp_path: Path) -> None: + """Test that --rm doesn't delete what replaced the archive at its path.""" + archive = tmp_path / "self.tar" + make_tar(archive, file_entry("self.tar", size=3)) + + invoke("extract", archive, "-f", "--rm", "-q") + + assert archive.stat().st_size == 3 + + @pytest.mark.parametrize( + "flags", + [ + ("-e", "x"), + ("--include", "x"), + ("--exclude", "x"), + ("--on-conflict", "skip"), + ], + ids=["entry", "include", "exclude", "skip"], + ) + def test_partial_extraction_is_refused( + self, make_archive: Callable[[str], Path], flags: tuple[str, ...] + ) -> None: + """Test that --rm can't take entries that were never extracted with it.""" + archive = make_archive("tar") + + result = invoke("extract", archive, *flags, "--rm", expect=EXIT_USAGE) + + assert "--rm contradicts" in result.output + assert archive.exists() + + @pytest.mark.parametrize("kind", ["archive", "single"]) + def test_undeletable_input_follows_the_summary( + self, + make_archive: Callable[[str], Path], + locked: Path, + tmp_path: Path, + kind: str, + ) -> None: + """Test that an input that won't go is a warning after a normal summary.""" + if kind == "archive": + source = make_archive("tar").rename(locked / "a.tar") + else: + source = locked / "a.gz" + source.write_bytes(gzip.compress(b"a")) + locked.chmod(0o555) + + result = invoke( + "extract", source, "-C", tmp_path / "unpacked", "--rm", expect=EXIT_FAILED + ) + + assert any((tmp_path / "unpacked").iterdir()) + assert source.exists() + warning = f"! {source}: not removed: Permission denied" + assert result.output.index("→") < result.output.index(warning) + + @pytest.mark.skipif( + sys.platform == "win32" or os.geteuid() == 0, reason="needs POSIX, not root" + ) + def test_uncreatable_node_keeps_the_archive(self, tmp_path: Path) -> None: + """Test that a device fails the input, so --rm can't take it with the archive.""" + archive = tmp_path / "nodes.tar" + make_tar(archive, file_entry("a"), node_entry("char", tarfile.CHRTYPE)) + out = tmp_path / "out" + + result = invoke("extract", archive, "-C", out, "--rm", expect=EXIT_FAILED) + + assert "Skipped, the rest was extracted: char" in result.output + assert (out / "a").is_file() + assert archive.exists() diff --git a/tests/python/cli/test_formats.py b/tests/python/cli/test_formats.py new file mode 100644 index 0000000..319b248 --- /dev/null +++ b/tests/python/cli/test_formats.py @@ -0,0 +1,54 @@ +"""Tests for `compresso formats`.""" + +from __future__ import annotations + +import subprocess +import sys + +from compresso.cli._render import ( + EXIT_OK, +) + +from .conftest import invoke + + +def section(output: str, name: str) -> str: + """Return one `- name` block of the output, up to the blank line after it.""" + return output[output.index(f"- {name}\n") :].split("\n\n")[0] + + +class TestFormats: + """Test the read-only `formats` command.""" + + def test_runs_as_a_module(self) -> None: + """Test that `python -m compresso.cli` works, which a package needs __main__ for.""" + result = subprocess.run( + [sys.executable, "-m", "compresso.cli", "formats"], + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == EXIT_OK + assert "zstd" in result.stdout + + def test_formats_shows_level_ranges(self) -> None: + """Test that formats shows each backend's levels, and none for snappy.""" + output = invoke("formats").output + + assert "Levels: 1-22" in section(output, "zstd") + assert "Levels: none" in section(output, "snappy") + + def test_formats_shows_standalone_and_archive_formats(self) -> None: + """Test that each standalone format shows its aliases and extensions, and + tar its compressed variants, named as an output would be.""" + output = invoke("formats").output + standalone = output[output.index("Standalone formats") :] + + assert "Aliases: zst" in section(standalone, "zstd") + assert "Extensions: .zst, .zstd" in section(standalone, "zstd") + assert "Aliases: none" in section(standalone, "lz4") + assert "tar.gz, tar.bz2, tar.xz, tar.zst, tar.lz4, tar.sz" in section( + output, "tar" + ) + assert section(output, "zip") == "- zip" diff --git a/tests/python/cli/test_info.py b/tests/python/cli/test_info.py new file mode 100644 index 0000000..6951410 --- /dev/null +++ b/tests/python/cli/test_info.py @@ -0,0 +1,129 @@ +"""Tests for `compresso info`.""" + +from __future__ import annotations + +import gzip +import time +from collections.abc import Callable +from pathlib import Path + +import orjson +import pytest + +from compresso.cli._render import EXIT_FAILED + +from .conftest import invoke + + +class TestInfo: + """Test the read-only `info` command.""" + + def test_info_reports_a_valid_file(self, payload: Path, tmp_path: Path) -> None: + """Test that info describes a file the CLI just wrote.""" + archive = tmp_path / "out.comp" + invoke("compress", payload, "-o", archive, "-q") + + result = invoke("info", archive) + + assert "Valid Compresso file" in result.output + + def test_info_archive_text(self, make_archive: Callable[[str], Path]) -> None: + """Test that an archive shows a summary, not its entries.""" + summary = invoke("info", make_archive("zip")).output + + assert "Entries: 4" in summary + assert "f0.bin" not in summary + + def test_standalone_codec_shows_format_and_size( + self, payload: Path, tmp_path: Path + ) -> None: + """Test that a .gz is described by its format, not refused as a bad .comp.""" + packed = tmp_path / "p.gz" + invoke("compress", payload, "-o", packed, "-q") + + output = invoke("info", packed).output + + assert "Format: gzip" in output + assert "Size:" in output + + def test_gzip_shows_its_stored_name_and_mtime(self, tmp_path: Path) -> None: + """Test that a gzip's header detail is shown, and is in the JSON.""" + packed = tmp_path / "p.gz" + with ( + packed.open("wb") as raw, + gzip.GzipFile("stored.txt", "wb", fileobj=raw, mtime=86400 * 365) as f, + ): + f.write(b"data") + + output = invoke("info", packed).output + item = orjson.loads(invoke("info", packed, "--json").output)[0] + + assert "Stored name: stored.txt" in output + assert ( + f"Modified: {time.strftime('%Y-%m-%d', time.localtime(item['mtime']))}" + in output + ) + assert "Original size" not in output + assert item["stored_name"] == "stored.txt" + assert item["mtime"] == 86400 * 365 + assert item["orig_size"] is None + + def test_gzip_name_need_not_be_utf8(self, tmp_path: Path) -> None: + """Test that a name of raw bytes can still be printed and put in JSON.""" + packed = tmp_path / "p.gz" + flagged = bytearray(gzip.compress(b"data", mtime=0)) + flagged[3] |= 0x08 # FNAME + packed.write_bytes(flagged[:10] + b"caf\xe9.txt\0" + flagged[10:]) + + items = orjson.loads(invoke("info", packed, "--json").output) + + assert items[0]["stored_name"] == "caf\ufffd.txt" + + @pytest.mark.parametrize("suffix", [".xz", ".zst", ".lz4", ".sz"]) + def test_stated_original_size_is_shown( + self, payload: Path, tmp_path: Path, suffix: str + ) -> None: + """Test that the size comes from the file, for the ratio to use too.""" + packed = tmp_path / f"p{suffix}" + invoke("compress", payload, "-o", packed, "-q") + + output = invoke("info", packed).output + item = orjson.loads(invoke("info", packed, "--json").output)[0] + + assert "Original size:" in output + assert "Compression:" in output + assert item["orig_size"] == payload.stat().st_size + assert item["stored_name"] is None + + def test_several_inputs_carry_on_past_an_unrecognised_one( + self, payload: Path, tmp_path: Path, make_archive: Callable[[str], Path] + ) -> None: + """Test that each described input gets a header, and an unrecognised one + fails alone, exiting 1 once the rest are described.""" + archive = make_archive("tar") + packed = tmp_path / "p.gz" + invoke("compress", payload, "-o", packed, "-q") + + result = invoke("info", archive, payload, packed, expect=EXIT_FAILED) + + assert f"{payload}: not a recognised archive" in result.output + assert result.stdout.startswith(f"==> {archive} <==\n") + assert f"\n\n==> {packed} <==\n" in result.stdout + assert f"==> {payload} <==" not in result.stdout + + def test_json_is_an_array_per_input( + self, payload: Path, tmp_path: Path, make_archive: Callable[[str], Path] + ) -> None: + """Test that --json is one array item per input, whatever its kind.""" + comp, packed = tmp_path / "out.comp", tmp_path / "p.gz" + invoke("compress", payload, "-o", comp, "-q") + invoke("compress", payload, "-o", packed, "-q") + + items = orjson.loads( + invoke("info", comp, make_archive("zip"), packed, "--json").output + ) + + assert [item["format"] for item in items] == ["compresso", "zip", "gzip"] + assert items[0]["is_compresso"] is True + assert items[1]["entry_count"] == 4 + assert items[2]["size"] == packed.stat().st_size diff --git a/tests/python/cli/test_list.py b/tests/python/cli/test_list.py new file mode 100644 index 0000000..16ab252 --- /dev/null +++ b/tests/python/cli/test_list.py @@ -0,0 +1,95 @@ +"""Tests for `compresso list`.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import orjson +import pytest + +from compresso.cli._render import EXIT_FAILED, format_size +from compresso.cli._render.info import list_entries +from compresso.frontend.archive_api import ArchiveEntry + +from .conftest import TREE_FILE_SIZE, invoke + + +class TestList: + """Test the read-only `list` command.""" + + def test_list_shows_sizes_and_paths( + self, make_archive: Callable[[str], Path] + ) -> None: + """Test that list prints each entry's size and path.""" + lines = invoke("list", make_archive("tar")).output.splitlines() + + f0_line = next(line for line in lines if "f0.bin" in line) + assert f0_line.split()[:2] == format_size(TREE_FILE_SIZE).split() + + # Directories have a blank size; their contents are indented beneath them + dir_line = next(line for line in lines if line.rstrip("/").endswith("tree")) + assert dir_line == f"{'':>10} {dir_line.strip()}" + assert f0_line.endswith(" tree/f0.bin") + + def test_list_indents_by_depth(self, capsys: pytest.CaptureFixture[str]) -> None: + """Test that list indents only under directories it has listed.""" + entries = [ + ArchiveEntry(path="a/", is_dir=True), + ArchiveEntry(path="a/b/", is_dir=True), + ArchiveEntry(path="a/b/c.txt", size=1024), + ArchiveEntry(path="a/d.txt", size=10), + ArchiveEntry(path="a/e", size=5, is_symlink=True, link_target="d.txt"), + ArchiveEntry(path="x/y.txt", size=0), # no "x/" entry was listed + ] + + list_entries(entries) + + assert capsys.readouterr().out.splitlines() == [ + " " * 12 + "a/", + " " * 14 + "a/b/", + " 1.00 KB" + " " * 6 + "a/b/c.txt", + " 10.00 B" + " " * 4 + "a/d.txt", + " " * 14 + "a/e -> d.txt", + " 0.00 B" + " " * 2 + "x/y.txt", + ] + + def test_long_shows_mode_and_zip_detail( + self, make_archive: Callable[[str], Path] + ) -> None: + """Test that -l shows each entry's mode, and a zip entry's CRC.""" + lines = invoke("list", "-l", make_archive("zip")).output.splitlines() + + f0_line = next(line for line in lines if "f0.bin" in line) + assert f0_line.startswith("-rw") + assert ", crc " in f0_line + + def test_several_inputs_carry_on_past_a_single_file( + self, payload: Path, tmp_path: Path, make_archive: Callable[[str], Path] + ) -> None: + """Test that each archive gets a header, and a single-file input fails + alone, exiting 1 once both archives are listed.""" + tar, zip_ = make_archive("tar"), make_archive("zip") + packed = tmp_path / "p.gz" + invoke("compress", payload, "-o", packed, "-q") + + result = invoke("list", tar, packed, zip_, expect=EXIT_FAILED) + + assert f"{packed}: holds a single file" in result.output + assert result.stdout.startswith(f"==> {tar} <==\n") + assert f"\n\n==> {zip_} <==\n" in result.stdout + assert f"==> {packed} <==" not in result.stdout + + def test_json_is_an_array_per_archive( + self, make_archive: Callable[[str], Path] + ) -> None: + """Test that --json is an array even for one input, with every entry field.""" + archive = make_archive("zip") + + (listing,) = orjson.loads(invoke("list", archive, "--json").output) + + assert listing["path"] == str(archive) + assert listing["format"] == "zip" + f0 = next(e for e in listing["entries"] if e["path"].endswith("f0.bin")) + assert f0["size"] == TREE_FILE_SIZE + assert f0["crc"] is not None diff --git a/tests/python/cli/test_verify.py b/tests/python/cli/test_verify.py new file mode 100644 index 0000000..c586837 --- /dev/null +++ b/tests/python/cli/test_verify.py @@ -0,0 +1,62 @@ +"""Tests for `compresso test`.""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +import pytest + +from compresso.cli._render import EXIT_FAILED + +from ..helpers import flip_middle_byte +from .conftest import invoke + + +@pytest.fixture +def packed(payload: Path, tmp_path: Path) -> Path: + """`payload` as a gzip file.""" + path = tmp_path / "p.gz" + invoke("compress", payload, "-o", path, "-q") + + return path + + +class TestVerify: + """Test the `test` command.""" + + def test_good_input_says_ok(self, packed: Path) -> None: + """Test that each input gets one line naming it.""" + assert f"OK: {packed}" in invoke("test", packed).output + + def test_quiet_prints_nothing(self, packed: Path) -> None: + """Test that -q leaves only the exit code.""" + assert invoke("test", "-q", packed).output == "" + + def test_damage_fails_and_the_rest_still_run( + self, packed: Path, tmp_path: Path + ) -> None: + """Test that a bad input is named, and the next input is still checked.""" + bad = tmp_path / "bad.gz" + shutil.copy(packed, bad) + flip_middle_byte(bad) + + output = invoke("test", bad, packed, expect=EXIT_FAILED).output + + assert "Test failed" in output + assert f"OK: {packed}" in output + + def test_trailing_data_fails(self, packed: Path) -> None: + """Test that it fails here, as with `gzip -t`, though `extract` only warns.""" + packed.write_bytes(packed.read_bytes() + b"not gzip data") + + output = invoke("test", packed, expect=EXIT_FAILED).output + + assert "trailing data" in output + + def test_unrecognised_file_fails(self, tmp_path: Path) -> None: + """Test that a file of no known format is an input failure.""" + notes = tmp_path / "notes.txt" + notes.write_text("just some text") + + assert "not a recognised" in invoke("test", notes, expect=EXIT_FAILED).output diff --git a/tests/python/conftest.py b/tests/python/conftest.py index 0623afa..0ae5987 100644 --- a/tests/python/conftest.py +++ b/tests/python/conftest.py @@ -2,6 +2,8 @@ import importlib import os +import sys +from collections.abc import Generator from pathlib import Path import pytest @@ -47,6 +49,18 @@ def needs_symlinks(tmp_path: Path) -> None: probe.unlink() +@pytest.fixture +def locked(tmp_path: Path) -> Generator[Path]: + """A directory whose files can't be deleted once a test makes it read-only.""" + if sys.platform == "win32" or os.geteuid() == 0: + pytest.skip("needs POSIX, not root") + + root = tmp_path / "locked" + root.mkdir() + yield root + root.chmod(0o755) + + @pytest.fixture def sample_text_file(tmp_path: Path) -> Path: """A small, compressible text file.""" diff --git a/tests/python/frontend/test_api.py b/tests/python/frontend/test_api.py index 4d1f876..1397b48 100644 --- a/tests/python/frontend/test_api.py +++ b/tests/python/frontend/test_api.py @@ -11,6 +11,7 @@ CompressionJob, CompressionOptions, DecompressionJob, + Strategy, plan_compression, plan_decompression, ) @@ -160,8 +161,15 @@ class TestPlanCompressionLevels: "opts,match", [ # No algo: the backend the strategy picks is the one checked - (CompressionOptions(strategy="fast", level=13), "lz4 compression level"), - (CompressionOptions(strategy="fsat"), "Unknown strategy: fsat"), + ( + CompressionOptions(strategy=Strategy.FAST, level=13), + "lz4 compression level", + ), + ( + CompressionOptions(strategy="fsat"), # ty: ignore[invalid-argument-type] - the point of the test + "Unknown strategy: fsat", + ), + (CompressionOptions(algo="zsdt"), "Unknown compression algorithm: zsdt"), ], ) def test_out_of_range_level_is_unrunnable( diff --git a/tests/python/frontend/test_archive_api.py b/tests/python/frontend/test_archive_api.py index 595aa13..3a87a41 100644 --- a/tests/python/frontend/test_archive_api.py +++ b/tests/python/frontend/test_archive_api.py @@ -36,10 +36,33 @@ def test_archive_options_default(self) -> None: """Test creating ArchiveOptions with defaults.""" opts = ArchiveOptions() - assert opts.format == "tar.zst" + assert opts.format is None assert opts.compression_level is None assert opts.overwrite is OverwriteMode.RENAME + @pytest.mark.parametrize( + ("name", "fmt", "expected"), + [ + ("out.tar.zst", None, "tar.zst"), + ("out.bin", None, "zip"), + ("out.tar.zst", "tar", "tar"), + ], + ids=["suffix", "unrecognised-suffix", "explicit-wins"], + ) + def test_format_comes_from_the_output_name( + self, + small_file: Path, + tmp_path: Path, + name: str, + fmt: str | None, + expected: str, + ) -> None: + """Test that an unset format is read from the output's suffix, else zip.""" + plan = plan_archive([small_file], tmp_path / name, ArchiveOptions(format=fmt)) + + assert plan.can_run, plan.reason_if_unavailable + assert plan.options.format == expected + def test_plan_archive_valid(self, sample_text_file: Path, tmp_path: Path) -> None: """Test that a plan over existing sources can run and sums input size.""" out = tmp_path / "out.tar.zst" @@ -239,6 +262,23 @@ def test_output_dir_default_ignores_a_relative_archive_path( assert plan.output_dir == tmp_path.resolve() + def test_files_select_the_planned_entries(self, tmp_path: Path) -> None: + """Test that the plan holds only what `files` selects, as the extractor + matches it: an entry, or a directory and everything below it.""" + archive = tmp_path / "a.tar" + make_tar( + archive, + dir_entry("d/"), + file_entry("d/x"), + file_entry("dx"), + file_entry("y"), + file_entry("z"), + ) + + plan = ExtractJob.from_archive(archive, files=["d/", "y", "nope"]).plan + + assert [e.path for e in plan.entries] == ["d/", "d/x", "y"] + class TestArchiveRoundTrip: """Test end-to-end archive -> extract round-trip.""" @@ -257,7 +297,6 @@ def test_round_trip(self, tmp_path: Path, fmt: str) -> None: assert result.ok, result.error extract_job = ExtractJob.from_archive(archive_path, tmp_path / "sortie") - assert all(isinstance(e, ArchiveEntry) for e in extract_job.list_contents()) result = extract_job.run() assert result.ok, result.error diff --git a/tests/python/frontend/test_dispatch.py b/tests/python/frontend/test_dispatch.py new file mode 100644 index 0000000..9f4ff50 --- /dev/null +++ b/tests/python/frontend/test_dispatch.py @@ -0,0 +1,340 @@ +"""Tests for the frontend dispatch rules.""" + +import gzip +import os +import zipfile +from pathlib import Path + +import pytest + +from compresso.frontend._formats import infer_format +from compresso.frontend.api import CompressionJob, DecompressionJob +from compresso.frontend.archive_api import ArchiveJob, ExtractJob +from compresso.frontend.dispatch import ( + CompressRequest, + ExtractRequest, + default_archive_output, + format_is_single_file, + is_archive_format, + looks_like_archive, + plan_compress, + plan_extract, +) + + +@pytest.fixture +def tree(tmp_path: Path) -> Path: + """A directory holding one file.""" + root = tmp_path / "tree" + root.mkdir() + (root / "a.txt").write_text("a") + + return root + + +class TestInferFormat: + """Test reading a format off an output filename.""" + + @pytest.mark.parametrize( + ("name", "expected"), + [ + ("x.tar.lz4", "tar.lz4"), + ("x.TAR.ZST", "tar.zst"), + ("x.tar", "tar"), + ("x.zip", "zip"), + ("x.7z", "7z"), + ("x.gz", "gz"), + ("x.zst", "zst"), + ("x.COMP", "comp"), + ("x.txt", None), + ("x", None), + ("x.foo.lz4", "lz4"), + ], + ) + def test_suffix(self, name: str, expected: str | None) -> None: + """Test that the longest known suffix wins, case-insensitively.""" + assert infer_format(Path(name)) == expected + + def test_no_output(self) -> None: + """Test that no output path infers no format.""" + assert infer_format(None) is None + + +class TestFormatKinds: + """Test classifying a format as an archive or a single-file container.""" + + @pytest.mark.parametrize( + ("fmt", "archive", "single"), + [ + ("zip", True, False), + ("TAR.GZ", True, False), + ("7z", True, False), + ("gz", False, True), + ("ZSTD", False, True), + ("comp", False, False), + ("nope", False, False), + (None, False, False), + ], + ) + def test_kind(self, fmt: str | None, archive: bool, single: bool) -> None: + """Test that `.comp` and unknown formats are neither kind.""" + assert is_archive_format(fmt) is archive + assert format_is_single_file(fmt) is single + + +class TestLooksLikeArchive: + """Test detecting an archive from its bytes rather than its name.""" + + def test_zip_under_any_name(self, tmp_path: Path) -> None: + """Test that a zip is recognised without a `.zip` suffix.""" + path = tmp_path / "data.bin" + with zipfile.ZipFile(path, "w") as zf: + zf.writestr("a.txt", "a") + + assert looks_like_archive(path) + + def test_single_file_container(self, tmp_path: Path) -> None: + """Test that a gzip stream is not an archive, even named `.zip`.""" + path = tmp_path / "data.zip" + path.write_bytes(gzip.compress(b"a")) + + assert not looks_like_archive(path) + + def test_missing_file(self, tmp_path: Path) -> None: + """Test that a missing file is not an archive rather than an error.""" + assert not looks_like_archive(tmp_path / "missing.zip") + + +class TestDefaultArchiveOutput: + """Test naming an archive the caller did not name.""" + + def test_beside_the_input(self, tmp_path: Path) -> None: + """Test that the archive is named after the input, next to it.""" + src = tmp_path / "src" + src.mkdir() + + assert default_archive_output(src, "tar.zst") == tmp_path / "src.tar.zst" + + @pytest.mark.parametrize("name", [".", f"..{os.sep}src{os.sep}"]) + def test_trailing_separator( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, name: str + ) -> None: + """Test that `.` and a trailing separator still name the directory.""" + src = tmp_path / "src" + src.mkdir() + monkeypatch.chdir(src) + + assert ( + default_archive_output(Path(name), "zip") == tmp_path.resolve() / "src.zip" + ) + + +class TestPlanCompress: + """Test `compress` picks the job from its inputs and the format asked for.""" + + def test_one_file_becomes_a_compresso_container(self, small_file: Path) -> None: + """Test that one file with no format defaults to a `.comp` beside it.""" + job = plan_compress(CompressRequest([small_file])) + + assert isinstance(job, CompressionJob) + assert job.plan.options.format is None + assert job.plan.dest == small_file.with_name("small.txt.comp") + + @pytest.mark.parametrize("fmt", ["gz", "bz2", "xz", "zst", "lz4", "sz"]) + def test_one_file_into_a_standalone_container( + self, small_file: Path, fmt: str + ) -> None: + """Test that each single-file format compresses one file beside it.""" + job = plan_compress(CompressRequest([small_file], format=fmt)) + + assert isinstance(job, CompressionJob) + assert job.plan.can_compress + assert job.plan.dest == small_file.with_name(f"small.txt.{fmt}") + + def test_a_directory_becomes_an_archive(self, tree: Path) -> None: + """Test that a directory with no format defaults to a zip beside it.""" + job = plan_compress(CompressRequest([tree])) + + assert isinstance(job, ArchiveJob) + assert job.plan.options.format == "zip" + assert job.plan.output == tree.with_name("tree.zip") + + def test_several_files_become_an_archive( + self, small_file: Path, tmp_path: Path + ) -> None: + """Test that several files are archived in the output's format.""" + second = tmp_path / "second.txt" + second.write_text("b") + output = tmp_path / "both.tar" + + job = plan_compress(CompressRequest([small_file, second], output=output)) + + assert isinstance(job, ArchiveJob) + assert job.plan.options.format == "tar" + assert job.plan.output == output + + @pytest.mark.parametrize( + ("name", "kind", "fmt"), + [ + ("out.gz", CompressionJob, "gz"), + ("out.zst", CompressionJob, "zst"), + ("out.zip", ArchiveJob, "zip"), + ], + ) + def test_format_inferred_from_the_output_name( + self, small_file: Path, tmp_path: Path, name: str, kind: type, fmt: str + ) -> None: + """Test that the output's suffix picks both the format and the job kind.""" + output = tmp_path / name + + job = plan_compress(CompressRequest([small_file], output=output)) + + assert isinstance(job, kind) + assert job.plan.options.format == fmt + + def test_explicit_format_beats_the_output_name( + self, small_file: Path, tmp_path: Path + ) -> None: + """Test that `--format` wins over the output's suffix, keeping its name.""" + output = tmp_path / "misleading.gz" + + job = plan_compress(CompressRequest([small_file], output=output, format="zst")) + + assert isinstance(job, CompressionJob) + assert job.plan.options.format == "zst" + assert job.plan.dest == output + + @pytest.mark.parametrize( + "name", ["out.tar.zst", "out.tar.lz4", "out.tlz4", "out.tar.sz"] + ) + def test_tar_inside_a_codec_is_an_archive( + self, tree: Path, tmp_path: Path, name: str + ) -> None: + """Test that compressed tar suffixes, short forms included, archive.""" + job = plan_compress(CompressRequest([tree], output=tmp_path / name)) + + assert isinstance(job, ArchiveJob) + assert job.plan.can_run + assert job.plan.options.format == name.removeprefix("out.") + + def test_single_file_format_refuses_several_inputs(self, tree: Path) -> None: + """Test that a single-file format on a directory errors, suggesting tar.""" + with pytest.raises(ValueError, match="tar.gz"): + plan_compress(CompressRequest([tree], format="gz")) + + def test_comp_format_is_the_compresso_container( + self, small_file: Path, tmp_path: Path + ) -> None: + """Test that naming comp writes the default container, whatever the output.""" + output = tmp_path / "misleading.gz" + + job = plan_compress(CompressRequest([small_file], output=output, format="COMP")) + + assert isinstance(job, CompressionJob) + assert job.plan.can_compress + assert job.plan.options.format is None + assert job.plan.dest == output + + @pytest.mark.parametrize( + ("fmt", "output"), [("comp", None), (None, Path("out.comp"))], ids=["-F", "-o"] + ) + def test_comp_format_refuses_several_inputs( + self, tree: Path, fmt: str | None, output: Path | None + ) -> None: + """Test that comp on a directory errors, suggesting zip, not tar.comp.""" + with pytest.raises(ValueError, match="such as zip"): + plan_compress(CompressRequest([tree], output=output, format=fmt)) + + def test_unknown_format_cannot_run(self, small_file: Path) -> None: + """Test that an unknown format plans a job that refuses to run, naming + the archive formats as well as the single-file ones.""" + job = plan_compress(CompressRequest([small_file], format="bogus")) + + assert isinstance(job, CompressionJob) + assert not job.plan.can_compress + assert job.plan.reason_if_unavailable is not None + assert job.plan.reason_if_unavailable.startswith("Unknown format: bogus") + assert all( + f" {fmt}," in job.plan.reason_if_unavailable + for fmt in ("gz", "zip", "tar", "tar.gz") + ) + + +class TestPlanExtract: + """Test `extract` picks each input's job from its own bytes.""" + + def test_dispatch_follows_the_bytes_not_the_name(self, tmp_path: Path) -> None: + """Test that a gzip stream with an unknown suffix still decompresses.""" + disguised = tmp_path / "mystery.dat" + disguised.write_bytes(gzip.compress(b"a")) + restored = tmp_path / "back.bin" + + [job] = plan_extract(ExtractRequest([disguised], output=restored)) + + assert isinstance(job, DecompressionJob) + assert job.plan.format == "gzip" + assert job.plan.dest == restored + + def test_an_archive_named_like_anything_still_extracts( + self, tmp_path: Path + ) -> None: + """Test that a zip without a `.zip` suffix plans a runnable extraction.""" + disguised = tmp_path / "riddle.bin" + with zipfile.ZipFile(disguised, "w") as zf: + zf.writestr("a.txt", "a") + dest = tmp_path / "out" + + [job] = plan_extract(ExtractRequest([disguised], directory=dest)) + + assert isinstance(job, ExtractJob) + assert job.plan.can_run + assert job.plan.output_dir == dest + + def test_an_archive_unpacks_beside_itself(self, tmp_path: Path) -> None: + """Test that without `-C` an archive extracts into its own directory.""" + archive = tmp_path / "x.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr("a.txt", "a") + + [job] = plan_extract(ExtractRequest([archive])) + + assert isinstance(job, ExtractJob) + assert job.plan.output_dir == tmp_path + + def test_directory_receives_a_single_file(self, tmp_path: Path) -> None: + """Test that `-C` gets the suffix-stripped name.""" + packed = tmp_path / "p.gz" + packed.write_bytes(gzip.compress(b"a")) + into = tmp_path / "into" + into.mkdir() + + [job] = plan_extract(ExtractRequest([packed], directory=into)) + + assert isinstance(job, DecompressionJob) + assert job.plan.dest == into / "p" + + def test_output_refuses_an_archive(self, tmp_path: Path) -> None: + """Test that `-o` on an archive plans a job that cannot run.""" + archive = tmp_path / "x.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr("a.txt", "a") + + [job] = plan_extract(ExtractRequest([archive], output=tmp_path / "out")) + + assert isinstance(job, ExtractJob) + assert not job.plan.can_run + assert "use -C" in str(job.plan.reason_if_unavailable) + + def test_each_input_is_dispatched_on_its_own(self, tmp_path: Path) -> None: + """Test that a mixed list plans one job per input, in order.""" + packed = tmp_path / "a.gz" + packed.write_bytes(gzip.compress(b"a")) + archive = tmp_path / "b.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr("b.txt", "b") + + first, second = plan_extract(ExtractRequest([packed, archive])) + + assert isinstance(first, DecompressionJob) + assert first.plan.dest == tmp_path / "a" + assert isinstance(second, ExtractJob) diff --git a/tests/python/frontend/test_verification.py b/tests/python/frontend/test_verification.py new file mode 100644 index 0000000..7bccdcb --- /dev/null +++ b/tests/python/frontend/test_verification.py @@ -0,0 +1,238 @@ +"""Tests for the frontend integrity check.""" + +import os +import stat +import tarfile +import zipfile +from pathlib import Path + +import pytest + +from compresso import CancelToken, TrailingDataWarning +from compresso.frontend.api import ( + CompressionJob, + CompressionOptions, + VerificationJob, + plan_verification, +) +from compresso.frontend.archive_api import ArchiveJob, ArchiveOptions + +from ..helpers import Recorder, flip_middle_byte + +CODECS = ["gz", "bz2", "xz", "zst", "lz4", "sz"] +ARCHIVES = ["tar.gz", "tar.bz2", "tar.xz", "tar.zst", "tar.lz4", "zip"] + + +@pytest.fixture +def source(tmp_path: Path) -> Path: + """A file holding enough incompressible data to survive one flipped byte.""" + path = tmp_path / "source.bin" + path.write_bytes(os.urandom(64 * 1024)) + + return path + + +@pytest.fixture +def tree(tmp_path: Path) -> Path: + """A directory of two files.""" + root = tmp_path / "tree" + root.mkdir() + (root / "a.bin").write_bytes(os.urandom(32 * 1024)) + (root / "b.bin").write_bytes(os.urandom(32 * 1024)) + + return root + + +def compressed(source: Path, suffix: str, algo: str | None = None) -> Path: + """Compress `source` beside itself.""" + output = source.with_suffix(f".{suffix}") + fmt = None if suffix == "comp" else suffix + options = CompressionOptions(algo=algo, format=fmt) + result = CompressionJob.from_file(source, output, options).run() + assert result.ok, result.error + + return output + + +def archived(tree: Path, fmt: str) -> Path: + """Archive `tree` beside itself.""" + output = tree.with_name(f"out.{fmt}") + result = ArchiveJob.from_paths([tree], output, ArchiveOptions(format=fmt)).run() + assert result.ok, result.error + + return output + + +def cut_short(path: Path) -> None: + """Drop the last third of `path`.""" + data = path.read_bytes() + path.write_bytes(data[: len(data) * 2 // 3]) + + +def assert_damage_caught(path: Path) -> None: + """Assert that `path` passes, then fails once flipped or cut short.""" + good = path.read_bytes() + assert VerificationJob.from_file(path).run().ok + + flip_middle_byte(path) + assert not VerificationJob.from_file(path).run().ok + + path.write_bytes(good) + cut_short(path) + assert not VerificationJob.from_file(path).run().ok + + +class TestSingleFile: + """Test standalone codecs and the `.comp` container.""" + + @pytest.mark.parametrize("suffix", CODECS) + def test_each_codec(self, source: Path, suffix: str) -> None: + """Test that a good file passes, and each kind of damage fails it.""" + assert_damage_caught(compressed(source, suffix)) + + def test_each_comp_algorithm(self, source: Path, compression_algo: str) -> None: + """Test that `.comp` damage is caught through every backend.""" + assert_damage_caught(compressed(source, "comp", compression_algo)) + + def test_trailing_data_only_warns_here(self, source: Path) -> None: + """Test that the job leaves the verdict on trailing data to its caller.""" + path = compressed(source, "gz") + path.write_bytes(path.read_bytes() + b"not gzip data") + + with pytest.warns(TrailingDataWarning): + assert VerificationJob.from_file(path).run().ok + + +class TestArchives: + """Test archives, whose entries are decoded too.""" + + @pytest.mark.parametrize("fmt", ARCHIVES) + def test_each_format(self, tree: Path, fmt: str) -> None: + """Test that a good archive passes, and each kind of damage fails it.""" + assert_damage_caught(archived(tree, fmt)) + + def test_plain_tar_has_no_data_checksum(self, tree: Path) -> None: + """Test that only what tar can detect fails it, unlike every other format.""" + path = archived(tree, "tar") + good = path.read_bytes() + with tarfile.open(path) as tar: + data_at = tar.getmembers()[-1].offset_data + + damaged = bytearray(good) + damaged[data_at] ^= 0xFF + path.write_bytes(damaged) + assert VerificationJob.from_file(path).run().ok + + # A header has a checksum of its own + damaged = bytearray(good) + damaged[data_at - 512] ^= 0xFF + path.write_bytes(damaged) + assert not VerificationJob.from_file(path).run().ok + + path.write_bytes(good) + cut_short(path) + assert not VerificationJob.from_file(path).run().ok + + def test_plain_tar_cut_between_entries_passes(self, tree: Path) -> None: + """Test that a cut leaving whole entries behind passes, as bsdtar has it.""" + path = archived(tree, "tar") + with tarfile.open(path) as tar: + last = tar.getmembers()[-1] + + path.write_bytes(path.read_bytes()[: last.offset]) + + assert VerificationJob.from_file(path).run().ok + + def test_stored_zip_entry_checks_its_crc(self, tmp_path: Path) -> None: + """Test that an uncompressed entry, which no decoder would catch, fails.""" + path = tmp_path / "stored.zip" + with zipfile.ZipFile(path, "w", zipfile.ZIP_STORED) as archive: + archive.writestr("data", os.urandom(4096)) + + flip_middle_byte(path) + + assert "CRC" in str(VerificationJob.from_file(path).run().error) + + def test_zip_symlink_target_checks_its_crc(self, tmp_path: Path) -> None: + """Test that a link, whose data is read with its header, is checked too.""" + path = tmp_path / "link.zip" + info = zipfile.ZipInfo("link") + info.create_system = 3 # Unix, or the mode below would be ignored + info.external_attr = (stat.S_IFLNK | 0o777) << 16 + with zipfile.ZipFile(path, "w") as archive: + archive.writestr(info, "some-link-target") + + assert VerificationJob.from_file(path).run().ok + + data = path.read_bytes() + path.write_bytes(data.replace(b"some-link-target", b"sOme-link-target")) + + assert "CRC" in str(VerificationJob.from_file(path).run().error) + + @pytest.mark.parametrize("fmt", ["tar", "zip"]) + def test_plain_archive_progress_counts_its_data(self, tree: Path, fmt: str) -> None: + """Test that the total is what the entries hold, not the archive's size.""" + recorder = Recorder() + + result = VerificationJob.from_file(archived(tree, fmt)).run(progress=recorder) + + assert result.ok + assert recorder.calls[-1] == (64 * 1024, 64 * 1024) + + def test_compressed_archive_progress_counts_its_input(self, tree: Path) -> None: + """Test that a codec stage reports the compressed bytes it consumes.""" + path = archived(tree, "tar.gz") + recorder = Recorder() + + VerificationJob.from_file(path).run(progress=recorder) + + assert recorder.calls[-1] == (path.stat().st_size, path.stat().st_size) + + +class TestRun: + """Test what holds for every input.""" + + @pytest.mark.parametrize("damaged", [False, True]) + @pytest.mark.parametrize("build", ["gz", "comp", "tar.gz", "zip"]) + def test_writes_nothing( + self, source: Path, tree: Path, build: str, damaged: bool + ) -> None: + """Test that the input and its directory are left as they were.""" + is_archive = build in ARCHIVES + path = archived(tree, build) if is_archive else compressed(source, build) + if damaged: + flip_middle_byte(path) + + before = sorted(path.parent.rglob("*")) + content = path.read_bytes() + + result = VerificationJob.from_file(path).run() + + assert result.ok is not damaged + assert sorted(path.parent.rglob("*")) == before + assert path.read_bytes() == content + + @pytest.mark.parametrize("fmt", ["tar", "tar.gz", "zip"]) + def test_cancelled_archive(self, tree: Path, fmt: str) -> None: + """Test that a cancel is reported as one rather than as damage.""" + token = CancelToken() + token.cancel() + + result = VerificationJob.from_file(archived(tree, fmt)).run(cancel=token) + + assert result.cancelled + assert result.error is None + + def test_unrecognised_file_cannot_run(self, tmp_path: Path) -> None: + """Test that a file of no known format is refused at the plan.""" + path = tmp_path / "notes.txt" + path.write_text("just some text") + + plan = plan_verification(path) + + assert not plan.can_run + assert not VerificationJob(plan).run().ok + + def test_missing_file_cannot_run(self, tmp_path: Path) -> None: + """Test that a missing input surfaces as an unavailable plan.""" + assert not plan_verification(tmp_path / "nope.gz").can_run diff --git a/tests/python/helpers.py b/tests/python/helpers.py index e3c867d..bc57ec4 100644 --- a/tests/python/helpers.py +++ b/tests/python/helpers.py @@ -12,7 +12,7 @@ from typing import Any, NamedTuple from compresso.frontend._job import ThreadedJob -from compresso.frontend.api import CompressionJob, DecompressionJob +from compresso.frontend.api import CompressionJob, DecompressionJob, VerificationJob from compresso.frontend.archive_api import ArchiveJob, ExtractJob # Crosses the 1 MiB report floor (`CTX_MIN_INTERVAL`) several times over @@ -45,6 +45,17 @@ def renamed(name: str, n: int = 2) -> str: return f"{stem}{_CONFLICT_SUFFIX.format(n)}{dot}{extension}" +def flip_middle_byte(path: Path) -> None: + """Damage a file while leaving its headers alone. + + Args: + path: The file to damage. + """ + data = bytearray(path.read_bytes()) + data[len(data) // 2] ^= 0xFF + path.write_bytes(data) + + class Recorder: """Collects progress callbacks for assertions.""" @@ -119,6 +130,9 @@ class JobFactory(NamedTuple): "source_tar", lambda src, out: ExtractJob.from_archive(archive=src, output_dir=out), ), + "verify": JobFactory( + "compressed_file", lambda src, _: VerificationJob.from_file(src=src) + ), } diff --git a/tests/python/introspect/test_benchmark.py b/tests/python/introspect/test_benchmark.py index 8aeeab7..3693254 100644 --- a/tests/python/introspect/test_benchmark.py +++ b/tests/python/introspect/test_benchmark.py @@ -5,7 +5,6 @@ from compresso.introspect.benchmark import ( BenchmarkResult, benchmark_file, - print_results, ) MIB = 1024 * 1024 @@ -66,45 +65,3 @@ def test_levels_outside_a_backends_range_are_skipped( ran = {(r.algo, r.level) for r in results} assert ran == {("zlib", None), ("zstd", None), ("zstd", 15), ("snappy", None)} - - -class TestPrintResults: - """Test the print_results function.""" - - def test_print_results_with_empty_list( - self, capsys: pytest.CaptureFixture[str] - ) -> None: - """Test that no results says so rather than printing an empty table.""" - print_results([]) - - assert capsys.readouterr().out == "No results to display.\n" - - def test_print_results_with_multiple_results(self, capsys) -> None: - """Test printing results with multiple results.""" - results = [ - BenchmarkResult( - algo="zlib", - strategy="balanced", - level=6, - compress_time=1.0, - decompress_time=0.5, - input_size=1000000, - compressed_size=500000, - ), - BenchmarkResult( - algo="zstd", - strategy="fast", - level=1, - compress_time=0.5, - decompress_time=0.3, - input_size=1000000, - compressed_size=450000, - ), - ] - - print_results(results) - captured = capsys.readouterr() - - # Should contain both algo names - assert "zlib" in captured.out.lower() - assert "zstd" in captured.out.lower() diff --git a/tests/python/test_archive_create.py b/tests/python/test_archive_create.py index c5a3c3e..6c105d5 100644 --- a/tests/python/test_archive_create.py +++ b/tests/python/test_archive_create.py @@ -1,6 +1,7 @@ """Tests for creating archives.""" import contextlib +import errno import logging import os import socket @@ -14,7 +15,13 @@ import orjson import pytest -from compresso import BackendError, _core +from compresso import ( + ArchiveJob, + ArchiveOptions, + BackendError, + SourceRemovalError, + _core, +) from .helpers import ARCHIVE_AND_LIST, run_child @@ -264,6 +271,430 @@ def test_entries_follow_their_directory(self, tmp_path: Path) -> None: assert listed[i + 1 : i + 1 + len(inside)] == inside, listed +class TestArchiveFilter: + """Test that `include` and `exclude` narrow the walk.""" + + @pytest.fixture + def tree(self, tmp_path: Path) -> Path: + """A tree with text, logs, a node_modules and an empty directory.""" + for rel in ("t/a/keep.txt", "t/a/drop.log", "t/a/node_modules/m.txt", "t/b/n"): + (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / rel).write_text(rel) + (tmp_path / "t/empty").mkdir() + return tmp_path / "t" + + @pytest.mark.parametrize( + ("include", "exclude", "expected"), + [ + ( + [], + ["node_modules", "*.log"], + ["t/", "t/a/", "t/a/keep.txt", "t/b/", "t/b/n", "t/empty/"], + ), + ( + ["*.txt"], + [], + [ + "t/", + "t/a/", + "t/a/keep.txt", + "t/a/node_modules/", + "t/a/node_modules/m.txt", + ], + ), + (["*.txt"], ["node_modules"], ["t/", "t/a/", "t/a/keep.txt"]), + ], + ids=["exclude", "include-keeps-holding-dirs", "exclude-wins"], + ) + def test_entries_kept( + self, + tree: Path, + tmp_path: Path, + include: list[str], + exclude: list[str], + expected: list[str], + ) -> None: + """Test which entries a filter stores.""" + output = tmp_path / "out.tar" + + _core.create_archive( + str(output), "tar", [str(tree)], include=include, exclude=exclude + ) + + listed = [e["path"] for e in _core.list_archive_contents(str(output))] + assert sorted(listed) == expected + + def test_bare_string_is_refused(self, tree: Path, tmp_path: Path) -> None: + """Test that a str isn't taken as a sequence of one-character patterns.""" + with pytest.raises(TypeError, match="sequence of str, not str"): + _core.create_archive( + str(tmp_path / "out.tar"), "tar", [str(tree)], include="*.txt" + ) + + @pytest.mark.parametrize( + ("include", "exclude"), + [([], ["node_modules", "*.log"]), (["*.txt"], []), (["t/a"], ["*.log"])], + ) + def test_plan_totals_what_the_filter_keeps( + self, tree: Path, tmp_path: Path, include: list[str], exclude: list[str] + ) -> None: + """Test that the plan's total matches the bytes the walk stores.""" + options = ArchiveOptions(format="tar", include=include, exclude=exclude) + job = ArchiveJob.from_paths([tree], tmp_path / "out.tar", options) + + result = job.run() + + assert result.ok, result.error + stored = _core.list_archive_contents(str(job.plan.output)) + assert job.plan.total_input_size == sum(e["size"] for e in stored) + + def test_plan_does_not_walk_an_excluded_tree( + self, tree: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Test that the plan prunes excluded directories, as the C walk does.""" + scanned: list[str] = [] + real_scandir = os.scandir + + def spy(path): + scanned.append(os.fspath(path)) + return real_scandir(path) + + monkeypatch.setattr(os, "scandir", spy) + + ArchiveJob.from_paths( + [tree], + tmp_path / "out.tar", + ArchiveOptions(format="tar", exclude=["node_modules"]), + ) + + assert str(tree / "a") in scanned + assert not any("node_modules" in path for path in scanned) + + def test_plan_counts_only_a_kept_source(self, tree: Path, tmp_path: Path) -> None: + """Test that an excluded source adds nothing to the plan's total.""" + keep = tree / "a" / "keep.txt" + options = ArchiveOptions(format="tar", exclude=["drop.log", "b"]) + + job = ArchiveJob.from_paths( + [keep, tree / "a" / "drop.log", tree / "b"], tmp_path / "out.tar", options + ) + + assert job.plan.total_input_size == keep.stat().st_size + + @pytest.mark.usefixtures("needs_symlinks") + def test_plan_counts_a_symlink_as_a_link(self, tree: Path, tmp_path: Path) -> None: + """Test that a link adds no bytes, wherever it is, like the walk.""" + target = tree / "a" / "keep.txt" + (tree / "b" / "link").symlink_to(target) + source_link = tmp_path / "source-link" + source_link.symlink_to(tree) + + def total(*sources: Path) -> int: + options = ArchiveOptions(format="tar") + job = ArchiveJob.from_paths(list(sources), tmp_path / "o.tar", options) + return job.plan.total_input_size + + assert total(tree / "b") == (tree / "b" / "n").stat().st_size + assert total(source_link) == 0 + + +@pytest.mark.usefixtures("needs_symlinks") +class TestDereference: + """Test that `dereference` stores what symlinks lead to, as bsdtar `-h`.""" + + @pytest.fixture + def tree(self, tmp_path: Path) -> Path: + """A tree with links to a file, and to a directory outside it.""" + other = tmp_path / "other" + other.mkdir() + (other / "f.txt").write_text("other") + + root = tmp_path / "tree" + root.mkdir() + (root / "a.txt").write_text("hello") + (root / "flink").symlink_to("a.txt") + (root / "dlink").symlink_to(other, target_is_directory=True) + + return root + + @staticmethod + def entries(output: Path) -> dict[str, "_core.ArchiveEntryDict"]: + return {e["path"].rstrip("/"): e for e in _core.list_archive_contents(output)} + + @pytest.mark.parametrize("fmt", ["tar", "tar.zst", "zip"]) + def test_links_become_their_targets( + self, tree: Path, tmp_path: Path, fmt: str + ) -> None: + """Test that a link to a file or a directory holds what it leads to.""" + output = tmp_path / f"out.{fmt}" + + _core.create_archive(str(output), fmt, [str(tree)], dereference=True) + + entries = self.entries(output) + assert entries["tree/flink"]["type"] == "file" + assert entries["tree/flink"]["size"] == 5 + assert entries["tree/dlink"]["type"] == "dir" + assert entries["tree/dlink/f.txt"]["size"] == 5 + + def test_links_stay_links_by_default(self, tree: Path, tmp_path: Path) -> None: + """Test that nothing is followed unless asked.""" + output = tmp_path / "out.tar" + + _core.create_archive(str(output), "tar", [str(tree)]) + + entries = self.entries(output) + assert entries["tree/dlink"]["type"] == "symlink" + assert "tree/dlink/f.txt" not in entries + + def test_source_link_is_followed(self, tree: Path, tmp_path: Path) -> None: + """Test that a link given as a source is stored under its own name.""" + link = tmp_path / "top" + link.symlink_to(tree, target_is_directory=True) + output = tmp_path / "out.tar" + + _core.create_archive(str(output), "tar", [str(link)], dereference=True) + + entries = self.entries(output) + assert entries["top"]["type"] == "dir" + assert entries["top/a.txt"]["size"] == 5 + + def test_dangling_link_stays_a_link(self, tree: Path, tmp_path: Path) -> None: + """Test that a link leading nowhere, or to itself, is stored as a link.""" + (tree / "dangling").symlink_to("missing") + (tree / "self").symlink_to("self") + output = tmp_path / "out.tar" + + _core.create_archive(str(output), "tar", [str(tree)], dereference=True) + + entries = self.entries(output) + assert entries["tree/dangling"]["type"] == "symlink" + assert entries["tree/dangling"]["link_target"] == "missing" + assert entries["tree/self"]["type"] == "symlink" + + @pytest.mark.parametrize( + ("as_source", "dereference", "warns"), + [ + (True, False, True), + (True, True, True), + (False, True, True), + (False, False, False), + ], + ) + def test_dangling_link_warns_where_data_was_expected( + self, + tree: Path, + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + as_source: bool, + dereference: bool, + warns: bool, + ) -> None: + """Test that only a named or followed dangling link is reported.""" + link = tree / "dangling" + link.symlink_to("missing") + source = link if as_source else tree + + with caplog.at_level(logging.WARNING, logger="compresso"): + _core.create_archive( + str(tmp_path / "out.tar"), "tar", [str(source)], dereference=dereference + ) + + expected = f"Stored {link} as a link: its target can't be reached" + assert (expected in caplog.text) is warns + + def test_link_to_an_ancestor_stays_a_link( + self, tree: Path, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + """Test that a loop, even through another directory, ends the walk quietly.""" + (tree / "up").symlink_to(".", target_is_directory=True) + (tree / "sub").mkdir() + (tree / "sub" / "up").symlink_to("..", target_is_directory=True) + (tmp_path / "other" / "back").symlink_to(tree, target_is_directory=True) + output = tmp_path / "out.tar" + + with caplog.at_level(logging.WARNING, logger="compresso"): + _core.create_archive(str(output), "tar", [str(tree)], dereference=True) + + entries = self.entries(output) + for name in ("tree/up", "tree/sub/up", "tree/dlink/back"): + assert entries[name]["type"] == "symlink" + assert "as a link" not in caplog.text + + def test_link_to_the_output_is_skipped(self, tree: Path, tmp_path: Path) -> None: + """Test that the archive being written isn't stored through a link.""" + output = tmp_path / "out.tar" + output.write_text("old") + (tree / "own").symlink_to(output) + + _core.create_archive( + str(output), "tar", [str(tree)], overwrite=2, dereference=True + ) + + assert "tree/own" not in self.entries(output) + + def test_excluded_link_is_not_walked(self, tree: Path, tmp_path: Path) -> None: + """Test that an exclude still prunes what a link leads to.""" + output = tmp_path / "out.tar" + + _core.create_archive( + str(output), "tar", [str(tree)], exclude=["*/dlink"], dereference=True + ) + + assert not any(name.startswith("tree/dlink") for name in self.entries(output)) + + def test_rm_deletes_links_but_not_targets(self, tree: Path, tmp_path: Path) -> None: + """Test that a link goes, source or not; its target stays, unlike GNU tar's.""" + other = tmp_path / "other" + top = tmp_path / "top" + top.symlink_to(other, target_is_directory=True) + + _core.create_archive( + str(tmp_path / "out.tar"), + "tar", + [str(tree), str(top)], + dereference=True, + remove_sources=True, + ) + + assert not os.path.lexists(tree) + assert not os.path.lexists(top) + assert (other / "f.txt").read_text() == "other" + + def test_plan_counts_what_the_walk_stores(self, tree: Path, tmp_path: Path) -> None: + """Test that the planned total follows links, loops and dangling ones.""" + (tree / "dangling").symlink_to("missing") + (tree / "up").symlink_to(".", target_is_directory=True) + (tmp_path / "other" / "back").symlink_to(tree, target_is_directory=True) + + def total(dereference: bool) -> int: + options = ArchiveOptions(format="tar", dereference=dereference) + job = ArchiveJob.from_paths([tree], tmp_path / "o.tar", options) + assert job.plan.can_run + return job.plan.total_input_size + + assert total(False) == 5 + assert total(True) == 15 + + def test_plan_keeps_a_dangling_source(self, tree: Path, tmp_path: Path) -> None: + """Test that a dangling link exists as a source, as it does to the walk.""" + link = tree / "dangling" + link.symlink_to("missing") + output = tmp_path / "o.tar" + + job = ArchiveJob.from_paths([link], output, ArchiveOptions(format="tar")) + assert job.run().ok + + assert self.entries(output)["dangling"]["type"] == "symlink" + + +class TestRemoveSources: + """Test that `remove_sources` deletes exactly what the archive stored.""" + + @pytest.fixture + def tree(self, tmp_path: Path) -> Path: + """A small tree with a nested directory.""" + root = tmp_path / "tree" + (root / "sub").mkdir(parents=True) + (root / "a.txt").write_text("a") + (root / "sub" / "b.log").write_text("b") + + return root + + @pytest.mark.parametrize("fmt", ["tar", "tar.zst", "zip"]) + def test_stored_tree_is_deleted(self, tree: Path, tmp_path: Path, fmt: str) -> None: + """Test that a stored tree goes, leaving a complete archive.""" + output = tmp_path / f"out.{fmt}" + + _core.create_archive(str(output), fmt, [str(tree)], remove_sources=True) + + assert not tree.exists() + names = {e["path"].rstrip("/") for e in _core.list_archive_contents(output)} + assert names == {"tree", "tree/a.txt", "tree/sub", "tree/sub/b.log"} + + def test_what_a_filter_left_out_stays(self, tree: Path, tmp_path: Path) -> None: + """Test that an excluded file keeps itself and the directory holding it.""" + _core.create_archive( + str(tmp_path / "out.tar"), + "tar", + [str(tree)], + exclude=["b.log"], + remove_sources=True, + ) + + assert (tree / "sub" / "b.log").is_file() + assert not (tree / "a.txt").exists() + + def test_output_inside_the_source_stays(self, tree: Path) -> None: + """Test that the walk skipping the archive saves it from the removal.""" + output = tree / "out.tar" + + _core.create_archive(str(output), "tar", [str(tree)], remove_sources=True) + + assert [p.name for p in tree.iterdir()] == ["out.tar"] + assert len(_core.list_archive_contents(output)) == 4 + + @pytest.mark.skipif(sys.platform == "win32", reason="needs a FIFO") + def test_unstored_special_file_stays(self, tree: Path, tmp_path: Path) -> None: + """Test that a FIFO zip can't hold survives with its directory.""" + os.mkfifo(tree / "pipe") + + _core.create_archive( + str(tmp_path / "out.zip"), "zip", [str(tree)], remove_sources=True + ) + + assert [p.name for p in tree.iterdir()] == ["pipe"] + + @pytest.mark.usefixtures("needs_symlinks") + def test_symlink_goes_but_not_its_target(self, tree: Path, tmp_path: Path) -> None: + """Test that a source symlink, stored as a link, is all that's deleted.""" + link = tmp_path / "link" + link.symlink_to(tree, target_is_directory=True) + + _core.create_archive( + str(tmp_path / "out.tar"), "tar", [str(link)], remove_sources=True + ) + + assert not link.is_symlink() + assert (tree / "a.txt").is_file() + + def test_skip_is_refused(self, tree: Path, tmp_path: Path) -> None: + """Test that skip, which can drop a finished archive, can't delete sources.""" + with pytest.raises(ValueError, match="remove_sources"): + _core.create_archive( + str(tmp_path / "out.tar"), + "tar", + [str(tree)], + overwrite=1, + remove_sources=True, + ) + + assert (tree / "a.txt").is_file() + assert not (tmp_path / "out.tar").exists() + + def test_undeletable_sources_are_each_reported( + self, tree: Path, locked: Path, tmp_path: Path + ) -> None: + """Test that the rest still go, and the error covers all that stayed.""" + (locked / "c.txt").write_text("c") + (locked / "d.txt").write_text("d") + locked.chmod(0o555) + output = tmp_path / "out.tar" + + with pytest.raises(SourceRemovalError) as caught: + _core.create_archive( + str(output), "tar", [str(tree), str(locked)], remove_sources=True + ) + + reason = os.strerror(errno.EACCES) + assert caught.value.output == str(output) + assert sorted(caught.value.failures) == [ + (str(locked / "c.txt"), reason), + (str(locked / "d.txt"), reason), + ] + assert not tree.exists() + assert len(_core.list_archive_contents(output)) == 7 + + class TestSizeChange: """Test a file whose size changes between the walk and its copy.""" @@ -436,6 +867,38 @@ def test_zip_keeps_copies(self, tmp_path: Path) -> None: with zipfile.ZipFile(archive) as zf: assert zf.read("src/b") == b"x" + @pytest.mark.parametrize( + ("name", "expected"), [("a.tar.gz", 1), ("a.tar", 1), ("a.zip", 2)] + ) + def test_plan_counts_a_file_as_the_format_stores_it( + self, tmp_path: Path, name: str, expected: int + ) -> None: + """Test that a link adds no bytes where it's stored as one, across sources.""" + first = tmp_path / "x" + second = tmp_path / "y" + first.mkdir() + second.mkdir() + (first / "a").write_text("data") + os.link(first / "a", second / "b") + + job = ArchiveJob.from_paths([first, second], tmp_path / name) + + assert job.plan.total_input_size == 4 * expected + + def test_plan_counts_a_link_whose_first_name_is_excluded( + self, tmp_path: Path + ) -> None: + """Test that a name the filter drops doesn't hide the one it keeps.""" + src = tmp_path / "src" + src.mkdir() + (src / "a.log").write_text("data") + os.link(src / "a.log", src / "b.txt") + options = ArchiveOptions(format="tar", exclude=["*.log"]) + + job = ArchiveJob.from_paths([src], tmp_path / "out.tar", options) + + assert job.plan.total_input_size == 4 + class TestZipModesAndSymlinks: """Test that zip keeps Unix modes and symlinks in its external attributes.""" diff --git a/tests/python/test_archive_extract.py b/tests/python/test_archive_extract.py index 6d04703..d818762 100644 --- a/tests/python/test_archive_extract.py +++ b/tests/python/test_archive_extract.py @@ -20,6 +20,8 @@ BackendError, CorruptDataError, ExtractionPolicyError, + ExtractJob, + ExtractOptions, IncompleteExtractionError, _core, ) @@ -132,6 +134,34 @@ def test_selects_at_component_boundaries( extracted = [p.relative_to(out).as_posix() for p in out.rglob("*")] assert sorted(p for p in extracted if (out / p).is_file()) == expected + @pytest.mark.parametrize( + ("files", "include", "exclude", "expected"), + [ + ([], ["*.txt"], ["sub"], ["d/x.txt", "dx.txt", "e.txt"]), + (["d"], [], ["y.txt"], ["d/x.txt"]), + (["d"], ["e.txt"], ["y.txt"], ["d/x.txt", "e.txt"]), + ], + ids=["include-and-exclude", "with-files", "files-or-include"], + ) + def test_filter_shapes_the_selection( + self, + archive: Path, + tmp_path: Path, + files: list[str], + include: list[str], + exclude: list[str], + expected: list[str], + ) -> None: + """Test that includes add to `files`, as in bsdtar, and excludes override both.""" + out = tmp_path / "out" + + _core.extract_archive( + str(archive), str(out), files, include=include, exclude=exclude + ) + + extracted = [p.relative_to(out).as_posix() for p in out.rglob("*")] + assert sorted(p for p in extracted if (out / p).is_file()) == expected + def test_unmatched_names_are_reported_after_the_rest( self, archive: Path, tmp_path: Path ) -> None: @@ -145,6 +175,78 @@ def test_unmatched_names_are_reported_after_the_rest( assert [p.name for p in out.iterdir()] == ["e.txt"] + def test_unmatched_includes_are_reported_after_the_rest( + self, archive: Path, tmp_path: Path + ) -> None: + """Test that an include matching nothing raises, as bsdtar's does.""" + out = tmp_path / "out" + + with pytest.raises( + IncompleteExtractionError, match=r"Not found in archive: nope, zzz\*$" + ): + _core.extract_archive( + str(archive), str(out), [], include=["e.txt", "nope", "zzz*"] + ) + + assert [p.name for p in out.iterdir()] == ["e.txt"] + + @pytest.mark.parametrize( + ("files", "include", "exclude"), + [ + (["e.txt"], [], ["*"]), + ([], ["e.txt"], ["*"]), + ], + ids=["excluded-name", "excluded-include"], + ) + def test_filtered_out_matches_are_not_reported( + self, + archive: Path, + tmp_path: Path, + files: list[str], + include: list[str], + exclude: list[str], + ) -> None: + """Test that what the filter drops still counts as found, as in tar.""" + out = tmp_path / "out" + + _core.extract_archive( + str(archive), str(out), files, include=include, exclude=exclude + ) + + assert not [p for p in out.rglob("*") if p.is_file()] + + @pytest.mark.parametrize( + ("files", "include", "exclude"), + [ + ([], [], ["sub"]), + ([], ["*.txt"], ["d/x.txt"]), + (["d/sub"], ["e.txt"], []), + (["d"], ["e.txt"], ["*x.txt"]), + (["d/x.txt"], [], ["*"]), + ], + ) + def test_plan_lists_what_is_extracted( + self, + archive: Path, + tmp_path: Path, + files: list[str], + include: list[str], + exclude: list[str], + ) -> None: + """Test that the plan's entries, and so the bar's total, match the run.""" + out = tmp_path / "out" + options = ExtractOptions(include=include, exclude=exclude) + job = ExtractJob.from_archive(archive, out, files, options) + + result = job.run() + + assert result.ok, result.error + written = sorted( + p.relative_to(out).as_posix() for p in out.rglob("*") if p.is_file() + ) + planned = sorted(e.path for e in job.plan.entries if not e.is_dir) + assert planned == written + class TestRecognisedButUnsupportedArchive: """Test that a detected archive format without a backend names itself.""" @@ -472,19 +574,32 @@ def test_fifo_is_created_with_its_metadata(self, tmp_path: Path) -> None: @pytest.mark.skipif( sys.platform == "win32" or os.geteuid() == 0, reason="needs POSIX, not root" ) - def test_device_is_skipped_without_root( - self, tmp_path: Path, caplog: pytest.LogCaptureFixture - ) -> None: - """Test that a device is skipped with a warning, the rest still extracted.""" + def test_device_is_reported_without_root(self, tmp_path: Path) -> None: + """Test that a device fails the extraction, the rest still extracted.""" archive = tmp_path / "nodes.tar" make_tar(archive, file_entry("a"), node_entry("char", tarfile.CHRTYPE)) out = tmp_path / "out" - with caplog.at_level(logging.WARNING, logger="compresso"): + with pytest.raises( + IncompleteExtractionError, + match=r"^Skipped, the rest was extracted: char \(devices are only created by root", + ): _core.extract_archive(str(archive), str(out), []) assert os.listdir(out) == ["a"] - assert "Skipped char: devices are only created by root" in caplog.text + + @pytest.mark.skipif( + sys.platform == "win32" or os.geteuid() == 0, reason="needs POSIX, not root" + ) + def test_excluded_device_is_not_reported(self, tmp_path: Path) -> None: + """Test that a device left out on purpose doesn't fail the extraction.""" + archive = tmp_path / "nodes.tar" + make_tar(archive, file_entry("a"), node_entry("char", tarfile.CHRTYPE)) + out = tmp_path / "out" + + _core.extract_archive(str(archive), str(out), [], exclude=["char"]) + + assert os.listdir(out) == ["a"] @pytest.mark.skipif( sys.platform == "win32" or os.geteuid() != 0, reason="needs root" @@ -501,19 +616,19 @@ def test_device_is_created_as_root(self, tmp_path: Path) -> None: assert st.st_rdev == os.makedev(1, 3) @pytest.mark.skipif(sys.platform != "win32", reason="Windows-only") - def test_windows_skips_a_fifo( - self, tmp_path: Path, caplog: pytest.LogCaptureFixture - ) -> None: - """Test that a FIFO is skipped with a warning, the rest still extracted.""" + def test_windows_reports_a_fifo(self, tmp_path: Path) -> None: + """Test that a FIFO fails the extraction, the rest still extracted.""" archive = tmp_path / "nodes.tar" make_tar(archive, file_entry("a"), node_entry("fifo", tarfile.FIFOTYPE)) out = tmp_path / "out" - with caplog.at_level(logging.WARNING, logger="compresso"): + with pytest.raises( + IncompleteExtractionError, + match=r"^Skipped, the rest was extracted: fifo \(FIFOs can't be created on this platform\)$", + ): _core.extract_archive(str(archive), str(out), []) assert os.listdir(out) == ["a"] - assert "Skipped fifo: FIFOs can't be created on this platform" in caplog.text class TestDirectoryMetadata: @@ -794,6 +909,27 @@ def test_target_text_counts_toward_the_size_cap(self, tmp_path: Path) -> None: class TestSymlinkExtraction: """Test allow_symlinks=1, which creates links whose targets stay inside.""" + def test_uncreatable_link_is_reported(self, tmp_path: Path) -> None: + """Test that a link this process can't make keeps the rest extracted.""" + try: + (tmp_path / "symlink-probe").symlink_to("x") + except OSError: + pass + else: + pytest.skip("symlinks are permitted here") + + archive = tmp_path / "links.tar" + make_tar(archive, file_entry("a"), link_entry("b", "a", kind=tarfile.SYMTYPE)) + out = tmp_path / "out" + + with pytest.raises( + IncompleteExtractionError, + match=r"^Skipped, the rest was extracted: b \(this user or filesystem", + ): + _core.extract_archive(str(archive), str(out), [], allow_symlinks=1) + + assert os.listdir(out) == ["a"] + @pytest.mark.usefixtures("needs_symlinks") def test_link_is_created_as_stored(self, tmp_path: Path) -> None: """Test that the link holds its target verbatim, and resolves to it.""" diff --git a/tests/python/test_cli.py b/tests/python/test_cli.py deleted file mode 100644 index 79ce367..0000000 --- a/tests/python/test_cli.py +++ /dev/null @@ -1,1034 +0,0 @@ -"""Tests for the Compresso CLI.""" - -from __future__ import annotations - -import gzip -import os -import subprocess -import sys -import tarfile -from collections.abc import Callable, Generator -from contextlib import contextmanager -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -import orjson -import pytest -from typer.testing import CliRunner, Result - -import compresso.cli.algos as algos_cmd -import compresso.cli.compress as compress_cmd -import compresso.cli.extract as extract_cmd -import compresso.cli.inspect as inspect_cmd -from compresso.cli import app -from compresso.cli._render import ( - BAR_THRESHOLD, - EXIT_CANCELLED, - EXIT_FAILED, - EXIT_OK, - EXIT_USAGE, - format_size, - format_time, - progress_bar, -) -from compresso.cli.extract import list_entries -from compresso.frontend._job import ProgressCallback -from compresso.frontend.api import CompressionJob, DecompressionJob -from compresso.frontend.archive_api import ( - ArchiveEntry, - ArchiveJob, - ArchiveOptions, - ExtractJob, -) - -from .helpers import file_entry, link_entry, make_tar, renamed, umask - -runner = CliRunner() - -# Incompressible throughout, so compressed sizes stay close to these -PAYLOAD_SIZE = 64 * 1024 -TREE_FILE_SIZE = 16 * 1024 - -# Past the progress bar threshold, for the tests that cover the bar -LARGE_PAYLOAD_SIZE = 2 * BAR_THRESHOLD - - -def invoke(*args: str | Path, expect: int | None = EXIT_OK) -> Result: - """Run the CLI, asserting it exits with `expect`. - - Args: - *args: The command line, without the program name. - expect: The exit code to assert, or None to leave it unchecked. - - Returns: - The runner's result. - """ - result = runner.invoke(app, [str(arg) for arg in args]) - if expect is not None: - assert result.exit_code == expect, result.output - - return result - - -@pytest.fixture -def payload(tmp_path: Path) -> Path: - """A small incompressible file, below the progress bar threshold.""" - path = tmp_path / "payload.bin" - path.write_bytes(os.urandom(PAYLOAD_SIZE)) - - return path - - -@pytest.fixture -def source_tree(tmp_path: Path) -> Path: - """A small directory tree to archive, function-scoped because some tests delete it.""" - root = tmp_path / "tree" - root.mkdir() - for i in range(3): - (root / f"f{i}.bin").write_bytes(os.urandom(TREE_FILE_SIZE)) - - return root - - -@pytest.fixture -def make_archive(source_tree: Path, tmp_path: Path) -> Callable[[str], Path]: - """Build an archive of `source_tree` as input for a test. - - Built through the API rather than the CLI, and checked, so a broken setup - cannot surface as a confusing failure later in the test. - """ - - def make(fmt: str) -> Path: - archive = tmp_path / f"out.{fmt}" - options = ArchiveOptions(format=fmt) - result = ArchiveJob.from_paths([source_tree], archive, options).run() - assert result.ok, result.error - return archive - - return make - - -@dataclass -class CompressTarget: - """What one `compress` call reads and writes.""" - - source: Path - dest: Path - args: tuple[str, ...] - - -@pytest.fixture(params=["file", "tree"]) -def compress_target( - request: pytest.FixtureRequest, payload: Path, source_tree: Path, tmp_path: Path -) -> CompressTarget: - """A single file into a .comp container, or a tree into a tar archive. - - The two take different paths through `compress`, but share its - overwrite-mode handling. - """ - if request.param == "file": - return CompressTarget(payload, tmp_path / "out.comp", ()) - - return CompressTarget(source_tree, tmp_path / "out.tar", ("-f", "tar")) - - -@dataclass -class Bar: - """One progress bar a command opened.""" - - enabled: bool - updates: int = 0 - - -@pytest.fixture -def bars(monkeypatch: pytest.MonkeyPatch) -> list[Bar]: - """Record each progress bar a command opens, in order, still drawing the real one.""" - opened: list[Bar] = [] - - @contextmanager - def spy( - label: str, total: int, *, enabled: bool - ) -> Generator[ProgressCallback | None]: - bar = Bar(enabled) - opened.append(bar) - with progress_bar(label, total, enabled=enabled) as on_progress: - if on_progress is None: - yield None - return - - def counting(fraction: float, done: int, reported_total: int) -> None: - bar.updates += 1 - on_progress(fraction, done, reported_total) - - yield counting - - # Each command imports progress_bar by name, so each binding is patched - monkeypatch.setattr(compress_cmd, "progress_bar", spy) - monkeypatch.setattr(extract_cmd, "progress_bar", spy) - - return opened - - -class TestFormatters: - """Test utility formatting functions.""" - - @pytest.mark.parametrize( - "size,expected", - [ - (0, "0.00 B"), - (500, "500.00 B"), - (1024, "1.00 KB"), - (1536, "1.50 KB"), - (1048576, "1.00 MB"), - (1073741824, "1.00 GB"), - (1099511627776, "1.00 TB"), - ], - ) - def test_format_size(self, size: int, expected: str): - """Test size formatting with various values.""" - result = format_size(size) - assert result == expected - - @pytest.mark.parametrize( - "seconds,expected", - [ - (0.0005, "0ms"), - (0.5, "500ms"), - (1.0, "1.00s"), - (5.5, "5.50s"), - (59.9, "59.90s"), - (60.0, "1m 0.0s"), - (90.5, "1m 30.5s"), - (125.0, "2m 5.0s"), - ], - ) - def test_format_time(self, seconds: float, expected: str): - """Test time formatting with various values.""" - assert format_time(seconds) == expected - - -class TestAppStructure: - """Test the package assembles into one app with every command registered.""" - - def test_commands_are_listed_in_workflow_order(self) -> None: - """Test that the order is chosen in `__init__`, not inherited from import sorting. - - Alphabetical imports would put `list` first. - """ - output = invoke("--help").output - - # Matched on the alias column, which is unique per row; a bare command - # name also appears inside other commands' descriptions - positions = [ - output.index(aliases) - for aliases in [ - "(c, comp", - "(x, ex", - "(i, info)", - "(b, bench)", - "(l, ls)", - ] - ] - assert positions == sorted(positions) - - @pytest.mark.parametrize( - "alias,command", - [ - ("c", "compress"), - ("comp", "compress"), - ("archive", "compress"), - ("a", "compress"), - ("ar", "compress"), - ("x", "extract"), - ("ex", "extract"), - ("decompress", "extract"), - ("d", "extract"), - ("decomp", "extract"), - ("i", "inspect"), - ("info", "inspect"), - ("b", "benchmark"), - ("bench", "benchmark"), - ("l", "list"), - ("ls", "list"), - ], - ) - def test_aliases_still_resolve(self, alias: str, command: str) -> None: - """Test that each short name reaches the command it stands for. - - Help is compared without its usage line, which echoes the name typed. - """ - - def help_body(name: str) -> list[str]: - lines = invoke(name, "--help").output.splitlines() - return [line for line in lines if "Usage:" not in line] - - assert help_body(alias) == help_body(command) - - -class TestProgressBar: - """Test the bar is drawn only for a large job that is not asked to be quiet.""" - - @pytest.mark.parametrize( - "size,quiet,expected", - [ - (LARGE_PAYLOAD_SIZE, False, True), - (LARGE_PAYLOAD_SIZE, True, False), - (PAYLOAD_SIZE, False, False), - ], - ids=["large", "large-quiet", "small"], - ) - def test_compress_bar_follows_size_and_quiet( - self, tmp_path: Path, bars: list[Bar], size: int, quiet: bool, expected: bool - ) -> None: - """Test that the bar needs both a job past the threshold and no --quiet.""" - source = tmp_path / "source.bin" - source.write_bytes(os.urandom(size)) - - invoke( - "compress", source, "-o", tmp_path / "out.comp", *(["-q"] if quiet else []) - ) - - assert [bar.enabled for bar in bars] == [expected] - - def test_file_round_trip_draws_both_bars( - self, tmp_path: Path, bars: list[Bar] - ) -> None: - """Test that compress and extract each drive a bar with real progress.""" - source = tmp_path / "source.bin" - source.write_bytes(os.urandom(LARGE_PAYLOAD_SIZE)) - packed = tmp_path / "out.comp" - restored = tmp_path / "restored.bin" - - invoke("compress", source, "-o", packed) - invoke("extract", packed, "-o", restored) - - assert [bar.enabled for bar in bars] == [True, True] - assert all(bar.updates > 0 for bar in bars) - assert restored.read_bytes() == source.read_bytes() - - def test_archive_round_trip_draws_both_bars( - self, tmp_path: Path, bars: list[Bar] - ) -> None: - """Test that archiving and extracting a tree each drive a bar too.""" - tree = tmp_path / "big_tree" - tree.mkdir() - for i in range(2): - (tree / f"f{i}.bin").write_bytes(os.urandom(LARGE_PAYLOAD_SIZE // 2)) - archive = tmp_path / "out.tar" - dest = tmp_path / "dest" - - invoke("compress", tree, "-o", archive, "-f", "tar") - invoke("extract", archive, "-o", dest) - - assert [bar.enabled for bar in bars] == [True, True] - assert all(bar.updates > 0 for bar in bars) - for original in tree.iterdir(): - assert (dest / tree.name / original.name).read_bytes() == ( - original.read_bytes() - ) - - -class TestCompressRoundTrip: - """Test compress and decompress, end to end.""" - - def test_compress_then_decompress(self, payload: Path, tmp_path: Path) -> None: - """Test that a file survives a round trip through the CLI unchanged.""" - archive = tmp_path / "out.comp" - restored = tmp_path / "restored.bin" - - result = invoke("compress", payload, "-o", archive) - assert "Compression successful" in result.output - - result = invoke("decompress", archive, "-o", restored) - assert "Decompression successful" in result.output - - assert restored.read_bytes() == payload.read_bytes() - - def test_quiet_suppresses_output(self, payload: Path, tmp_path: Path) -> None: - """Test that --quiet prints nothing on success.""" - result = invoke("compress", payload, "-o", tmp_path / "q.comp", "-q") - assert result.output.strip() == "" - - # The backends are covered in core and C tests; this checks the flags - @pytest.mark.parametrize( - "args", [("-a", "bzip2"), ("-a", "zstd", "-l", "19")], ids=["algo", "level"] - ) - def test_algorithms_round_trip( - self, payload: Path, tmp_path: Path, args: tuple[str, ...] - ) -> None: - """Test that -a and -l reach the backend.""" - archive = tmp_path / "out.comp" - restored = tmp_path / "restored.bin" - - invoke("compress", payload, "-o", archive, *args, "-q") - invoke("decompress", archive, "-o", restored, "-q") - - assert restored.read_bytes() == payload.read_bytes() - - -class TestArchiveRoundTrip: - """Test archive and extract, end to end.""" - - @pytest.mark.parametrize("fmt", ["tar", "tar.gz", "tar.zst", "zip"]) - def test_archive_then_extract( - self, source_tree: Path, tmp_path: Path, fmt: str - ) -> None: - """Test that a tree survives a round trip through each archive format.""" - archive = tmp_path / f"out.{fmt}" - dest = tmp_path / f"dest.{fmt}" - - result = invoke("compress", source_tree, "-o", archive, "-f", fmt) - assert "Archive created" in result.output - - result = invoke("extract", archive, "-o", dest) - assert "Extraction successful" in result.output - - for original in source_tree.iterdir(): - restored = dest / source_tree.name / original.name - assert restored.read_bytes() == original.read_bytes() - - def test_extract_list_only( - self, - tmp_path: Path, - make_archive: Callable[[str], Path], - ) -> None: - """Test that --list prints each entry's size and path without writing anything.""" - archive = make_archive("tar") - dest = tmp_path / "dest" - - result = invoke("extract", archive, "--list", "-o", dest) - - lines = result.output.splitlines() - f0_line = next(line for line in lines if "f0.bin" in line) - assert f0_line.split()[:2] == format_size(TREE_FILE_SIZE).split() - assert not dest.exists() - - # Directories have a blank size; their contents are indented beneath them - dir_line = next(line for line in lines if line.rstrip("/").endswith("tree")) - assert dir_line == f"{'':>10} {dir_line.strip()}" - assert f0_line.endswith(" tree/f0.bin") - - def test_extract_list_indents_by_depth( - self, capsys: pytest.CaptureFixture[str] - ) -> None: - """Test that --list indents only under directories it has listed.""" - entries = [ - ArchiveEntry(path="a/", is_dir=True), - ArchiveEntry(path="a/b/", is_dir=True), - ArchiveEntry(path="a/b/c.txt", size=1024), - ArchiveEntry(path="a/d.txt", size=10), - ArchiveEntry(path="a/e", size=5, is_symlink=True, link_target="d.txt"), - ArchiveEntry(path="x/y.txt", size=0), # no "x/" entry was listed - ] - - list_entries(entries) - - assert capsys.readouterr().out.splitlines() == [ - " " * 12 + "a/", - " " * 14 + "a/b/", - " 1.00 KB" + " " * 6 + "a/b/c.txt", - " 10.00 B" + " " * 4 + "a/d.txt", - " " * 14 + "a/e -> d.txt", - " 0.00 B" + " " * 2 + "x/y.txt", - ] - - @pytest.mark.parametrize( - "flags,expect,renames", - [ - ((), EXIT_OK, True), - (("--error-on-conflict",), EXIT_FAILED, False), - (("--skip-existing",), EXIT_OK, False), - (("--overwrite",), EXIT_OK, False), - ], - ids=["rename", "error", "skip", "overwrite"], - ) - def test_extract_over_an_existing_tree( - self, - source_tree: Path, - tmp_path: Path, - make_archive: Callable[[str], Path], - flags: tuple[str, ...], - expect: int, - renames: bool, - ) -> None: - """Test that a repeat extraction renames the clashing top-level directory - by default, and follows the overwrite mode otherwise.""" - archive = make_archive("tar") - dest = tmp_path / "dest" - invoke("extract", archive, "-o", dest, "-q") - - invoke("extract", archive, "-o", dest, *flags, "-q", expect=expect) - - top_level = [source_tree.name, *([renamed(source_tree.name)] * renames)] - assert sorted(p.name for p in dest.iterdir()) == sorted(top_level) - # The whole directory is renamed, never a file inside it - names = sorted(p.name for p in source_tree.iterdir()) - for top in top_level: - assert sorted(p.name for p in (dest / top).iterdir()) == names - - @pytest.mark.skipif( - sys.platform == "win32", reason="Windows has no POSIX permission bits" - ) - @pytest.mark.parametrize("exact", [False, True], ids=["umask", "same-perms"]) - def test_same_permissions_skips_the_umask( - self, tmp_path: Path, exact: bool - ) -> None: - """Test that -p restores an entry's mode without the umask applied.""" - archive = tmp_path / "modes.tar" - make_tar(archive, file_entry("a.txt", mode=0o777)) - dest = tmp_path / "dest" - - invoke("extract", archive, "-o", dest, *(["-p"] if exact else []), "-q") - - expected = 0o777 if exact else 0o777 & ~umask() - assert (dest / "a.txt").stat().st_mode & 0o7777 == expected - - def test_symlinks_flag_reaches_the_policy(self, tmp_path: Path) -> None: - """Test that a symlink refused by default extracts with `--symlinks file`.""" - archive = tmp_path / "link.tar" - make_tar(archive, link_entry("link", "target", kind=tarfile.SYMTYPE)) - dest = tmp_path / "dest" - - invoke("extract", archive, "-o", dest, "-q", expect=EXIT_FAILED) - invoke("extract", archive, "-o", dest, "--symlinks", "file", "-q") - - assert (dest / "link").read_bytes() == b"target" - - -# Each overwrite mode against an existing output: whether the old output -# survives, and whether the new one is written beside it under a new name -CONFLICT_MODES = pytest.mark.parametrize( - "flags,expect,kept,renames", - [ - ((), EXIT_OK, True, True), - (("--error-on-conflict",), EXIT_FAILED, True, False), - (("--skip-existing",), EXIT_OK, True, False), - (("--overwrite",), EXIT_OK, False, False), - ], - ids=["rename", "error", "skip", "overwrite"], -) - - -class TestCompressConflicts: - """Test what `compress` does when its output already exists.""" - - @CONFLICT_MODES - def test_existing_output( - self, - compress_target: CompressTarget, - flags: tuple[str, ...], - expect: int, - kept: bool, - renames: bool, - ) -> None: - """Test that each overwrite mode keeps, replaces or renames around the output.""" - t = compress_target - t.dest.write_bytes(b"stale") - - invoke("compress", t.source, "-o", t.dest, *t.args, *flags, "-q", expect=expect) - - assert (t.dest.read_bytes() == b"stale") == kept - assert (t.dest.parent / renamed(t.dest.name)).is_file() == renames - - -class TestDecompressConflicts: - """Test what decompressing one file does when its output already exists.""" - - @CONFLICT_MODES - def test_existing_output( - self, - payload: Path, - tmp_path: Path, - flags: tuple[str, ...], - expect: int, - kept: bool, - renames: bool, - ) -> None: - """Test that each overwrite mode keeps, replaces or renames around the output.""" - archive = tmp_path / "out.comp" - restored = tmp_path / "restored.bin" - invoke("compress", payload, "-o", archive, "-q") - restored.write_bytes(b"stale") - - result = invoke("decompress", archive, "-o", restored, *flags, expect=expect) - - sibling = tmp_path / renamed(restored.name) - assert (f"Renamed to: {sibling}" in result.output) == renames - assert restored.read_bytes() == (b"stale" if kept else payload.read_bytes()) - assert sibling.exists() == renames - - -class TestDecompressName: - """Test that -N names a .gz file's output as its header does, as gzip -N does.""" - - @pytest.mark.parametrize( - ("flags", "expected"), [((), "renamed"), (("-N",), "orig.txt")] - ) - def test_name_flag( - self, tmp_path: Path, flags: tuple[str, ...], expected: str - ) -> None: - """Test that the stored name is used only with -N.""" - src = tmp_path / "renamed.gz" - with src.open("wb") as f, gzip.GzipFile("orig.txt", "wb", fileobj=f) as gz: - gz.write(b"payload") - - invoke("extract", src, *flags, "-q") - - assert (tmp_path / expected).read_bytes() == b"payload" - - -class TestInspectAndList: - """Test the read-only commands.""" - - def test_inspect_reports_a_valid_file(self, payload: Path, tmp_path: Path) -> None: - """Test that inspect describes a file the CLI just wrote.""" - archive = tmp_path / "out.comp" - invoke("compress", payload, "-o", archive, "-q") - - result = invoke("inspect", archive) - - assert "Valid Compresso file" in result.output - - def test_inspect_json(self, payload: Path, tmp_path: Path) -> None: - """Test that --json emits parseable output.""" - archive = tmp_path / "out.comp" - invoke("compress", payload, "-o", archive, "-q") - - result = invoke("inspect", archive, "--json") - - assert orjson.loads(result.output)["is_compresso"] is True - - def test_runs_as_a_module(self) -> None: - """Test that `python -m compresso.cli` works, which a package needs __main__ for.""" - result = subprocess.run( - [sys.executable, "-m", "compresso.cli", "list"], - capture_output=True, - text=True, - check=False, - ) - - assert result.returncode == EXIT_OK - assert "zstd" in result.stdout - - def test_list_shows_level_ranges(self) -> None: - """Test that list shows each backend's levels, and none for snappy.""" - output = invoke("list").output - - zstd = output[output.index("- zstd") :].split("\n\n")[0] - snappy = output[output.index("- snappy") :].split("\n\n")[0] - assert "Levels: 1-22" in zstd - assert "Levels: none" in snappy - - def test_inspect_archive_text(self, make_archive: Callable[[str], Path]) -> None: - """Test that an archive shows only a summary unless --entries is given.""" - archive = make_archive("zip") - - summary = invoke("inspect", archive).output - listing = invoke("inspect", archive, "--entries").output - - assert "Entries: 4" in summary - assert "f0.bin" not in summary - assert "f0.bin" in listing - - def test_inspect_archive_json(self, make_archive: Callable[[str], Path]) -> None: - """Test that --json carries entries only when --entries is given.""" - archive = make_archive("zip") - - summary = orjson.loads(invoke("inspect", archive, "--json").output) - listing = orjson.loads(invoke("inspect", archive, "--entries", "--json").output) - - assert summary["entry_count"] == 4 - assert "entries" not in summary - assert sum("f0.bin" in e["path"] for e in listing["entries"]) == 1 - - -class TestExitCodes: - """Test 0 success, 1 the operation failed, 2 the request could not be made.""" - - @pytest.mark.parametrize( - "command,name", - [ - ("compress", "nope.bin"), - ("inspect", "payload.bin"), - ("extract", "payload.bin"), - ], - ) - def test_bad_input_is_usage(self, payload: Path, command: str, name: str) -> None: - """Test that a missing or unrecognised input is wrong, rather than the - operation having failed.""" - invoke(command, payload.parent / name, expect=EXIT_USAGE) - - @pytest.mark.parametrize("command", ["compress", "extract"]) - @pytest.mark.parametrize( - "flags", - [ - ["--overwrite", "--skip-existing"], - ["--overwrite", "--error-on-conflict"], - ["--skip-existing", "--error-on-conflict"], - ], - ids=["overwrite+skip", "overwrite+error", "skip+error"], - ) - def test_mutually_exclusive_flag_usage( - self, - source_tree: Path, - tmp_path: Path, - make_archive: Callable[[str], Path], - command: str, - flags: list[str], - ) -> None: - """Test that combining any two overwrite-mode flags is contradictory.""" - if command == "compress": - args = [source_tree, "-o", tmp_path / "out.tar", "-f", "tar"] - else: - args = [make_archive("tar"), "-o", tmp_path / "d"] - - result = invoke(command, *args, *flags, expect=EXIT_USAGE) - assert "mutually exclusive" in result.output - - # An archive's level is checked against the stage that compresses - @pytest.mark.parametrize( - "compress_target,args,message", - [ - ( - "file", - ["-a", "zlib", "-l", "15"], - "zlib compression level 15 out of range (0-9", - ), - ("tree", ["-l", "5"], "tar has no compression levels"), - ], - indirect=["compress_target"], - ) - def test_out_of_range_level_is_usage( - self, compress_target: CompressTarget, args: list[str], message: str - ) -> None: - """Test that a level the backend rejects is refused up front, naming the range.""" - t = compress_target - result = invoke( - "compress", t.source, "-o", t.dest, *t.args, *args, expect=EXIT_USAGE - ) - assert message in result.output - assert not t.dest.exists() - - def test_bad_benchmark_level_is_usage(self, payload: Path) -> None: - """Test that a level that is not a number is a usage error.""" - invoke("benchmark", payload, "--levels", "nope", expect=EXIT_USAGE) - - @pytest.mark.parametrize("flags", [(), ("-f", "7z")], ids=["suffix", "flag"]) - def test_recognised_but_unsupported_archive_names_itself( - self, source_tree: Path, tmp_path: Path, flags: tuple[str, ...] - ) -> None: - """Test that 7z, asked for by -f or by `-o x.7z` alone, reports its own name - rather than being written as a zip.""" - dest = tmp_path / "o.7z" - result = invoke("compress", source_tree, "-o", dest, *flags, expect=EXIT_FAILED) - assert "7z archives are recognised but not supported yet" in result.output - assert not dest.exists() - - -class TestInterruption: - """Test Ctrl-C reports and exits 130. - - ExtendedTyper catches KeyboardInterrupt around the command callback and turns - it into a silent `Exit(130)`, so each command catches it first to say what - was interrupted. - """ - - @pytest.mark.parametrize( - "command,expected", - [ - (["compress"], "Compression cancelled"), - (["inspect"], "Inspection cancelled"), - (["list"], "Listing cancelled"), - ], - ) - def test_interrupt_is_reported( - self, - payload: Path, - monkeypatch: pytest.MonkeyPatch, - command: list[str], - expected: str, - ) -> None: - """Test that each command names itself rather than dying silently.""" - - # Interrupt the first real call each command makes - def interrupt(*args: object, **kwargs: object) -> None: - raise KeyboardInterrupt - - monkeypatch.setattr(CompressionJob, "from_file", interrupt) - monkeypatch.setattr(inspect_cmd, "inspect_file", interrupt) - monkeypatch.setattr(algos_cmd, "list_capabilities", interrupt) - - args: list[str | Path] = [*command] - if command[0] != "list": - args.append(payload) - - result = invoke(*args, expect=EXIT_CANCELLED) - - assert expected in result.output - - -class TestSmartCompress: - """Test `compress` picks the job from its inputs and the format asked for.""" - - def test_one_file_becomes_a_compresso_container(self, payload: Path) -> None: - """Test that with no format, a single file gets the .comp container.""" - invoke("compress", payload, "-q") - - assert payload.with_suffix(payload.suffix + ".comp").exists() - - @pytest.mark.parametrize("fmt", ["gz", "bz2", "xz", "zst", "lz4", "sz"]) - def test_one_file_into_a_standalone_container( - self, payload: Path, tmp_path: Path, fmt: str - ) -> None: - """Test that -f names a single-file container, the suffix follows from it, - and it comes back.""" - restored = tmp_path / "restored.bin" - - invoke("compress", payload, "-f", fmt, "-q") - invoke("extract", f"{payload}.{fmt}", "-o", restored, "-q") - - assert restored.read_bytes() == payload.read_bytes() - - def test_standalone_output_is_a_real_gzip( - self, payload: Path, tmp_path: Path - ) -> None: - """Test that the .gz written is the format, not merely the name.""" - output = tmp_path / "out.gz" - invoke("compress", payload, "-o", output, "-q") - - with gzip.open(output, "rb") as f: - assert f.read() == payload.read_bytes() - - def test_a_directory_becomes_an_archive(self, source_tree: Path) -> None: - """Test that a directory needs a container that holds many entries.""" - invoke("compress", source_tree, "-q") - - assert (source_tree.parent / f"{source_tree.name}.zip").exists() - - def test_several_files_become_an_archive( - self, payload: Path, tmp_path: Path - ) -> None: - """Test that multiple files become an archive.""" - second = tmp_path / "second.bin" - second.write_bytes(os.urandom(1024)) - output = tmp_path / "both.tar" - - invoke("compress", payload, second, "-o", output, "-q") - - with tarfile.open(output) as tar: - assert sorted(tar.getnames()) == [payload.name, second.name] - - @pytest.mark.parametrize( - "name,expected_magic", - [ - ("out.gz", b"\x1f\x8b"), - ("out.zst", b"\x28\xb5\x2f\xfd"), - ("out.zip", b"PK"), - ], - ) - def test_format_inferred_from_the_output_name( - self, payload: Path, tmp_path: Path, name: str, expected_magic: bytes - ) -> None: - """Test that a recognisable extension on -o chooses the format by itself.""" - output = tmp_path / name - - invoke("compress", payload, "-o", output, "-q") - - assert output.read_bytes().startswith(expected_magic) - - def test_explicit_format_beats_the_output_name( - self, payload: Path, tmp_path: Path - ) -> None: - """Test that -f overrides whatever the extension suggests.""" - output = tmp_path / "misleading.gz" - - invoke("compress", payload, "-o", output, "-f", "zst", "-q") - - assert output.read_bytes().startswith(b"\x28\xb5\x2f\xfd") - - @pytest.mark.parametrize( - "name", ["out.tar.zst", "out.tar.lz4", "out.tlz4", "out.tar.sz"] - ) - def test_tar_inside_a_codec_is_an_archive( - self, source_tree: Path, tmp_path: Path, name: str - ) -> None: - """Test that two-part extensions name an archive, not their codec alone.""" - output = tmp_path / name - - invoke("compress", source_tree, "-o", output, "-q") - - # Round-trips as an archive rather than as a lone compressed file - dest = tmp_path / "back" - invoke("extract", output, "-o", dest, "-q") - assert (dest / source_tree.name).is_dir() - - def test_single_file_format_refuses_several_inputs( - self, source_tree: Path, tmp_path: Path - ) -> None: - """Test that a .gz holds one file, so a directory cannot go into one.""" - result = invoke( - "compress", - source_tree, - "-f", - "gz", - "-o", - tmp_path / "o.gz", - expect=EXIT_USAGE, - ) - - assert "single file" in result.output - - def test_unknown_format_is_refused(self, payload: Path) -> None: - """Test that an unrecognised -f names neither container.""" - invoke("compress", payload, "-f", "bogus", expect=EXIT_USAGE) - - -class TestThreads: - """Test that -T reaches each kind of job, and refuses a negative count.""" - - def test_threads_reach_every_job( - self, - payload: Path, - source_tree: Path, - make_archive: Callable[[str], Path], - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """Test that compress and extract pass -T on, one file or many.""" - archive = make_archive("tar") - seen: list[int] = [] - - # Record the count each job was planned with, then stop before it runs - def record(**kwargs: Any) -> None: - options = kwargs.get("options") - seen.append(kwargs["threads"] if options is None else options.threads) - raise KeyboardInterrupt - - monkeypatch.setattr(CompressionJob, "from_file", record) - monkeypatch.setattr(ArchiveJob, "from_paths", record) - monkeypatch.setattr(DecompressionJob, "from_file", record) - monkeypatch.setattr(ExtractJob, "from_archive", record) - - for command, source in [ - ("compress", payload), - ("compress", source_tree), - ("extract", payload), - ("extract", archive), - ]: - invoke(command, source, "-T", "3", expect=EXIT_CANCELLED) - - assert seen == [3, 3, 3, 3] - - @pytest.mark.parametrize("command", ["compress", "extract"]) - def test_negative_threads_is_usage(self, payload: Path, command: str) -> None: - """Test that a count below 0 is refused before any work starts.""" - invoke(command, payload, "-T", "-1", expect=EXIT_USAGE) - - -class TestSmartExtract: - """Test `extract` picks the job from each file's own bytes.""" - - def test_dispatch_follows_the_bytes_not_the_name( - self, payload: Path, tmp_path: Path - ) -> None: - """Test that a renamed .gz is still recognised as one.""" - packed = tmp_path / "p.gz" - invoke("compress", payload, "-o", packed, "-q") - - disguised = tmp_path / "mystery.dat" - disguised.write_bytes(packed.read_bytes()) - restored = tmp_path / "back.bin" - - invoke("extract", disguised, "-o", restored, "-q") - - assert restored.read_bytes() == payload.read_bytes() - - def test_an_archive_named_like_anything_still_extracts( - self, - source_tree: Path, - tmp_path: Path, - make_archive: Callable[[str], Path], - ) -> None: - """Test that a renamed archive still extracts.""" - disguised = tmp_path / "riddle.bin" - disguised.write_bytes(make_archive("tar").read_bytes()) - dest = tmp_path / "out" - - invoke("extract", disguised, "-o", dest, "-q") - - assert (dest / source_tree.name).is_dir() - - def test_an_archive_unpacks_beside_itself( - self, - source_tree: Path, - tmp_path: Path, - make_archive: Callable[[str], Path], - ) -> None: - """Test that with no -o, entries land next to the archive, not in the cwd.""" - packed = make_archive("tar") - - # The sources are beside the archive, so they would be in the way - for f in source_tree.iterdir(): - f.unlink() - source_tree.rmdir() - - invoke("extract", packed, "-q") - - assert (tmp_path / source_tree.name).is_dir() - assert not (Path.cwd() / source_tree.name).exists() - - def test_output_directory_receives_a_single_file( - self, payload: Path, tmp_path: Path - ) -> None: - """Test that -o naming an existing directory puts the file inside it.""" - packed = tmp_path / "p.gz" - invoke("compress", payload, "-o", packed, "-q") - - into = tmp_path / "into" - into.mkdir() - - invoke("extract", packed, "-o", into, "-q") - - assert (into / "p").read_bytes() == payload.read_bytes() - - def test_several_inputs_in_one_go(self, payload: Path, tmp_path: Path) -> None: - """Test that each input is dispatched on its own.""" - first = tmp_path / "a.gz" - second = tmp_path / "b.zst" - invoke("compress", payload, "-o", first, "-q") - invoke("compress", payload, "-o", second, "-q") - - invoke("extract", first, second, "-q") - - assert (tmp_path / "a").read_bytes() == payload.read_bytes() - assert (tmp_path / "b").read_bytes() == payload.read_bytes() - - def test_listing_a_single_file_container_is_refused( - self, payload: Path, tmp_path: Path - ) -> None: - """Test that there is nothing to list in a container holding one file.""" - packed = tmp_path / "p.gz" - invoke("compress", payload, "-o", packed, "-q") - - result = invoke("extract", packed, "--list", expect=EXIT_USAGE) - - assert "nothing to list" in result.output - - -class TestMergedVerbCompatibility: - """Test that the old names still work, and the old argument order says so.""" - - def test_decompress_now_handles_archives_too( - self, - source_tree: Path, - tmp_path: Path, - make_archive: Callable[[str], Path], - ) -> None: - """Test that the merged verb does not care which name was typed.""" - dest = tmp_path / "out" - - invoke("decompress", make_archive("tar"), "-o", dest, "-q") - - for original in source_tree.iterdir(): - restored = dest / source_tree.name / original.name - assert restored.read_bytes() == original.read_bytes() diff --git a/tests/python/test_core.py b/tests/python/test_core.py index 5d175e5..1dd66c0 100644 --- a/tests/python/test_core.py +++ b/tests/python/test_core.py @@ -1,5 +1,6 @@ """Tests for the core compression/decompression functionality.""" +import gzip import importlib.util import lzma import os @@ -19,6 +20,7 @@ ExtractionPolicyError, HeaderError, IncompleteExtractionError, + SourceRemovalError, _core, compress_file, decompress_file, @@ -55,6 +57,7 @@ class TestCoreExceptions: CorruptDataError, ExtractionPolicyError, IncompleteExtractionError, + SourceRemovalError, Cancelled, ], ) @@ -83,6 +86,22 @@ def test_each_module_raises_its_own(self, tmp_path: Path) -> None: fresh.compress_file(source, tmp_path / "out.comp", cancel=token) +class TestCancelledBeforeReading: + """Test a cancel landing while a compressed archive's reader opens.""" + + def test_extract_reports_the_cancel(self, tmp_path: Path) -> None: + """Test that the cancel is raised, not lost with the failed open.""" + source = tmp_path / "f" + source.write_bytes(b"data") + archive = tmp_path / "a.tar.gz" + _core.create_archive(archive, "tar.gz", [source]) + token = _core.CancelToken() + token.cancel() + + with pytest.raises(Cancelled): + _core.extract_archive(archive, tmp_path / "out", [], cancel=token) + + class TestCapabilities: """Test the get_capabilities function.""" @@ -690,3 +709,227 @@ def test_negative_count_is_refused( str(sample_text_file), str(tmp_path / "out.zst"), "zstd", threads=-1 ) assert not (tmp_path / "out.zst").exists() + + +def _zstd_frame(*blocks: bytes, size: int | None = None) -> bytes: + """Build a zstd frame from its encoded blocks, stating `size` if given.""" + magic = b"\x28\xb5\x2f\xfd" + if size is None: + return magic + b"\x00\x50" + b"".join(blocks) # Descriptor, then window + + return magic + b"\xe0" + size.to_bytes(8, "little") + b"".join(blocks) + + +def _zstd_block(payload: bytes, *, last: bool = True) -> bytes: + """Encode a raw zstd block.""" + return ((len(payload) << 3) | last).to_bytes(3, "little") + payload + + +def _lz4_frame( + *blocks: bytes, size: int | None = 3, flg: int = 0, extra: bytes = b"" +) -> bytes: + """Build an lz4 frame of stored blocks, stating `size` if given. + + `extra` follows the content size, for a dictionary ID; every checksum `flg` + asks for is left as zero, as nothing here checks one. + """ + flg |= 0x60 | (0x08 if size is not None else 0) # Version 01, independent + header = bytes([flg, 0x40]) + if size is not None: + header += size.to_bytes(8, "little") + block_checksum = b"\0" * 4 if flg & 0x10 else b"" + stored = b"".join( + (len(block) | 0x8000_0000).to_bytes(4, "little") + block + block_checksum + for block in blocks + ) + content_checksum = b"\0" * 4 if flg & 0x04 else b"" + return ( + b"\x04\x22\x4d\x18" + + header + + extra + + b"\x00" + + stored + + b"\0" * 4 + + content_checksum + ) + + +def _snappy_chunk(kind: int, body: bytes) -> bytes: + """Frame a snappy chunk, its checksum left as zero.""" + return bytes([kind]) + (4 + len(body)).to_bytes(3, "little") + b"\0" * 4 + body + + +SNAPPY_ID = b"\xff\x06\x00\x00sNaPpY" +# A block of one literal, which opens with its decoded length +SNAPPY_ABC = _snappy_chunk(0x00, b"\x03\x08abc") +SNAPPY_DE = _snappy_chunk(0x01, b"de") + + +class TestStoredContentSize: + """Test the size read from a file's metadata, which nothing decodes.""" + + FRAME = _zstd_frame(_zstd_block(b"abc"), size=3) + SKIPPABLE = b"\x50\x2a\x4d\x18" + (2).to_bytes(4, "little") + b"xx" + LZ4 = _lz4_frame(b"abc") + + @pytest.mark.parametrize( + ("data", "size"), + [ + (LZ4, 3), + (LZ4 + LZ4, 6), + (SKIPPABLE + LZ4 + SKIPPABLE, 3), + (_lz4_frame(b"ab", b"c"), 3), + # Block and content checksums, and a dictionary ID + (_lz4_frame(b"ab", b"c", flg=0x15, extra=b"\x01\0\0\0"), 3), + ], + ) + def test_lz4_frames_are_walked( + self, tmp_path: Path, data: bytes, size: int + ) -> None: + """Test the sum over every frame, whatever each one's layout.""" + path = tmp_path / "f.lz4" + path.write_bytes(data) + + assert _core.stored_content_size(path) == size + + @pytest.mark.parametrize( + "data", + [ + _lz4_frame(b"abc", size=None), # Written from a stream + LZ4 + _lz4_frame(b"abc", size=None), + LZ4[:-1], + LZ4 + b"junk", + LZ4 + LZ4[:10], + ], + ) + def test_lz4_states_no_size_unless_fully_described( + self, tmp_path: Path, data: bytes + ) -> None: + """Test that a partial sum is never reported as the size.""" + path = tmp_path / "f.lz4" + path.write_bytes(data) + + assert _core.stored_content_size(path) is None + + @pytest.mark.parametrize( + ("data", "size"), + [ + (SNAPPY_ID + SNAPPY_ABC, 3), + (SNAPPY_ID + SNAPPY_ABC + SNAPPY_DE, 5), + (SNAPPY_ID + b"\xfe\x02\x00\x00xx" + SNAPPY_DE, 2), # Padding + (SNAPPY_ID + SNAPPY_ABC + SNAPPY_ID + SNAPPY_DE, 5), # Concatenated + (SNAPPY_ID, 0), + ], + ) + def test_snappy_chunks_are_walked( + self, tmp_path: Path, data: bytes, size: int + ) -> None: + """Test the sum over every data chunk, skipping the rest.""" + path = tmp_path / "f.sz" + path.write_bytes(data) + + assert _core.stored_content_size(path) == size + + @pytest.mark.parametrize( + "data", + [ + SNAPPY_ID + SNAPPY_ABC[:-1], + SNAPPY_ID + SNAPPY_ABC + b"j", + SNAPPY_ID + _snappy_chunk(0x02, b""), # Reserved unskippable + SNAPPY_ID + _snappy_chunk(0x00, b"\x81\x80\x04"), # States 65537 + SNAPPY_ID + _snappy_chunk(0x01, b"x" * 65537), + ], + # The bytes as an ID overflow Windows' 32767-character PYTEST_CURRENT_TEST + ids=["truncated", "trailing", "reserved", "states-65537", "chunk-65537"], + ) + def test_snappy_states_no_size_unless_fully_described( + self, tmp_path: Path, data: bytes + ) -> None: + """Test that a partial sum, or one no decoder accepts, is never stated.""" + path = tmp_path / "f.sz" + path.write_bytes(data) + + assert _core.stored_content_size(path) is None + + @pytest.mark.parametrize( + ("data", "size"), + [ + (FRAME, 3), + (FRAME + FRAME, 6), + (SKIPPABLE + FRAME + SKIPPABLE, 3), + (_zstd_frame(_zstd_block(b"ab", last=False), _zstd_block(b"c"), size=3), 3), + (_zstd_frame(b"\x0b\x00\x00a", size=1), 1), # RLE, stores one byte + # Single segment with a one-byte size field + (b"\x28\xb5\x2f\xfd\x20\x03" + _zstd_block(b"abc"), 3), + # Content checksum follows the blocks + ( + b"\x28\xb5\x2f\xfd\x24\x03" + _zstd_block(b"abc") + b"\xaa" * 4, + 3, + ), + # Window byte, then a one-byte dictionary ID, then an eight-byte size + ( + b"\x28\xb5\x2f\xfd\xc1\x50\x07" + + (3).to_bytes(8, "little") + + _zstd_block(b"abc"), + 3, + ), + ], + ) + def test_zstd_frames_are_walked( + self, tmp_path: Path, data: bytes, size: int | None + ) -> None: + """Test the sum over every frame, whatever each one's layout.""" + path = tmp_path / "f.zst" + path.write_bytes(data) + + assert _core.stored_content_size(path) == size + + @pytest.mark.parametrize( + "data", + [ + _zstd_frame(_zstd_block(b"abc")), # Written from a stream + FRAME + _zstd_frame(_zstd_block(b"abc")), + FRAME[:-1], + FRAME + b"junk", + FRAME + FRAME[:6], + FRAME + SKIPPABLE[:-1], + ], + ) + def test_zstd_states_no_size_unless_fully_described( + self, tmp_path: Path, data: bytes + ) -> None: + """Test that a partial sum is never reported as the size.""" + path = tmp_path / "f.zst" + path.write_bytes(data) + + assert _core.stored_content_size(path) is None + + def test_xz_sums_every_stream(self, tmp_path: Path) -> None: + """Test concatenated streams and the padding allowed between them.""" + path = tmp_path / "f.xz" + path.write_bytes( + lzma.compress(b"a" * 100) + b"\0" * 4 + lzma.compress(b"b" * 200) + ) + + assert _core.stored_content_size(path) == 300 + + @pytest.mark.parametrize( + "damage", + [lambda d: d[:-1], lambda d: d + b"junk"], + ids=["truncated", "trailing"], + ) + def test_xz_states_no_size_unless_fully_described( + self, tmp_path: Path, damage: Callable[[bytes], bytes] + ) -> None: + """Test that a cut or extended file has no footer where it should be.""" + path = tmp_path / "f.xz" + path.write_bytes(damage(lzma.compress(b"a" * 100))) + + assert _core.stored_content_size(path) is None + + def test_other_formats_state_nothing(self, tmp_path: Path) -> None: + """Test that a format with no such metadata gets no answer.""" + path = tmp_path / "f.gz" + path.write_bytes(gzip.compress(b"data")) + + assert _core.stored_content_size(path) is None diff --git a/vcpkg.json b/vcpkg.json index 21ed388..4e269d0 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -4,13 +4,13 @@ "description": "C libraries the compresso._core extension links against on Windows", "builtin-baseline": "04a9d8e5212d01ee1dd9478eadd9caade4f8b0d4", "dependencies": [ - "zlib", "bzip2", + "libarchive", "liblzma", - "zstd", + "libzip", "lz4", "snappy", - "libzip", - "libarchive" + "zlib", + "zstd" ] }