From 307cf365fd81c1057b5d30d96de44047aae93192 Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Thu, 22 Oct 2026 13:50:23 +0100 Subject: [PATCH 01/31] test(cli): split test_cli.py into a package per command - New 'tests/python/cli/' now has a module per command: test_compress.py, test_extract.py, test_inspect.py, and test_list.py - New test_app.py holds the shared formatters, app structure, progress bar, exit codes, interruption, and threads - New CLI conftest.py has the shared fixtures and helpers used in CLI tests - 'TestInspectAndList' becomes 'TestInspect' and 'TestList', one per module --- tests/python/cli/__init__.py | 0 tests/python/cli/conftest.py | 166 +++++ tests/python/cli/test_app.py | 352 ++++++++++ tests/python/cli/test_compress.py | 193 ++++++ tests/python/cli/test_extract.py | 309 +++++++++ tests/python/cli/test_inspect.py | 54 ++ tests/python/cli/test_list.py | 37 ++ tests/python/test_cli.py | 1034 ----------------------------- 8 files changed, 1111 insertions(+), 1034 deletions(-) create mode 100644 tests/python/cli/__init__.py create mode 100644 tests/python/cli/conftest.py create mode 100644 tests/python/cli/test_app.py create mode 100644 tests/python/cli/test_compress.py create mode 100644 tests/python/cli/test_extract.py create mode 100644 tests/python/cli/test_inspect.py create mode 100644 tests/python/cli/test_list.py delete mode 100644 tests/python/test_cli.py diff --git a/tests/python/cli/__init__.py b/tests/python/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/python/cli/conftest.py b/tests/python/cli/conftest.py new file mode 100644 index 0000000..d600291 --- /dev/null +++ b/tests/python/cli/conftest.py @@ -0,0 +1,166 @@ +"""Shared helpers and fixtures for the CLI tests.""" + +from __future__ import annotations + +import os +from collections.abc import Callable, Generator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path + +import pytest +from typer.testing import CliRunner, Result + +import compresso.cli.compress as compress_cmd +import compresso.cli.extract as extract_cmd +from compresso.cli import app +from compresso.cli._render import ( + BAR_THRESHOLD, + EXIT_FAILED, + EXIT_OK, + progress_bar, +) +from compresso.frontend._job import ProgressCallback +from compresso.frontend.archive_api import ( + ArchiveJob, + ArchiveOptions, +) + +runner = CliRunner() + +# Incompressible throughout, so compressed sizes stay close to these +PAYLOAD_SIZE = 64 * 1024 +TREE_FILE_SIZE = 16 * 1024 + +# Past the progress bar threshold, for the tests that cover the bar +LARGE_PAYLOAD_SIZE = 2 * BAR_THRESHOLD + + +def invoke(*args: str | Path, expect: int | None = EXIT_OK) -> Result: + """Run the CLI, asserting it exits with `expect`. + + Args: + *args: The command line, without the program name. + expect: The exit code to assert, or None to leave it unchecked. + + Returns: + The runner's result. + """ + result = runner.invoke(app, [str(arg) for arg in args]) + if expect is not None: + assert result.exit_code == expect, result.output + + return result + + +@pytest.fixture +def payload(tmp_path: Path) -> Path: + """A small incompressible file, below the progress bar threshold.""" + path = tmp_path / "payload.bin" + path.write_bytes(os.urandom(PAYLOAD_SIZE)) + + return path + + +@pytest.fixture +def source_tree(tmp_path: Path) -> Path: + """A small directory tree to archive, function-scoped because some tests delete it.""" + root = tmp_path / "tree" + root.mkdir() + for i in range(3): + (root / f"f{i}.bin").write_bytes(os.urandom(TREE_FILE_SIZE)) + + return root + + +@pytest.fixture +def make_archive(source_tree: Path, tmp_path: Path) -> Callable[[str], Path]: + """Build an archive of `source_tree` as input for a test. + + Built through the API rather than the CLI, and checked, so a broken setup + cannot surface as a confusing failure later in the test. + """ + + def make(fmt: str) -> Path: + archive = tmp_path / f"out.{fmt}" + options = ArchiveOptions(format=fmt) + result = ArchiveJob.from_paths([source_tree], archive, options).run() + assert result.ok, result.error + return archive + + return make + + +@dataclass +class CompressTarget: + """What one `compress` call reads and writes.""" + + source: Path + dest: Path + args: tuple[str, ...] + + +@pytest.fixture(params=["file", "tree"]) +def compress_target( + request: pytest.FixtureRequest, payload: Path, source_tree: Path, tmp_path: Path +) -> CompressTarget: + """A single file into a .comp container, or a tree into a tar archive. + + The two take different paths through `compress`, but share its + overwrite-mode handling. + """ + if request.param == "file": + return CompressTarget(payload, tmp_path / "out.comp", ()) + + return CompressTarget(source_tree, tmp_path / "out.tar", ("-f", "tar")) + + +@dataclass +class Bar: + """One progress bar a command opened.""" + + enabled: bool + updates: int = 0 + + +@pytest.fixture +def bars(monkeypatch: pytest.MonkeyPatch) -> list[Bar]: + """Record each progress bar a command opens, in order, still drawing the real one.""" + opened: list[Bar] = [] + + @contextmanager + def spy( + label: str, total: int, *, enabled: bool + ) -> Generator[ProgressCallback | None]: + bar = Bar(enabled) + opened.append(bar) + with progress_bar(label, total, enabled=enabled) as on_progress: + if on_progress is None: + yield None + return + + def counting(fraction: float, done: int, reported_total: int) -> None: + bar.updates += 1 + on_progress(fraction, done, reported_total) + + yield counting + + # Each command imports progress_bar by name, so each binding is patched + monkeypatch.setattr(compress_cmd, "progress_bar", spy) + monkeypatch.setattr(extract_cmd, "progress_bar", spy) + + return opened + + +# Each overwrite mode against an existing output: whether the old output +# survives, and whether the new one is written beside it under a new name +CONFLICT_MODES = pytest.mark.parametrize( + "flags,expect,kept,renames", + [ + ((), EXIT_OK, True, True), + (("--error-on-conflict",), EXIT_FAILED, True, False), + (("--skip-existing",), EXIT_OK, True, False), + (("--overwrite",), EXIT_OK, False, False), + ], + ids=["rename", "error", "skip", "overwrite"], +) diff --git a/tests/python/cli/test_app.py b/tests/python/cli/test_app.py new file mode 100644 index 0000000..5206d77 --- /dev/null +++ b/tests/python/cli/test_app.py @@ -0,0 +1,352 @@ +"""Tests for the CLI app as a whole, and what every command shares.""" + +from __future__ import annotations + +import os +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +import compresso.cli.algos as algos_cmd +import compresso.cli.inspect as inspect_cmd +from compresso.cli._render import ( + EXIT_CANCELLED, + EXIT_FAILED, + EXIT_USAGE, + format_size, + format_time, +) +from compresso.frontend.api import CompressionJob, DecompressionJob +from compresso.frontend.archive_api import ( + ArchiveJob, + ExtractJob, +) + +from .conftest import LARGE_PAYLOAD_SIZE, PAYLOAD_SIZE, Bar, CompressTarget, invoke + + +class TestFormatters: + """Test utility formatting functions.""" + + @pytest.mark.parametrize( + "size,expected", + [ + (0, "0.00 B"), + (500, "500.00 B"), + (1024, "1.00 KB"), + (1536, "1.50 KB"), + (1048576, "1.00 MB"), + (1073741824, "1.00 GB"), + (1099511627776, "1.00 TB"), + ], + ) + def test_format_size(self, size: int, expected: str): + """Test size formatting with various values.""" + result = format_size(size) + assert result == expected + + @pytest.mark.parametrize( + "seconds,expected", + [ + (0.0005, "0ms"), + (0.5, "500ms"), + (1.0, "1.00s"), + (5.5, "5.50s"), + (59.9, "59.90s"), + (60.0, "1m 0.0s"), + (90.5, "1m 30.5s"), + (125.0, "2m 5.0s"), + ], + ) + def test_format_time(self, seconds: float, expected: str): + """Test time formatting with various values.""" + assert format_time(seconds) == expected + + +class TestAppStructure: + """Test the package assembles into one app with every command registered.""" + + def test_commands_are_listed_in_workflow_order(self) -> None: + """Test that the order is chosen in `__init__`, not inherited from import sorting. + + Alphabetical imports would put `list` first. + """ + output = invoke("--help").output + + # Matched on the alias column, which is unique per row; a bare command + # name also appears inside other commands' descriptions + positions = [ + output.index(aliases) + for aliases in [ + "(c, comp", + "(x, ex", + "(i, info)", + "(b, bench)", + "(l, ls)", + ] + ] + assert positions == sorted(positions) + + @pytest.mark.parametrize( + "alias,command", + [ + ("c", "compress"), + ("comp", "compress"), + ("archive", "compress"), + ("a", "compress"), + ("ar", "compress"), + ("x", "extract"), + ("ex", "extract"), + ("decompress", "extract"), + ("d", "extract"), + ("decomp", "extract"), + ("i", "inspect"), + ("info", "inspect"), + ("b", "benchmark"), + ("bench", "benchmark"), + ("l", "list"), + ("ls", "list"), + ], + ) + def test_aliases_still_resolve(self, alias: str, command: str) -> None: + """Test that each short name reaches the command it stands for. + + Help is compared without its usage line, which echoes the name typed. + """ + + def help_body(name: str) -> list[str]: + lines = invoke(name, "--help").output.splitlines() + return [line for line in lines if "Usage:" not in line] + + assert help_body(alias) == help_body(command) + + +class TestProgressBar: + """Test the bar is drawn only for a large job that is not asked to be quiet.""" + + @pytest.mark.parametrize( + "size,quiet,expected", + [ + (LARGE_PAYLOAD_SIZE, False, True), + (LARGE_PAYLOAD_SIZE, True, False), + (PAYLOAD_SIZE, False, False), + ], + ids=["large", "large-quiet", "small"], + ) + def test_compress_bar_follows_size_and_quiet( + self, tmp_path: Path, bars: list[Bar], size: int, quiet: bool, expected: bool + ) -> None: + """Test that the bar needs both a job past the threshold and no --quiet.""" + source = tmp_path / "source.bin" + source.write_bytes(os.urandom(size)) + + invoke( + "compress", source, "-o", tmp_path / "out.comp", *(["-q"] if quiet else []) + ) + + assert [bar.enabled for bar in bars] == [expected] + + def test_file_round_trip_draws_both_bars( + self, tmp_path: Path, bars: list[Bar] + ) -> None: + """Test that compress and extract each drive a bar with real progress.""" + source = tmp_path / "source.bin" + source.write_bytes(os.urandom(LARGE_PAYLOAD_SIZE)) + packed = tmp_path / "out.comp" + restored = tmp_path / "restored.bin" + + invoke("compress", source, "-o", packed) + invoke("extract", packed, "-o", restored) + + assert [bar.enabled for bar in bars] == [True, True] + assert all(bar.updates > 0 for bar in bars) + assert restored.read_bytes() == source.read_bytes() + + def test_archive_round_trip_draws_both_bars( + self, tmp_path: Path, bars: list[Bar] + ) -> None: + """Test that archiving and extracting a tree each drive a bar too.""" + tree = tmp_path / "big_tree" + tree.mkdir() + for i in range(2): + (tree / f"f{i}.bin").write_bytes(os.urandom(LARGE_PAYLOAD_SIZE // 2)) + archive = tmp_path / "out.tar" + dest = tmp_path / "dest" + + invoke("compress", tree, "-o", archive, "-f", "tar") + invoke("extract", archive, "-o", dest) + + assert [bar.enabled for bar in bars] == [True, True] + assert all(bar.updates > 0 for bar in bars) + for original in tree.iterdir(): + assert (dest / tree.name / original.name).read_bytes() == ( + original.read_bytes() + ) + + +class TestExitCodes: + """Test 0 success, 1 the operation failed, 2 the request could not be made.""" + + @pytest.mark.parametrize( + "command,name", + [ + ("compress", "nope.bin"), + ("inspect", "payload.bin"), + ("extract", "payload.bin"), + ], + ) + def test_bad_input_is_usage(self, payload: Path, command: str, name: str) -> None: + """Test that a missing or unrecognised input is wrong, rather than the + operation having failed.""" + invoke(command, payload.parent / name, expect=EXIT_USAGE) + + @pytest.mark.parametrize("command", ["compress", "extract"]) + @pytest.mark.parametrize( + "flags", + [ + ["--overwrite", "--skip-existing"], + ["--overwrite", "--error-on-conflict"], + ["--skip-existing", "--error-on-conflict"], + ], + ids=["overwrite+skip", "overwrite+error", "skip+error"], + ) + def test_mutually_exclusive_flag_usage( + self, + source_tree: Path, + tmp_path: Path, + make_archive: Callable[[str], Path], + command: str, + flags: list[str], + ) -> None: + """Test that combining any two overwrite-mode flags is contradictory.""" + if command == "compress": + args = [source_tree, "-o", tmp_path / "out.tar", "-f", "tar"] + else: + args = [make_archive("tar"), "-o", tmp_path / "d"] + + result = invoke(command, *args, *flags, expect=EXIT_USAGE) + assert "mutually exclusive" in result.output + + # An archive's level is checked against the stage that compresses + @pytest.mark.parametrize( + "compress_target,args,message", + [ + ( + "file", + ["-a", "zlib", "-l", "15"], + "zlib compression level 15 out of range (0-9", + ), + ("tree", ["-l", "5"], "tar has no compression levels"), + ], + indirect=["compress_target"], + ) + def test_out_of_range_level_is_usage( + self, compress_target: CompressTarget, args: list[str], message: str + ) -> None: + """Test that a level the backend rejects is refused up front, naming the range.""" + t = compress_target + result = invoke( + "compress", t.source, "-o", t.dest, *t.args, *args, expect=EXIT_USAGE + ) + assert message in result.output + assert not t.dest.exists() + + def test_bad_benchmark_level_is_usage(self, payload: Path) -> None: + """Test that a level that is not a number is a usage error.""" + invoke("benchmark", payload, "--levels", "nope", expect=EXIT_USAGE) + + @pytest.mark.parametrize("flags", [(), ("-f", "7z")], ids=["suffix", "flag"]) + def test_recognised_but_unsupported_archive_names_itself( + self, source_tree: Path, tmp_path: Path, flags: tuple[str, ...] + ) -> None: + """Test that 7z, asked for by -f or by `-o x.7z` alone, reports its own name + rather than being written as a zip.""" + dest = tmp_path / "o.7z" + result = invoke("compress", source_tree, "-o", dest, *flags, expect=EXIT_FAILED) + assert "7z archives are recognised but not supported yet" in result.output + assert not dest.exists() + + +class TestInterruption: + """Test Ctrl-C reports and exits 130. + + ExtendedTyper catches KeyboardInterrupt around the command callback and turns + it into a silent `Exit(130)`, so each command catches it first to say what + was interrupted. + """ + + @pytest.mark.parametrize( + "command,expected", + [ + (["compress"], "Compression cancelled"), + (["inspect"], "Inspection cancelled"), + (["list"], "Listing cancelled"), + ], + ) + def test_interrupt_is_reported( + self, + payload: Path, + monkeypatch: pytest.MonkeyPatch, + command: list[str], + expected: str, + ) -> None: + """Test that each command names itself rather than dying silently.""" + + # Interrupt the first real call each command makes + def interrupt(*args: object, **kwargs: object) -> None: + raise KeyboardInterrupt + + monkeypatch.setattr(CompressionJob, "from_file", interrupt) + monkeypatch.setattr(inspect_cmd, "inspect_file", interrupt) + monkeypatch.setattr(algos_cmd, "list_capabilities", interrupt) + + args: list[str | Path] = [*command] + if command[0] != "list": + args.append(payload) + + result = invoke(*args, expect=EXIT_CANCELLED) + + assert expected in result.output + + +class TestThreads: + """Test that -T reaches each kind of job, and refuses a negative count.""" + + def test_threads_reach_every_job( + self, + payload: Path, + source_tree: Path, + make_archive: Callable[[str], Path], + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Test that compress and extract pass -T on, one file or many.""" + archive = make_archive("tar") + seen: list[int] = [] + + # Record the count each job was planned with, then stop before it runs + def record(**kwargs: Any) -> None: + options = kwargs.get("options") + seen.append(kwargs["threads"] if options is None else options.threads) + raise KeyboardInterrupt + + monkeypatch.setattr(CompressionJob, "from_file", record) + monkeypatch.setattr(ArchiveJob, "from_paths", record) + monkeypatch.setattr(DecompressionJob, "from_file", record) + monkeypatch.setattr(ExtractJob, "from_archive", record) + + for command, source in [ + ("compress", payload), + ("compress", source_tree), + ("extract", payload), + ("extract", archive), + ]: + invoke(command, source, "-T", "3", expect=EXIT_CANCELLED) + + assert seen == [3, 3, 3, 3] + + @pytest.mark.parametrize("command", ["compress", "extract"]) + def test_negative_threads_is_usage(self, payload: Path, command: str) -> None: + """Test that a count below 0 is refused before any work starts.""" + invoke(command, payload, "-T", "-1", expect=EXIT_USAGE) diff --git a/tests/python/cli/test_compress.py b/tests/python/cli/test_compress.py new file mode 100644 index 0000000..7a060f7 --- /dev/null +++ b/tests/python/cli/test_compress.py @@ -0,0 +1,193 @@ +"""Tests for `compresso compress`.""" + +from __future__ import annotations + +import gzip +import os +import tarfile +from pathlib import Path + +import pytest + +from compresso.cli._render import ( + EXIT_USAGE, +) + +from ..helpers import renamed +from .conftest import CONFLICT_MODES, CompressTarget, invoke + + +class TestCompressRoundTrip: + """Test compress and decompress, end to end.""" + + def test_compress_then_decompress(self, payload: Path, tmp_path: Path) -> None: + """Test that a file survives a round trip through the CLI unchanged.""" + archive = tmp_path / "out.comp" + restored = tmp_path / "restored.bin" + + result = invoke("compress", payload, "-o", archive) + assert "Compression successful" in result.output + + result = invoke("decompress", archive, "-o", restored) + assert "Decompression successful" in result.output + + assert restored.read_bytes() == payload.read_bytes() + + def test_quiet_suppresses_output(self, payload: Path, tmp_path: Path) -> None: + """Test that --quiet prints nothing on success.""" + result = invoke("compress", payload, "-o", tmp_path / "q.comp", "-q") + assert result.output.strip() == "" + + # The backends are covered in core and C tests; this checks the flags + @pytest.mark.parametrize( + "args", [("-a", "bzip2"), ("-a", "zstd", "-l", "19")], ids=["algo", "level"] + ) + def test_algorithms_round_trip( + self, payload: Path, tmp_path: Path, args: tuple[str, ...] + ) -> None: + """Test that -a and -l reach the backend.""" + archive = tmp_path / "out.comp" + restored = tmp_path / "restored.bin" + + invoke("compress", payload, "-o", archive, *args, "-q") + invoke("decompress", archive, "-o", restored, "-q") + + assert restored.read_bytes() == payload.read_bytes() + + +class TestCompressConflicts: + """Test what `compress` does when its output already exists.""" + + @CONFLICT_MODES + def test_existing_output( + self, + compress_target: CompressTarget, + flags: tuple[str, ...], + expect: int, + kept: bool, + renames: bool, + ) -> None: + """Test that each overwrite mode keeps, replaces or renames around the output.""" + t = compress_target + t.dest.write_bytes(b"stale") + + invoke("compress", t.source, "-o", t.dest, *t.args, *flags, "-q", expect=expect) + + assert (t.dest.read_bytes() == b"stale") == kept + assert (t.dest.parent / renamed(t.dest.name)).is_file() == renames + + +class TestSmartCompress: + """Test `compress` picks the job from its inputs and the format asked for.""" + + def test_one_file_becomes_a_compresso_container(self, payload: Path) -> None: + """Test that with no format, a single file gets the .comp container.""" + invoke("compress", payload, "-q") + + assert payload.with_suffix(payload.suffix + ".comp").exists() + + @pytest.mark.parametrize("fmt", ["gz", "bz2", "xz", "zst", "lz4", "sz"]) + def test_one_file_into_a_standalone_container( + self, payload: Path, tmp_path: Path, fmt: str + ) -> None: + """Test that -f names a single-file container, the suffix follows from it, + and it comes back.""" + restored = tmp_path / "restored.bin" + + invoke("compress", payload, "-f", fmt, "-q") + invoke("extract", f"{payload}.{fmt}", "-o", restored, "-q") + + assert restored.read_bytes() == payload.read_bytes() + + def test_standalone_output_is_a_real_gzip( + self, payload: Path, tmp_path: Path + ) -> None: + """Test that the .gz written is the format, not merely the name.""" + output = tmp_path / "out.gz" + invoke("compress", payload, "-o", output, "-q") + + with gzip.open(output, "rb") as f: + assert f.read() == payload.read_bytes() + + def test_a_directory_becomes_an_archive(self, source_tree: Path) -> None: + """Test that a directory needs a container that holds many entries.""" + invoke("compress", source_tree, "-q") + + assert (source_tree.parent / f"{source_tree.name}.zip").exists() + + def test_several_files_become_an_archive( + self, payload: Path, tmp_path: Path + ) -> None: + """Test that multiple files become an archive.""" + second = tmp_path / "second.bin" + second.write_bytes(os.urandom(1024)) + output = tmp_path / "both.tar" + + invoke("compress", payload, second, "-o", output, "-q") + + with tarfile.open(output) as tar: + assert sorted(tar.getnames()) == [payload.name, second.name] + + @pytest.mark.parametrize( + "name,expected_magic", + [ + ("out.gz", b"\x1f\x8b"), + ("out.zst", b"\x28\xb5\x2f\xfd"), + ("out.zip", b"PK"), + ], + ) + def test_format_inferred_from_the_output_name( + self, payload: Path, tmp_path: Path, name: str, expected_magic: bytes + ) -> None: + """Test that a recognisable extension on -o chooses the format by itself.""" + output = tmp_path / name + + invoke("compress", payload, "-o", output, "-q") + + assert output.read_bytes().startswith(expected_magic) + + def test_explicit_format_beats_the_output_name( + self, payload: Path, tmp_path: Path + ) -> None: + """Test that -f overrides whatever the extension suggests.""" + output = tmp_path / "misleading.gz" + + invoke("compress", payload, "-o", output, "-f", "zst", "-q") + + assert output.read_bytes().startswith(b"\x28\xb5\x2f\xfd") + + @pytest.mark.parametrize( + "name", ["out.tar.zst", "out.tar.lz4", "out.tlz4", "out.tar.sz"] + ) + def test_tar_inside_a_codec_is_an_archive( + self, source_tree: Path, tmp_path: Path, name: str + ) -> None: + """Test that two-part extensions name an archive, not their codec alone.""" + output = tmp_path / name + + invoke("compress", source_tree, "-o", output, "-q") + + # Round-trips as an archive rather than as a lone compressed file + dest = tmp_path / "back" + invoke("extract", output, "-o", dest, "-q") + assert (dest / source_tree.name).is_dir() + + def test_single_file_format_refuses_several_inputs( + self, source_tree: Path, tmp_path: Path + ) -> None: + """Test that a .gz holds one file, so a directory cannot go into one.""" + result = invoke( + "compress", + source_tree, + "-f", + "gz", + "-o", + tmp_path / "o.gz", + expect=EXIT_USAGE, + ) + + assert "single file" in result.output + + def test_unknown_format_is_refused(self, payload: Path) -> None: + """Test that an unrecognised -f names neither container.""" + invoke("compress", payload, "-f", "bogus", expect=EXIT_USAGE) diff --git a/tests/python/cli/test_extract.py b/tests/python/cli/test_extract.py new file mode 100644 index 0000000..6aeaa59 --- /dev/null +++ b/tests/python/cli/test_extract.py @@ -0,0 +1,309 @@ +"""Tests for `compresso extract`.""" + +from __future__ import annotations + +import gzip +import sys +import tarfile +from collections.abc import Callable +from pathlib import Path + +import pytest + +from compresso.cli._render import ( + EXIT_FAILED, + EXIT_OK, + EXIT_USAGE, + format_size, +) +from compresso.cli.extract import list_entries +from compresso.frontend.archive_api import ( + ArchiveEntry, +) + +from ..helpers import file_entry, link_entry, make_tar, renamed, umask +from .conftest import CONFLICT_MODES, TREE_FILE_SIZE, invoke + + +class TestArchiveRoundTrip: + """Test archive and extract, end to end.""" + + @pytest.mark.parametrize("fmt", ["tar", "tar.gz", "tar.zst", "zip"]) + def test_archive_then_extract( + self, source_tree: Path, tmp_path: Path, fmt: str + ) -> None: + """Test that a tree survives a round trip through each archive format.""" + archive = tmp_path / f"out.{fmt}" + dest = tmp_path / f"dest.{fmt}" + + result = invoke("compress", source_tree, "-o", archive, "-f", fmt) + assert "Archive created" in result.output + + result = invoke("extract", archive, "-o", dest) + assert "Extraction successful" in result.output + + for original in source_tree.iterdir(): + restored = dest / source_tree.name / original.name + assert restored.read_bytes() == original.read_bytes() + + def test_extract_list_only( + self, + tmp_path: Path, + make_archive: Callable[[str], Path], + ) -> None: + """Test that --list prints each entry's size and path without writing anything.""" + archive = make_archive("tar") + dest = tmp_path / "dest" + + result = invoke("extract", archive, "--list", "-o", dest) + + lines = result.output.splitlines() + f0_line = next(line for line in lines if "f0.bin" in line) + assert f0_line.split()[:2] == format_size(TREE_FILE_SIZE).split() + assert not dest.exists() + + # Directories have a blank size; their contents are indented beneath them + dir_line = next(line for line in lines if line.rstrip("/").endswith("tree")) + assert dir_line == f"{'':>10} {dir_line.strip()}" + assert f0_line.endswith(" tree/f0.bin") + + def test_extract_list_indents_by_depth( + self, capsys: pytest.CaptureFixture[str] + ) -> None: + """Test that --list indents only under directories it has listed.""" + entries = [ + ArchiveEntry(path="a/", is_dir=True), + ArchiveEntry(path="a/b/", is_dir=True), + ArchiveEntry(path="a/b/c.txt", size=1024), + ArchiveEntry(path="a/d.txt", size=10), + ArchiveEntry(path="a/e", size=5, is_symlink=True, link_target="d.txt"), + ArchiveEntry(path="x/y.txt", size=0), # no "x/" entry was listed + ] + + list_entries(entries) + + assert capsys.readouterr().out.splitlines() == [ + " " * 12 + "a/", + " " * 14 + "a/b/", + " 1.00 KB" + " " * 6 + "a/b/c.txt", + " 10.00 B" + " " * 4 + "a/d.txt", + " " * 14 + "a/e -> d.txt", + " 0.00 B" + " " * 2 + "x/y.txt", + ] + + @pytest.mark.parametrize( + "flags,expect,renames", + [ + ((), EXIT_OK, True), + (("--error-on-conflict",), EXIT_FAILED, False), + (("--skip-existing",), EXIT_OK, False), + (("--overwrite",), EXIT_OK, False), + ], + ids=["rename", "error", "skip", "overwrite"], + ) + def test_extract_over_an_existing_tree( + self, + source_tree: Path, + tmp_path: Path, + make_archive: Callable[[str], Path], + flags: tuple[str, ...], + expect: int, + renames: bool, + ) -> None: + """Test that a repeat extraction renames the clashing top-level directory + by default, and follows the overwrite mode otherwise.""" + archive = make_archive("tar") + dest = tmp_path / "dest" + invoke("extract", archive, "-o", dest, "-q") + + invoke("extract", archive, "-o", dest, *flags, "-q", expect=expect) + + top_level = [source_tree.name, *([renamed(source_tree.name)] * renames)] + assert sorted(p.name for p in dest.iterdir()) == sorted(top_level) + # The whole directory is renamed, never a file inside it + names = sorted(p.name for p in source_tree.iterdir()) + for top in top_level: + assert sorted(p.name for p in (dest / top).iterdir()) == names + + @pytest.mark.skipif( + sys.platform == "win32", reason="Windows has no POSIX permission bits" + ) + @pytest.mark.parametrize("exact", [False, True], ids=["umask", "same-perms"]) + def test_same_permissions_skips_the_umask( + self, tmp_path: Path, exact: bool + ) -> None: + """Test that -p restores an entry's mode without the umask applied.""" + archive = tmp_path / "modes.tar" + make_tar(archive, file_entry("a.txt", mode=0o777)) + dest = tmp_path / "dest" + + invoke("extract", archive, "-o", dest, *(["-p"] if exact else []), "-q") + + expected = 0o777 if exact else 0o777 & ~umask() + assert (dest / "a.txt").stat().st_mode & 0o7777 == expected + + def test_symlinks_flag_reaches_the_policy(self, tmp_path: Path) -> None: + """Test that a symlink refused by default extracts with `--symlinks file`.""" + archive = tmp_path / "link.tar" + make_tar(archive, link_entry("link", "target", kind=tarfile.SYMTYPE)) + dest = tmp_path / "dest" + + invoke("extract", archive, "-o", dest, "-q", expect=EXIT_FAILED) + invoke("extract", archive, "-o", dest, "--symlinks", "file", "-q") + + assert (dest / "link").read_bytes() == b"target" + + +class TestDecompressConflicts: + """Test what decompressing one file does when its output already exists.""" + + @CONFLICT_MODES + def test_existing_output( + self, + payload: Path, + tmp_path: Path, + flags: tuple[str, ...], + expect: int, + kept: bool, + renames: bool, + ) -> None: + """Test that each overwrite mode keeps, replaces or renames around the output.""" + archive = tmp_path / "out.comp" + restored = tmp_path / "restored.bin" + invoke("compress", payload, "-o", archive, "-q") + restored.write_bytes(b"stale") + + result = invoke("decompress", archive, "-o", restored, *flags, expect=expect) + + sibling = tmp_path / renamed(restored.name) + assert (f"Renamed to: {sibling}" in result.output) == renames + assert restored.read_bytes() == (b"stale" if kept else payload.read_bytes()) + assert sibling.exists() == renames + + +class TestDecompressName: + """Test that -N names a .gz file's output as its header does, as gzip -N does.""" + + @pytest.mark.parametrize( + ("flags", "expected"), [((), "renamed"), (("-N",), "orig.txt")] + ) + def test_name_flag( + self, tmp_path: Path, flags: tuple[str, ...], expected: str + ) -> None: + """Test that the stored name is used only with -N.""" + src = tmp_path / "renamed.gz" + with src.open("wb") as f, gzip.GzipFile("orig.txt", "wb", fileobj=f) as gz: + gz.write(b"payload") + + invoke("extract", src, *flags, "-q") + + assert (tmp_path / expected).read_bytes() == b"payload" + + +class TestSmartExtract: + """Test `extract` picks the job from each file's own bytes.""" + + def test_dispatch_follows_the_bytes_not_the_name( + self, payload: Path, tmp_path: Path + ) -> None: + """Test that a renamed .gz is still recognised as one.""" + packed = tmp_path / "p.gz" + invoke("compress", payload, "-o", packed, "-q") + + disguised = tmp_path / "mystery.dat" + disguised.write_bytes(packed.read_bytes()) + restored = tmp_path / "back.bin" + + invoke("extract", disguised, "-o", restored, "-q") + + assert restored.read_bytes() == payload.read_bytes() + + def test_an_archive_named_like_anything_still_extracts( + self, + source_tree: Path, + tmp_path: Path, + make_archive: Callable[[str], Path], + ) -> None: + """Test that a renamed archive still extracts.""" + disguised = tmp_path / "riddle.bin" + disguised.write_bytes(make_archive("tar").read_bytes()) + dest = tmp_path / "out" + + invoke("extract", disguised, "-o", dest, "-q") + + assert (dest / source_tree.name).is_dir() + + def test_an_archive_unpacks_beside_itself( + self, + source_tree: Path, + tmp_path: Path, + make_archive: Callable[[str], Path], + ) -> None: + """Test that with no -o, entries land next to the archive, not in the cwd.""" + packed = make_archive("tar") + + # The sources are beside the archive, so they would be in the way + for f in source_tree.iterdir(): + f.unlink() + source_tree.rmdir() + + invoke("extract", packed, "-q") + + assert (tmp_path / source_tree.name).is_dir() + assert not (Path.cwd() / source_tree.name).exists() + + def test_output_directory_receives_a_single_file( + self, payload: Path, tmp_path: Path + ) -> None: + """Test that -o naming an existing directory puts the file inside it.""" + packed = tmp_path / "p.gz" + invoke("compress", payload, "-o", packed, "-q") + + into = tmp_path / "into" + into.mkdir() + + invoke("extract", packed, "-o", into, "-q") + + assert (into / "p").read_bytes() == payload.read_bytes() + + def test_several_inputs_in_one_go(self, payload: Path, tmp_path: Path) -> None: + """Test that each input is dispatched on its own.""" + first = tmp_path / "a.gz" + second = tmp_path / "b.zst" + invoke("compress", payload, "-o", first, "-q") + invoke("compress", payload, "-o", second, "-q") + + invoke("extract", first, second, "-q") + + assert (tmp_path / "a").read_bytes() == payload.read_bytes() + assert (tmp_path / "b").read_bytes() == payload.read_bytes() + + def test_listing_a_single_file_container_is_refused( + self, payload: Path, tmp_path: Path + ) -> None: + """Test that there is nothing to list in a container holding one file.""" + packed = tmp_path / "p.gz" + invoke("compress", payload, "-o", packed, "-q") + + result = invoke("extract", packed, "--list", expect=EXIT_USAGE) + + assert "nothing to list" in result.output + + +class TestMergedVerbCompatibility: + """Test that the old names still work, and the old argument order says so.""" + + def test_decompress_now_handles_archives_too( + self, + source_tree: Path, + tmp_path: Path, + make_archive: Callable[[str], Path], + ) -> None: + """Test that the merged verb does not care which name was typed.""" + dest = tmp_path / "out" + + invoke("decompress", make_archive("tar"), "-o", dest, "-q") + + for original in source_tree.iterdir(): + restored = dest / source_tree.name / original.name + assert restored.read_bytes() == original.read_bytes() diff --git a/tests/python/cli/test_inspect.py b/tests/python/cli/test_inspect.py new file mode 100644 index 0000000..4f73c5f --- /dev/null +++ b/tests/python/cli/test_inspect.py @@ -0,0 +1,54 @@ +"""Tests for `compresso inspect`.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import orjson + +from .conftest import invoke + + +class TestInspect: + """Test the read-only `inspect` command.""" + + def test_inspect_reports_a_valid_file(self, payload: Path, tmp_path: Path) -> None: + """Test that inspect describes a file the CLI just wrote.""" + archive = tmp_path / "out.comp" + invoke("compress", payload, "-o", archive, "-q") + + result = invoke("inspect", archive) + + assert "Valid Compresso file" in result.output + + def test_inspect_json(self, payload: Path, tmp_path: Path) -> None: + """Test that --json emits parseable output.""" + archive = tmp_path / "out.comp" + invoke("compress", payload, "-o", archive, "-q") + + result = invoke("inspect", archive, "--json") + + assert orjson.loads(result.output)["is_compresso"] is True + + def test_inspect_archive_text(self, make_archive: Callable[[str], Path]) -> None: + """Test that an archive shows only a summary unless --entries is given.""" + archive = make_archive("zip") + + summary = invoke("inspect", archive).output + listing = invoke("inspect", archive, "--entries").output + + assert "Entries: 4" in summary + assert "f0.bin" not in summary + assert "f0.bin" in listing + + def test_inspect_archive_json(self, make_archive: Callable[[str], Path]) -> None: + """Test that --json carries entries only when --entries is given.""" + archive = make_archive("zip") + + summary = orjson.loads(invoke("inspect", archive, "--json").output) + listing = orjson.loads(invoke("inspect", archive, "--entries", "--json").output) + + assert summary["entry_count"] == 4 + assert "entries" not in summary + assert sum("f0.bin" in e["path"] for e in listing["entries"]) == 1 diff --git a/tests/python/cli/test_list.py b/tests/python/cli/test_list.py new file mode 100644 index 0000000..2d7e671 --- /dev/null +++ b/tests/python/cli/test_list.py @@ -0,0 +1,37 @@ +"""Tests for `compresso list`.""" + +from __future__ import annotations + +import subprocess +import sys + +from compresso.cli._render import ( + EXIT_OK, +) + +from .conftest import invoke + + +class TestList: + """Test the read-only `list` command.""" + + def test_runs_as_a_module(self) -> None: + """Test that `python -m compresso.cli` works, which a package needs __main__ for.""" + result = subprocess.run( + [sys.executable, "-m", "compresso.cli", "list"], + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == EXIT_OK + assert "zstd" in result.stdout + + def test_list_shows_level_ranges(self) -> None: + """Test that list shows each backend's levels, and none for snappy.""" + output = invoke("list").output + + zstd = output[output.index("- zstd") :].split("\n\n")[0] + snappy = output[output.index("- snappy") :].split("\n\n")[0] + assert "Levels: 1-22" in zstd + assert "Levels: none" in snappy diff --git a/tests/python/test_cli.py b/tests/python/test_cli.py deleted file mode 100644 index 79ce367..0000000 --- a/tests/python/test_cli.py +++ /dev/null @@ -1,1034 +0,0 @@ -"""Tests for the Compresso CLI.""" - -from __future__ import annotations - -import gzip -import os -import subprocess -import sys -import tarfile -from collections.abc import Callable, Generator -from contextlib import contextmanager -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -import orjson -import pytest -from typer.testing import CliRunner, Result - -import compresso.cli.algos as algos_cmd -import compresso.cli.compress as compress_cmd -import compresso.cli.extract as extract_cmd -import compresso.cli.inspect as inspect_cmd -from compresso.cli import app -from compresso.cli._render import ( - BAR_THRESHOLD, - EXIT_CANCELLED, - EXIT_FAILED, - EXIT_OK, - EXIT_USAGE, - format_size, - format_time, - progress_bar, -) -from compresso.cli.extract import list_entries -from compresso.frontend._job import ProgressCallback -from compresso.frontend.api import CompressionJob, DecompressionJob -from compresso.frontend.archive_api import ( - ArchiveEntry, - ArchiveJob, - ArchiveOptions, - ExtractJob, -) - -from .helpers import file_entry, link_entry, make_tar, renamed, umask - -runner = CliRunner() - -# Incompressible throughout, so compressed sizes stay close to these -PAYLOAD_SIZE = 64 * 1024 -TREE_FILE_SIZE = 16 * 1024 - -# Past the progress bar threshold, for the tests that cover the bar -LARGE_PAYLOAD_SIZE = 2 * BAR_THRESHOLD - - -def invoke(*args: str | Path, expect: int | None = EXIT_OK) -> Result: - """Run the CLI, asserting it exits with `expect`. - - Args: - *args: The command line, without the program name. - expect: The exit code to assert, or None to leave it unchecked. - - Returns: - The runner's result. - """ - result = runner.invoke(app, [str(arg) for arg in args]) - if expect is not None: - assert result.exit_code == expect, result.output - - return result - - -@pytest.fixture -def payload(tmp_path: Path) -> Path: - """A small incompressible file, below the progress bar threshold.""" - path = tmp_path / "payload.bin" - path.write_bytes(os.urandom(PAYLOAD_SIZE)) - - return path - - -@pytest.fixture -def source_tree(tmp_path: Path) -> Path: - """A small directory tree to archive, function-scoped because some tests delete it.""" - root = tmp_path / "tree" - root.mkdir() - for i in range(3): - (root / f"f{i}.bin").write_bytes(os.urandom(TREE_FILE_SIZE)) - - return root - - -@pytest.fixture -def make_archive(source_tree: Path, tmp_path: Path) -> Callable[[str], Path]: - """Build an archive of `source_tree` as input for a test. - - Built through the API rather than the CLI, and checked, so a broken setup - cannot surface as a confusing failure later in the test. - """ - - def make(fmt: str) -> Path: - archive = tmp_path / f"out.{fmt}" - options = ArchiveOptions(format=fmt) - result = ArchiveJob.from_paths([source_tree], archive, options).run() - assert result.ok, result.error - return archive - - return make - - -@dataclass -class CompressTarget: - """What one `compress` call reads and writes.""" - - source: Path - dest: Path - args: tuple[str, ...] - - -@pytest.fixture(params=["file", "tree"]) -def compress_target( - request: pytest.FixtureRequest, payload: Path, source_tree: Path, tmp_path: Path -) -> CompressTarget: - """A single file into a .comp container, or a tree into a tar archive. - - The two take different paths through `compress`, but share its - overwrite-mode handling. - """ - if request.param == "file": - return CompressTarget(payload, tmp_path / "out.comp", ()) - - return CompressTarget(source_tree, tmp_path / "out.tar", ("-f", "tar")) - - -@dataclass -class Bar: - """One progress bar a command opened.""" - - enabled: bool - updates: int = 0 - - -@pytest.fixture -def bars(monkeypatch: pytest.MonkeyPatch) -> list[Bar]: - """Record each progress bar a command opens, in order, still drawing the real one.""" - opened: list[Bar] = [] - - @contextmanager - def spy( - label: str, total: int, *, enabled: bool - ) -> Generator[ProgressCallback | None]: - bar = Bar(enabled) - opened.append(bar) - with progress_bar(label, total, enabled=enabled) as on_progress: - if on_progress is None: - yield None - return - - def counting(fraction: float, done: int, reported_total: int) -> None: - bar.updates += 1 - on_progress(fraction, done, reported_total) - - yield counting - - # Each command imports progress_bar by name, so each binding is patched - monkeypatch.setattr(compress_cmd, "progress_bar", spy) - monkeypatch.setattr(extract_cmd, "progress_bar", spy) - - return opened - - -class TestFormatters: - """Test utility formatting functions.""" - - @pytest.mark.parametrize( - "size,expected", - [ - (0, "0.00 B"), - (500, "500.00 B"), - (1024, "1.00 KB"), - (1536, "1.50 KB"), - (1048576, "1.00 MB"), - (1073741824, "1.00 GB"), - (1099511627776, "1.00 TB"), - ], - ) - def test_format_size(self, size: int, expected: str): - """Test size formatting with various values.""" - result = format_size(size) - assert result == expected - - @pytest.mark.parametrize( - "seconds,expected", - [ - (0.0005, "0ms"), - (0.5, "500ms"), - (1.0, "1.00s"), - (5.5, "5.50s"), - (59.9, "59.90s"), - (60.0, "1m 0.0s"), - (90.5, "1m 30.5s"), - (125.0, "2m 5.0s"), - ], - ) - def test_format_time(self, seconds: float, expected: str): - """Test time formatting with various values.""" - assert format_time(seconds) == expected - - -class TestAppStructure: - """Test the package assembles into one app with every command registered.""" - - def test_commands_are_listed_in_workflow_order(self) -> None: - """Test that the order is chosen in `__init__`, not inherited from import sorting. - - Alphabetical imports would put `list` first. - """ - output = invoke("--help").output - - # Matched on the alias column, which is unique per row; a bare command - # name also appears inside other commands' descriptions - positions = [ - output.index(aliases) - for aliases in [ - "(c, comp", - "(x, ex", - "(i, info)", - "(b, bench)", - "(l, ls)", - ] - ] - assert positions == sorted(positions) - - @pytest.mark.parametrize( - "alias,command", - [ - ("c", "compress"), - ("comp", "compress"), - ("archive", "compress"), - ("a", "compress"), - ("ar", "compress"), - ("x", "extract"), - ("ex", "extract"), - ("decompress", "extract"), - ("d", "extract"), - ("decomp", "extract"), - ("i", "inspect"), - ("info", "inspect"), - ("b", "benchmark"), - ("bench", "benchmark"), - ("l", "list"), - ("ls", "list"), - ], - ) - def test_aliases_still_resolve(self, alias: str, command: str) -> None: - """Test that each short name reaches the command it stands for. - - Help is compared without its usage line, which echoes the name typed. - """ - - def help_body(name: str) -> list[str]: - lines = invoke(name, "--help").output.splitlines() - return [line for line in lines if "Usage:" not in line] - - assert help_body(alias) == help_body(command) - - -class TestProgressBar: - """Test the bar is drawn only for a large job that is not asked to be quiet.""" - - @pytest.mark.parametrize( - "size,quiet,expected", - [ - (LARGE_PAYLOAD_SIZE, False, True), - (LARGE_PAYLOAD_SIZE, True, False), - (PAYLOAD_SIZE, False, False), - ], - ids=["large", "large-quiet", "small"], - ) - def test_compress_bar_follows_size_and_quiet( - self, tmp_path: Path, bars: list[Bar], size: int, quiet: bool, expected: bool - ) -> None: - """Test that the bar needs both a job past the threshold and no --quiet.""" - source = tmp_path / "source.bin" - source.write_bytes(os.urandom(size)) - - invoke( - "compress", source, "-o", tmp_path / "out.comp", *(["-q"] if quiet else []) - ) - - assert [bar.enabled for bar in bars] == [expected] - - def test_file_round_trip_draws_both_bars( - self, tmp_path: Path, bars: list[Bar] - ) -> None: - """Test that compress and extract each drive a bar with real progress.""" - source = tmp_path / "source.bin" - source.write_bytes(os.urandom(LARGE_PAYLOAD_SIZE)) - packed = tmp_path / "out.comp" - restored = tmp_path / "restored.bin" - - invoke("compress", source, "-o", packed) - invoke("extract", packed, "-o", restored) - - assert [bar.enabled for bar in bars] == [True, True] - assert all(bar.updates > 0 for bar in bars) - assert restored.read_bytes() == source.read_bytes() - - def test_archive_round_trip_draws_both_bars( - self, tmp_path: Path, bars: list[Bar] - ) -> None: - """Test that archiving and extracting a tree each drive a bar too.""" - tree = tmp_path / "big_tree" - tree.mkdir() - for i in range(2): - (tree / f"f{i}.bin").write_bytes(os.urandom(LARGE_PAYLOAD_SIZE // 2)) - archive = tmp_path / "out.tar" - dest = tmp_path / "dest" - - invoke("compress", tree, "-o", archive, "-f", "tar") - invoke("extract", archive, "-o", dest) - - assert [bar.enabled for bar in bars] == [True, True] - assert all(bar.updates > 0 for bar in bars) - for original in tree.iterdir(): - assert (dest / tree.name / original.name).read_bytes() == ( - original.read_bytes() - ) - - -class TestCompressRoundTrip: - """Test compress and decompress, end to end.""" - - def test_compress_then_decompress(self, payload: Path, tmp_path: Path) -> None: - """Test that a file survives a round trip through the CLI unchanged.""" - archive = tmp_path / "out.comp" - restored = tmp_path / "restored.bin" - - result = invoke("compress", payload, "-o", archive) - assert "Compression successful" in result.output - - result = invoke("decompress", archive, "-o", restored) - assert "Decompression successful" in result.output - - assert restored.read_bytes() == payload.read_bytes() - - def test_quiet_suppresses_output(self, payload: Path, tmp_path: Path) -> None: - """Test that --quiet prints nothing on success.""" - result = invoke("compress", payload, "-o", tmp_path / "q.comp", "-q") - assert result.output.strip() == "" - - # The backends are covered in core and C tests; this checks the flags - @pytest.mark.parametrize( - "args", [("-a", "bzip2"), ("-a", "zstd", "-l", "19")], ids=["algo", "level"] - ) - def test_algorithms_round_trip( - self, payload: Path, tmp_path: Path, args: tuple[str, ...] - ) -> None: - """Test that -a and -l reach the backend.""" - archive = tmp_path / "out.comp" - restored = tmp_path / "restored.bin" - - invoke("compress", payload, "-o", archive, *args, "-q") - invoke("decompress", archive, "-o", restored, "-q") - - assert restored.read_bytes() == payload.read_bytes() - - -class TestArchiveRoundTrip: - """Test archive and extract, end to end.""" - - @pytest.mark.parametrize("fmt", ["tar", "tar.gz", "tar.zst", "zip"]) - def test_archive_then_extract( - self, source_tree: Path, tmp_path: Path, fmt: str - ) -> None: - """Test that a tree survives a round trip through each archive format.""" - archive = tmp_path / f"out.{fmt}" - dest = tmp_path / f"dest.{fmt}" - - result = invoke("compress", source_tree, "-o", archive, "-f", fmt) - assert "Archive created" in result.output - - result = invoke("extract", archive, "-o", dest) - assert "Extraction successful" in result.output - - for original in source_tree.iterdir(): - restored = dest / source_tree.name / original.name - assert restored.read_bytes() == original.read_bytes() - - def test_extract_list_only( - self, - tmp_path: Path, - make_archive: Callable[[str], Path], - ) -> None: - """Test that --list prints each entry's size and path without writing anything.""" - archive = make_archive("tar") - dest = tmp_path / "dest" - - result = invoke("extract", archive, "--list", "-o", dest) - - lines = result.output.splitlines() - f0_line = next(line for line in lines if "f0.bin" in line) - assert f0_line.split()[:2] == format_size(TREE_FILE_SIZE).split() - assert not dest.exists() - - # Directories have a blank size; their contents are indented beneath them - dir_line = next(line for line in lines if line.rstrip("/").endswith("tree")) - assert dir_line == f"{'':>10} {dir_line.strip()}" - assert f0_line.endswith(" tree/f0.bin") - - def test_extract_list_indents_by_depth( - self, capsys: pytest.CaptureFixture[str] - ) -> None: - """Test that --list indents only under directories it has listed.""" - entries = [ - ArchiveEntry(path="a/", is_dir=True), - ArchiveEntry(path="a/b/", is_dir=True), - ArchiveEntry(path="a/b/c.txt", size=1024), - ArchiveEntry(path="a/d.txt", size=10), - ArchiveEntry(path="a/e", size=5, is_symlink=True, link_target="d.txt"), - ArchiveEntry(path="x/y.txt", size=0), # no "x/" entry was listed - ] - - list_entries(entries) - - assert capsys.readouterr().out.splitlines() == [ - " " * 12 + "a/", - " " * 14 + "a/b/", - " 1.00 KB" + " " * 6 + "a/b/c.txt", - " 10.00 B" + " " * 4 + "a/d.txt", - " " * 14 + "a/e -> d.txt", - " 0.00 B" + " " * 2 + "x/y.txt", - ] - - @pytest.mark.parametrize( - "flags,expect,renames", - [ - ((), EXIT_OK, True), - (("--error-on-conflict",), EXIT_FAILED, False), - (("--skip-existing",), EXIT_OK, False), - (("--overwrite",), EXIT_OK, False), - ], - ids=["rename", "error", "skip", "overwrite"], - ) - def test_extract_over_an_existing_tree( - self, - source_tree: Path, - tmp_path: Path, - make_archive: Callable[[str], Path], - flags: tuple[str, ...], - expect: int, - renames: bool, - ) -> None: - """Test that a repeat extraction renames the clashing top-level directory - by default, and follows the overwrite mode otherwise.""" - archive = make_archive("tar") - dest = tmp_path / "dest" - invoke("extract", archive, "-o", dest, "-q") - - invoke("extract", archive, "-o", dest, *flags, "-q", expect=expect) - - top_level = [source_tree.name, *([renamed(source_tree.name)] * renames)] - assert sorted(p.name for p in dest.iterdir()) == sorted(top_level) - # The whole directory is renamed, never a file inside it - names = sorted(p.name for p in source_tree.iterdir()) - for top in top_level: - assert sorted(p.name for p in (dest / top).iterdir()) == names - - @pytest.mark.skipif( - sys.platform == "win32", reason="Windows has no POSIX permission bits" - ) - @pytest.mark.parametrize("exact", [False, True], ids=["umask", "same-perms"]) - def test_same_permissions_skips_the_umask( - self, tmp_path: Path, exact: bool - ) -> None: - """Test that -p restores an entry's mode without the umask applied.""" - archive = tmp_path / "modes.tar" - make_tar(archive, file_entry("a.txt", mode=0o777)) - dest = tmp_path / "dest" - - invoke("extract", archive, "-o", dest, *(["-p"] if exact else []), "-q") - - expected = 0o777 if exact else 0o777 & ~umask() - assert (dest / "a.txt").stat().st_mode & 0o7777 == expected - - def test_symlinks_flag_reaches_the_policy(self, tmp_path: Path) -> None: - """Test that a symlink refused by default extracts with `--symlinks file`.""" - archive = tmp_path / "link.tar" - make_tar(archive, link_entry("link", "target", kind=tarfile.SYMTYPE)) - dest = tmp_path / "dest" - - invoke("extract", archive, "-o", dest, "-q", expect=EXIT_FAILED) - invoke("extract", archive, "-o", dest, "--symlinks", "file", "-q") - - assert (dest / "link").read_bytes() == b"target" - - -# Each overwrite mode against an existing output: whether the old output -# survives, and whether the new one is written beside it under a new name -CONFLICT_MODES = pytest.mark.parametrize( - "flags,expect,kept,renames", - [ - ((), EXIT_OK, True, True), - (("--error-on-conflict",), EXIT_FAILED, True, False), - (("--skip-existing",), EXIT_OK, True, False), - (("--overwrite",), EXIT_OK, False, False), - ], - ids=["rename", "error", "skip", "overwrite"], -) - - -class TestCompressConflicts: - """Test what `compress` does when its output already exists.""" - - @CONFLICT_MODES - def test_existing_output( - self, - compress_target: CompressTarget, - flags: tuple[str, ...], - expect: int, - kept: bool, - renames: bool, - ) -> None: - """Test that each overwrite mode keeps, replaces or renames around the output.""" - t = compress_target - t.dest.write_bytes(b"stale") - - invoke("compress", t.source, "-o", t.dest, *t.args, *flags, "-q", expect=expect) - - assert (t.dest.read_bytes() == b"stale") == kept - assert (t.dest.parent / renamed(t.dest.name)).is_file() == renames - - -class TestDecompressConflicts: - """Test what decompressing one file does when its output already exists.""" - - @CONFLICT_MODES - def test_existing_output( - self, - payload: Path, - tmp_path: Path, - flags: tuple[str, ...], - expect: int, - kept: bool, - renames: bool, - ) -> None: - """Test that each overwrite mode keeps, replaces or renames around the output.""" - archive = tmp_path / "out.comp" - restored = tmp_path / "restored.bin" - invoke("compress", payload, "-o", archive, "-q") - restored.write_bytes(b"stale") - - result = invoke("decompress", archive, "-o", restored, *flags, expect=expect) - - sibling = tmp_path / renamed(restored.name) - assert (f"Renamed to: {sibling}" in result.output) == renames - assert restored.read_bytes() == (b"stale" if kept else payload.read_bytes()) - assert sibling.exists() == renames - - -class TestDecompressName: - """Test that -N names a .gz file's output as its header does, as gzip -N does.""" - - @pytest.mark.parametrize( - ("flags", "expected"), [((), "renamed"), (("-N",), "orig.txt")] - ) - def test_name_flag( - self, tmp_path: Path, flags: tuple[str, ...], expected: str - ) -> None: - """Test that the stored name is used only with -N.""" - src = tmp_path / "renamed.gz" - with src.open("wb") as f, gzip.GzipFile("orig.txt", "wb", fileobj=f) as gz: - gz.write(b"payload") - - invoke("extract", src, *flags, "-q") - - assert (tmp_path / expected).read_bytes() == b"payload" - - -class TestInspectAndList: - """Test the read-only commands.""" - - def test_inspect_reports_a_valid_file(self, payload: Path, tmp_path: Path) -> None: - """Test that inspect describes a file the CLI just wrote.""" - archive = tmp_path / "out.comp" - invoke("compress", payload, "-o", archive, "-q") - - result = invoke("inspect", archive) - - assert "Valid Compresso file" in result.output - - def test_inspect_json(self, payload: Path, tmp_path: Path) -> None: - """Test that --json emits parseable output.""" - archive = tmp_path / "out.comp" - invoke("compress", payload, "-o", archive, "-q") - - result = invoke("inspect", archive, "--json") - - assert orjson.loads(result.output)["is_compresso"] is True - - def test_runs_as_a_module(self) -> None: - """Test that `python -m compresso.cli` works, which a package needs __main__ for.""" - result = subprocess.run( - [sys.executable, "-m", "compresso.cli", "list"], - capture_output=True, - text=True, - check=False, - ) - - assert result.returncode == EXIT_OK - assert "zstd" in result.stdout - - def test_list_shows_level_ranges(self) -> None: - """Test that list shows each backend's levels, and none for snappy.""" - output = invoke("list").output - - zstd = output[output.index("- zstd") :].split("\n\n")[0] - snappy = output[output.index("- snappy") :].split("\n\n")[0] - assert "Levels: 1-22" in zstd - assert "Levels: none" in snappy - - def test_inspect_archive_text(self, make_archive: Callable[[str], Path]) -> None: - """Test that an archive shows only a summary unless --entries is given.""" - archive = make_archive("zip") - - summary = invoke("inspect", archive).output - listing = invoke("inspect", archive, "--entries").output - - assert "Entries: 4" in summary - assert "f0.bin" not in summary - assert "f0.bin" in listing - - def test_inspect_archive_json(self, make_archive: Callable[[str], Path]) -> None: - """Test that --json carries entries only when --entries is given.""" - archive = make_archive("zip") - - summary = orjson.loads(invoke("inspect", archive, "--json").output) - listing = orjson.loads(invoke("inspect", archive, "--entries", "--json").output) - - assert summary["entry_count"] == 4 - assert "entries" not in summary - assert sum("f0.bin" in e["path"] for e in listing["entries"]) == 1 - - -class TestExitCodes: - """Test 0 success, 1 the operation failed, 2 the request could not be made.""" - - @pytest.mark.parametrize( - "command,name", - [ - ("compress", "nope.bin"), - ("inspect", "payload.bin"), - ("extract", "payload.bin"), - ], - ) - def test_bad_input_is_usage(self, payload: Path, command: str, name: str) -> None: - """Test that a missing or unrecognised input is wrong, rather than the - operation having failed.""" - invoke(command, payload.parent / name, expect=EXIT_USAGE) - - @pytest.mark.parametrize("command", ["compress", "extract"]) - @pytest.mark.parametrize( - "flags", - [ - ["--overwrite", "--skip-existing"], - ["--overwrite", "--error-on-conflict"], - ["--skip-existing", "--error-on-conflict"], - ], - ids=["overwrite+skip", "overwrite+error", "skip+error"], - ) - def test_mutually_exclusive_flag_usage( - self, - source_tree: Path, - tmp_path: Path, - make_archive: Callable[[str], Path], - command: str, - flags: list[str], - ) -> None: - """Test that combining any two overwrite-mode flags is contradictory.""" - if command == "compress": - args = [source_tree, "-o", tmp_path / "out.tar", "-f", "tar"] - else: - args = [make_archive("tar"), "-o", tmp_path / "d"] - - result = invoke(command, *args, *flags, expect=EXIT_USAGE) - assert "mutually exclusive" in result.output - - # An archive's level is checked against the stage that compresses - @pytest.mark.parametrize( - "compress_target,args,message", - [ - ( - "file", - ["-a", "zlib", "-l", "15"], - "zlib compression level 15 out of range (0-9", - ), - ("tree", ["-l", "5"], "tar has no compression levels"), - ], - indirect=["compress_target"], - ) - def test_out_of_range_level_is_usage( - self, compress_target: CompressTarget, args: list[str], message: str - ) -> None: - """Test that a level the backend rejects is refused up front, naming the range.""" - t = compress_target - result = invoke( - "compress", t.source, "-o", t.dest, *t.args, *args, expect=EXIT_USAGE - ) - assert message in result.output - assert not t.dest.exists() - - def test_bad_benchmark_level_is_usage(self, payload: Path) -> None: - """Test that a level that is not a number is a usage error.""" - invoke("benchmark", payload, "--levels", "nope", expect=EXIT_USAGE) - - @pytest.mark.parametrize("flags", [(), ("-f", "7z")], ids=["suffix", "flag"]) - def test_recognised_but_unsupported_archive_names_itself( - self, source_tree: Path, tmp_path: Path, flags: tuple[str, ...] - ) -> None: - """Test that 7z, asked for by -f or by `-o x.7z` alone, reports its own name - rather than being written as a zip.""" - dest = tmp_path / "o.7z" - result = invoke("compress", source_tree, "-o", dest, *flags, expect=EXIT_FAILED) - assert "7z archives are recognised but not supported yet" in result.output - assert not dest.exists() - - -class TestInterruption: - """Test Ctrl-C reports and exits 130. - - ExtendedTyper catches KeyboardInterrupt around the command callback and turns - it into a silent `Exit(130)`, so each command catches it first to say what - was interrupted. - """ - - @pytest.mark.parametrize( - "command,expected", - [ - (["compress"], "Compression cancelled"), - (["inspect"], "Inspection cancelled"), - (["list"], "Listing cancelled"), - ], - ) - def test_interrupt_is_reported( - self, - payload: Path, - monkeypatch: pytest.MonkeyPatch, - command: list[str], - expected: str, - ) -> None: - """Test that each command names itself rather than dying silently.""" - - # Interrupt the first real call each command makes - def interrupt(*args: object, **kwargs: object) -> None: - raise KeyboardInterrupt - - monkeypatch.setattr(CompressionJob, "from_file", interrupt) - monkeypatch.setattr(inspect_cmd, "inspect_file", interrupt) - monkeypatch.setattr(algos_cmd, "list_capabilities", interrupt) - - args: list[str | Path] = [*command] - if command[0] != "list": - args.append(payload) - - result = invoke(*args, expect=EXIT_CANCELLED) - - assert expected in result.output - - -class TestSmartCompress: - """Test `compress` picks the job from its inputs and the format asked for.""" - - def test_one_file_becomes_a_compresso_container(self, payload: Path) -> None: - """Test that with no format, a single file gets the .comp container.""" - invoke("compress", payload, "-q") - - assert payload.with_suffix(payload.suffix + ".comp").exists() - - @pytest.mark.parametrize("fmt", ["gz", "bz2", "xz", "zst", "lz4", "sz"]) - def test_one_file_into_a_standalone_container( - self, payload: Path, tmp_path: Path, fmt: str - ) -> None: - """Test that -f names a single-file container, the suffix follows from it, - and it comes back.""" - restored = tmp_path / "restored.bin" - - invoke("compress", payload, "-f", fmt, "-q") - invoke("extract", f"{payload}.{fmt}", "-o", restored, "-q") - - assert restored.read_bytes() == payload.read_bytes() - - def test_standalone_output_is_a_real_gzip( - self, payload: Path, tmp_path: Path - ) -> None: - """Test that the .gz written is the format, not merely the name.""" - output = tmp_path / "out.gz" - invoke("compress", payload, "-o", output, "-q") - - with gzip.open(output, "rb") as f: - assert f.read() == payload.read_bytes() - - def test_a_directory_becomes_an_archive(self, source_tree: Path) -> None: - """Test that a directory needs a container that holds many entries.""" - invoke("compress", source_tree, "-q") - - assert (source_tree.parent / f"{source_tree.name}.zip").exists() - - def test_several_files_become_an_archive( - self, payload: Path, tmp_path: Path - ) -> None: - """Test that multiple files become an archive.""" - second = tmp_path / "second.bin" - second.write_bytes(os.urandom(1024)) - output = tmp_path / "both.tar" - - invoke("compress", payload, second, "-o", output, "-q") - - with tarfile.open(output) as tar: - assert sorted(tar.getnames()) == [payload.name, second.name] - - @pytest.mark.parametrize( - "name,expected_magic", - [ - ("out.gz", b"\x1f\x8b"), - ("out.zst", b"\x28\xb5\x2f\xfd"), - ("out.zip", b"PK"), - ], - ) - def test_format_inferred_from_the_output_name( - self, payload: Path, tmp_path: Path, name: str, expected_magic: bytes - ) -> None: - """Test that a recognisable extension on -o chooses the format by itself.""" - output = tmp_path / name - - invoke("compress", payload, "-o", output, "-q") - - assert output.read_bytes().startswith(expected_magic) - - def test_explicit_format_beats_the_output_name( - self, payload: Path, tmp_path: Path - ) -> None: - """Test that -f overrides whatever the extension suggests.""" - output = tmp_path / "misleading.gz" - - invoke("compress", payload, "-o", output, "-f", "zst", "-q") - - assert output.read_bytes().startswith(b"\x28\xb5\x2f\xfd") - - @pytest.mark.parametrize( - "name", ["out.tar.zst", "out.tar.lz4", "out.tlz4", "out.tar.sz"] - ) - def test_tar_inside_a_codec_is_an_archive( - self, source_tree: Path, tmp_path: Path, name: str - ) -> None: - """Test that two-part extensions name an archive, not their codec alone.""" - output = tmp_path / name - - invoke("compress", source_tree, "-o", output, "-q") - - # Round-trips as an archive rather than as a lone compressed file - dest = tmp_path / "back" - invoke("extract", output, "-o", dest, "-q") - assert (dest / source_tree.name).is_dir() - - def test_single_file_format_refuses_several_inputs( - self, source_tree: Path, tmp_path: Path - ) -> None: - """Test that a .gz holds one file, so a directory cannot go into one.""" - result = invoke( - "compress", - source_tree, - "-f", - "gz", - "-o", - tmp_path / "o.gz", - expect=EXIT_USAGE, - ) - - assert "single file" in result.output - - def test_unknown_format_is_refused(self, payload: Path) -> None: - """Test that an unrecognised -f names neither container.""" - invoke("compress", payload, "-f", "bogus", expect=EXIT_USAGE) - - -class TestThreads: - """Test that -T reaches each kind of job, and refuses a negative count.""" - - def test_threads_reach_every_job( - self, - payload: Path, - source_tree: Path, - make_archive: Callable[[str], Path], - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """Test that compress and extract pass -T on, one file or many.""" - archive = make_archive("tar") - seen: list[int] = [] - - # Record the count each job was planned with, then stop before it runs - def record(**kwargs: Any) -> None: - options = kwargs.get("options") - seen.append(kwargs["threads"] if options is None else options.threads) - raise KeyboardInterrupt - - monkeypatch.setattr(CompressionJob, "from_file", record) - monkeypatch.setattr(ArchiveJob, "from_paths", record) - monkeypatch.setattr(DecompressionJob, "from_file", record) - monkeypatch.setattr(ExtractJob, "from_archive", record) - - for command, source in [ - ("compress", payload), - ("compress", source_tree), - ("extract", payload), - ("extract", archive), - ]: - invoke(command, source, "-T", "3", expect=EXIT_CANCELLED) - - assert seen == [3, 3, 3, 3] - - @pytest.mark.parametrize("command", ["compress", "extract"]) - def test_negative_threads_is_usage(self, payload: Path, command: str) -> None: - """Test that a count below 0 is refused before any work starts.""" - invoke(command, payload, "-T", "-1", expect=EXIT_USAGE) - - -class TestSmartExtract: - """Test `extract` picks the job from each file's own bytes.""" - - def test_dispatch_follows_the_bytes_not_the_name( - self, payload: Path, tmp_path: Path - ) -> None: - """Test that a renamed .gz is still recognised as one.""" - packed = tmp_path / "p.gz" - invoke("compress", payload, "-o", packed, "-q") - - disguised = tmp_path / "mystery.dat" - disguised.write_bytes(packed.read_bytes()) - restored = tmp_path / "back.bin" - - invoke("extract", disguised, "-o", restored, "-q") - - assert restored.read_bytes() == payload.read_bytes() - - def test_an_archive_named_like_anything_still_extracts( - self, - source_tree: Path, - tmp_path: Path, - make_archive: Callable[[str], Path], - ) -> None: - """Test that a renamed archive still extracts.""" - disguised = tmp_path / "riddle.bin" - disguised.write_bytes(make_archive("tar").read_bytes()) - dest = tmp_path / "out" - - invoke("extract", disguised, "-o", dest, "-q") - - assert (dest / source_tree.name).is_dir() - - def test_an_archive_unpacks_beside_itself( - self, - source_tree: Path, - tmp_path: Path, - make_archive: Callable[[str], Path], - ) -> None: - """Test that with no -o, entries land next to the archive, not in the cwd.""" - packed = make_archive("tar") - - # The sources are beside the archive, so they would be in the way - for f in source_tree.iterdir(): - f.unlink() - source_tree.rmdir() - - invoke("extract", packed, "-q") - - assert (tmp_path / source_tree.name).is_dir() - assert not (Path.cwd() / source_tree.name).exists() - - def test_output_directory_receives_a_single_file( - self, payload: Path, tmp_path: Path - ) -> None: - """Test that -o naming an existing directory puts the file inside it.""" - packed = tmp_path / "p.gz" - invoke("compress", payload, "-o", packed, "-q") - - into = tmp_path / "into" - into.mkdir() - - invoke("extract", packed, "-o", into, "-q") - - assert (into / "p").read_bytes() == payload.read_bytes() - - def test_several_inputs_in_one_go(self, payload: Path, tmp_path: Path) -> None: - """Test that each input is dispatched on its own.""" - first = tmp_path / "a.gz" - second = tmp_path / "b.zst" - invoke("compress", payload, "-o", first, "-q") - invoke("compress", payload, "-o", second, "-q") - - invoke("extract", first, second, "-q") - - assert (tmp_path / "a").read_bytes() == payload.read_bytes() - assert (tmp_path / "b").read_bytes() == payload.read_bytes() - - def test_listing_a_single_file_container_is_refused( - self, payload: Path, tmp_path: Path - ) -> None: - """Test that there is nothing to list in a container holding one file.""" - packed = tmp_path / "p.gz" - invoke("compress", payload, "-o", packed, "-q") - - result = invoke("extract", packed, "--list", expect=EXIT_USAGE) - - assert "nothing to list" in result.output - - -class TestMergedVerbCompatibility: - """Test that the old names still work, and the old argument order says so.""" - - def test_decompress_now_handles_archives_too( - self, - source_tree: Path, - tmp_path: Path, - make_archive: Callable[[str], Path], - ) -> None: - """Test that the merged verb does not care which name was typed.""" - dest = tmp_path / "out" - - invoke("decompress", make_archive("tar"), "-o", dest, "-q") - - for original in source_tree.iterdir(): - restored = dest / source_tree.name / original.name - assert restored.read_bytes() == original.read_bytes() From b611962939cf0d95de64a78c83f21a5296865fe8 Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Thu, 22 Oct 2026 16:56:04 +0100 Subject: [PATCH 02/31] refactor(frontend): move the dispatch rules out of the CLI - cli/_dispatch.py becomes frontend/dispatch.py, since it only holds dispatch rules and no CLI logic - New test_dispatch.py tests format detection, '.comp' handling, zip recognition, directory handling, and '-o' into an existing directory - looks_like_archive()'s bare 'except' gains the reason it is bare, that an unreadable file is not an archive --- src/compresso/cli/compress.py | 4 +- src/compresso/cli/extract.py | 2 +- src/compresso/cli/inspect.py | 2 +- .../_dispatch.py => frontend/dispatch.py} | 6 +- src/compresso/meson.build | 2 +- tests/python/frontend/test_dispatch.py | 137 ++++++++++++++++++ 6 files changed, 145 insertions(+), 8 deletions(-) rename src/compresso/{cli/_dispatch.py => frontend/dispatch.py} (93%) create mode 100644 tests/python/frontend/test_dispatch.py diff --git a/src/compresso/cli/compress.py b/src/compresso/cli/compress.py index 00f53a4..9f724bd 100644 --- a/src/compresso/cli/compress.py +++ b/src/compresso/cli/compress.py @@ -9,8 +9,7 @@ from .._core import BackendError, Error, HeaderError from ..frontend.api import CompressionJob, CompressionOptions from ..frontend.archive_api import ArchiveJob, ArchiveOptions, OverwriteMode -from ._app import app -from ._dispatch import ( +from ..frontend.dispatch import ( DEFAULT_ARCHIVE_FORMAT, default_archive_output, format_is_single_file, @@ -18,6 +17,7 @@ is_archive_format, resolve_single_output, ) +from ._app import app from ._render import ( BAR_THRESHOLD, EXIT_USAGE, diff --git a/src/compresso/cli/extract.py b/src/compresso/cli/extract.py index e16b9c7..0f6bec5 100644 --- a/src/compresso/cli/extract.py +++ b/src/compresso/cli/extract.py @@ -15,8 +15,8 @@ OverwriteMode, SymlinkMode, ) +from ..frontend.dispatch import looks_like_archive, resolve_single_output from ._app import app -from ._dispatch import looks_like_archive, resolve_single_output from ._render import ( BAR_THRESHOLD, EXIT_USAGE, diff --git a/src/compresso/cli/inspect.py b/src/compresso/cli/inspect.py index 3d3bbce..ff41d5d 100644 --- a/src/compresso/cli/inspect.py +++ b/src/compresso/cli/inspect.py @@ -12,9 +12,9 @@ from .._core import detect_format from ..frontend.archive_api import ArchiveEntry from ..frontend.archive_api import plan_extraction as plan_archive_extraction +from ..frontend.dispatch import looks_like_archive from ..introspect.file_inspect import inspect as inspect_file from ._app import app -from ._dispatch import looks_like_archive from ._render import ( EXIT_USAGE, cancelled, diff --git a/src/compresso/cli/_dispatch.py b/src/compresso/frontend/dispatch.py similarity index 93% rename from src/compresso/cli/_dispatch.py rename to src/compresso/frontend/dispatch.py index 484c1bc..0d9c3ab 100644 --- a/src/compresso/cli/_dispatch.py +++ b/src/compresso/frontend/dispatch.py @@ -1,11 +1,11 @@ -"""Dispatcher for the two smart commands `compress` and `extract`.""" +"""Dispatch rules for the smart `compress` and `extract` commands.""" from __future__ import annotations from pathlib import Path from .._core import detect_format, format_is_archive -from ..frontend.api import STANDALONE_FORMATS, canonical_standalone_format +from .api import STANDALONE_FORMATS, canonical_standalone_format # What `compress` reaches for when given several inputs and no format DEFAULT_ARCHIVE_FORMAT = "zip" @@ -50,7 +50,7 @@ def looks_like_archive(path: Path) -> bool: try: return bool(format_is_archive(detect_format(path))) - except Exception: + except Exception: # noqa: BLE001 - unreadable means not an archive return False diff --git a/src/compresso/meson.build b/src/compresso/meson.build index 283792f..50ae493 100644 --- a/src/compresso/meson.build +++ b/src/compresso/meson.build @@ -52,7 +52,6 @@ py.install_sources( 'cli/__init__.py', 'cli/__main__.py', 'cli/_app.py', - 'cli/_dispatch.py', 'cli/_render.py', 'cli/algos.py', 'cli/benchmark.py', @@ -67,6 +66,7 @@ py.install_sources( 'frontend/_job.py', 'frontend/api.py', 'frontend/archive_api.py', + 'frontend/dispatch.py', subdir: 'compresso/frontend', ) diff --git a/tests/python/frontend/test_dispatch.py b/tests/python/frontend/test_dispatch.py new file mode 100644 index 0000000..5391a98 --- /dev/null +++ b/tests/python/frontend/test_dispatch.py @@ -0,0 +1,137 @@ +"""Tests for the frontend dispatch rules.""" + +import gzip +import os +import zipfile +from pathlib import Path + +import pytest + +from compresso.frontend.dispatch import ( + default_archive_output, + format_is_single_file, + infer_format, + is_archive_format, + looks_like_archive, + resolve_single_output, +) + + +class TestInferFormat: + """Test reading a format off an output filename.""" + + @pytest.mark.parametrize( + ("name", "expected"), + [ + ("x.tar.lz4", "tar.lz4"), + ("x.TAR.ZST", "tar.zst"), + ("x.tar", "tar"), + ("x.zip", "zip"), + ("x.7z", "7z"), + ("x.gz", "gz"), + ("x.zst", "zst"), + ("x.comp", None), + ("x.txt", None), + ("x", None), + ("x.foo.lz4", "lz4"), + ], + ) + def test_suffix(self, name: str, expected: str | None) -> None: + """Test that the longest known suffix wins, case-insensitively.""" + assert infer_format(Path(name)) == expected + + def test_no_output(self) -> None: + """Test that no output path infers no format.""" + assert infer_format(None) is None + + +class TestFormatKinds: + """Test classifying a format as an archive or a single-file container.""" + + @pytest.mark.parametrize( + ("fmt", "archive", "single"), + [ + ("zip", True, False), + ("TAR.GZ", True, False), + ("7z", True, False), + ("gz", False, True), + ("ZSTD", False, True), + ("comp", False, False), + ("nope", False, False), + (None, False, False), + ], + ) + def test_kind(self, fmt: str | None, archive: bool, single: bool) -> None: + """Test that `.comp` and unknown formats are neither kind.""" + assert is_archive_format(fmt) is archive + assert format_is_single_file(fmt) is single + + +class TestLooksLikeArchive: + """Test detecting an archive from its bytes rather than its name.""" + + def test_zip_under_any_name(self, tmp_path: Path) -> None: + """Test that a zip is recognised without a `.zip` suffix.""" + path = tmp_path / "data.bin" + with zipfile.ZipFile(path, "w") as zf: + zf.writestr("a.txt", "a") + + assert looks_like_archive(path) + + def test_single_file_container(self, tmp_path: Path) -> None: + """Test that a gzip stream is not an archive, even named `.zip`.""" + path = tmp_path / "data.zip" + path.write_bytes(gzip.compress(b"a")) + + assert not looks_like_archive(path) + + def test_missing_file(self, tmp_path: Path) -> None: + """Test that a missing file is not an archive rather than an error.""" + assert not looks_like_archive(tmp_path / "missing.zip") + + +class TestDefaultArchiveOutput: + """Test naming an archive the caller did not name.""" + + def test_beside_the_input(self, tmp_path: Path) -> None: + """Test that the archive is named after the input, next to it.""" + src = tmp_path / "src" + src.mkdir() + + assert default_archive_output(src, "tar.zst") == tmp_path / "src.tar.zst" + + @pytest.mark.parametrize("name", [".", f"..{os.sep}src{os.sep}"]) + def test_trailing_separator( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, name: str + ) -> None: + """Test that `.` and a trailing separator still name the directory.""" + src = tmp_path / "src" + src.mkdir() + monkeypatch.chdir(src) + + assert ( + default_archive_output(Path(name), "zip") == tmp_path.resolve() / "src.zip" + ) + + +class TestResolveSingleOutput: + """Test where `-o` sends a single-file job.""" + + def test_no_output_keeps_default(self, tmp_path: Path) -> None: + """Test that without `-o` the default path is used.""" + default = tmp_path / "x.gz" + + assert resolve_single_output(None, default) == default + + def test_existing_directory_receives_the_file(self, tmp_path: Path) -> None: + """Test that an existing directory gets the default filename inside it.""" + out = tmp_path / "out" + out.mkdir() + + assert resolve_single_output(out, tmp_path / "x.gz") == out / "x.gz" + + def test_anything_else_is_the_file(self, tmp_path: Path) -> None: + """Test that any other path is taken as the output file itself.""" + out = tmp_path / "out" / "y.gz" + + assert resolve_single_output(out, tmp_path / "x.gz") == out From a8e82582ba6e52064c8be685602e815dbfa05396 Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Thu, 22 Oct 2026 19:48:55 +0100 Subject: [PATCH 03/31] refactor(frontend): plan the compress and extract jobs in the frontend - plan_compress() and plan_extract() now take a frozen 'CompressRequest' or 'ExtractRequest' and return the planned job, or one per input - Command handlers take a job and are left with rendering, and read the plan rather than their own arguments - Single-file format with several inputs is now a 'ValueError' from the planner, which 'compress' turns into its usage exit - test_dispatch.py now asserts on the job each request plans - 'TestSmartCompress' and 'TestSmartExtract' keep one round trip each, asserting the commands run what the planner returns --- src/compresso/cli/compress.py | 135 ++++------------- src/compresso/cli/extract.py | 72 +++------ src/compresso/frontend/dispatch.py | 156 +++++++++++++++++++- tests/python/cli/test_compress.py | 106 ++----------- tests/python/cli/test_extract.py | 65 +------- tests/python/frontend/test_dispatch.py | 197 +++++++++++++++++++++++++ 6 files changed, 417 insertions(+), 314 deletions(-) diff --git a/src/compresso/cli/compress.py b/src/compresso/cli/compress.py index 9f724bd..ee69bc0 100644 --- a/src/compresso/cli/compress.py +++ b/src/compresso/cli/compress.py @@ -7,16 +7,9 @@ from typing import Annotated from .._core import BackendError, Error, HeaderError -from ..frontend.api import CompressionJob, CompressionOptions -from ..frontend.archive_api import ArchiveJob, ArchiveOptions, OverwriteMode -from ..frontend.dispatch import ( - DEFAULT_ARCHIVE_FORMAT, - default_archive_output, - format_is_single_file, - infer_format, - is_archive_format, - resolve_single_output, -) +from ..frontend.api import CompressionJob +from ..frontend.archive_api import ArchiveJob, OverwriteMode +from ..frontend.dispatch import CompressRequest, plan_compress from ._app import app from ._render import ( BAR_THRESHOLD, @@ -31,47 +24,17 @@ ) -def _compress_one_file( - source: Path, - output: Path | None, - fmt: str | None, - algo: str | None, - strategy: str, - level: int | None, - overwrite: OverwriteMode, - threads: int, - quiet: bool, -) -> None: +def _compress_one_file(job: CompressionJob, quiet: bool) -> None: """Compress a single file into a one-file container. Args: - source: The path to the file to compress. - output: The path to the output file or directory. - fmt: The format to use for compression. - algo: The algorithm to use for compression. - strategy: The strategy to use for compression. - level: The compression level to use. - overwrite: What to do when the destination file already exists. - threads: Worker threads, or 0 for one per core. + job: The planned job. quiet: Whether to suppress progress output. """ - algo_lower: str | None = algo.lower() if algo else None - options = CompressionOptions( - algo=None if algo_lower == "auto" else algo_lower, - strategy=strategy.lower(), - level=level, - format=fmt, - overwrite=overwrite, - threads=threads, - ) - - job = CompressionJob.from_file(src=source, dest=None, options=options) - # Planned once to learn the default name, then repointed if `-o` moves it - dest: Path = resolve_single_output(output, job.plan.dest) - if dest != job.plan.dest: - job = CompressionJob.from_file(src=source, dest=dest, options=options) - plan = job.plan + dest: Path = plan.dest + fmt: str | None = plan.options.format + level: int | None = plan.options.level if not plan.can_compress: fail(f"Error: {plan.reason_if_unavailable}", EXIT_USAGE) @@ -83,7 +46,7 @@ def _compress_one_file( f"Algorithm: {plan.backend_name}", ] if not fmt: - lines.append(f"Strategy: {strategy}") + lines.append(f"Strategy: {plan.options.strategy}") if level is not None: lines.append(f"Level: {level}") print("\n".join(lines) + "\n") @@ -127,38 +90,15 @@ def _compress_one_file( ) -def _archive_many( - sources: list[Path], - output: Path | None, - fmt: str, - level: int | None, - overwrite: OverwriteMode, - threads: int, - quiet: bool, -) -> None: +def _archive_many(job: ArchiveJob, quiet: bool) -> None: """Pack several paths, or a directory, into one archive. Args: - sources: The paths to the files or directories to archive. - output: The path to the output file or directory. - fmt: The format to use for compression. - level: The compression level to use. - overwrite: What to do when the destination archive already exists. - threads: Worker threads, or 0 for one per core. + job: The planned job. quiet: Whether to suppress progress output. """ - destination: Path = ( - output if output is not None else default_archive_output(sources[0], fmt) - ) - - options = ArchiveOptions( - format=fmt.lower(), - compression_level=level, - overwrite=overwrite, - threads=threads, - ) - job = ArchiveJob.from_paths(sources=sources, output=destination, options=options) plan = job.plan + destination: Path = plan.output if not plan.can_run: fail(f"Error: {plan.reason_if_unavailable}", EXIT_USAGE) @@ -324,39 +264,28 @@ def compress( if not inputs: fail("Error: No input paths given", EXIT_USAGE) - chosen: str | None = format or infer_format(output) - many: bool = len(inputs) > 1 or any(p.is_dir() for p in inputs) - - if format_is_single_file(chosen) and many: - fail( - f"Error: {chosen} holds a single file; " - f"use an archive format such as tar.{chosen} for several", - EXIT_USAGE, - ) - - if is_archive_format(chosen) or (many and chosen is None): - _archive_many( - sources=inputs, - output=output, - fmt=chosen or DEFAULT_ARCHIVE_FORMAT, - level=level, - overwrite=overwrite_mode, - threads=threads, - quiet=quiet, - ) + algo_lower: str | None = algo.lower() if algo else None + request = CompressRequest( + inputs=inputs, + output=output, + format=format, + algo=None if algo_lower == "auto" else algo_lower, + strategy=strategy.lower(), + level=level, + overwrite=overwrite_mode, + threads=threads, + ) + + try: + job = plan_compress(request) + + except ValueError as e: + fail(f"Error: {e}", EXIT_USAGE) + if isinstance(job, ArchiveJob): + _archive_many(job, quiet) else: - _compress_one_file( - source=inputs[0], - output=output, - fmt=chosen, - algo=algo, - strategy=strategy, - level=level, - overwrite=overwrite_mode, - threads=threads, - quiet=quiet, - ) + _compress_one_file(job, quiet) except KeyboardInterrupt: cancelled("Compression") diff --git a/src/compresso/cli/extract.py b/src/compresso/cli/extract.py index 0f6bec5..6cec6e4 100644 --- a/src/compresso/cli/extract.py +++ b/src/compresso/cli/extract.py @@ -15,7 +15,7 @@ OverwriteMode, SymlinkMode, ) -from ..frontend.dispatch import looks_like_archive, resolve_single_output +from ..frontend.dispatch import ExtractRequest, plan_extract from ._app import app from ._render import ( BAR_THRESHOLD, @@ -61,25 +61,14 @@ def list_entries(entries: list[ArchiveEntry]) -> None: print(f"{size:>10} {' ' * depth}{entry.path}{target}") -def _extract_archive( - source: Path, - output_dir: Path | None, - list_only: bool, - options: ExtractOptions, - quiet: bool, -) -> None: +def _extract_archive(job: ExtractJob, list_only: bool, quiet: bool) -> None: """Unpack one archive. Args: - source: The path to the archive to extract. - output_dir: The directory to extract to. + job: The planned job. list_only: Whether to list entries only. - options: The extract options. quiet: Whether to suppress output. """ - job = ExtractJob.from_archive( - archive=source, output_dir=output_dir, options=options - ) plan = job.plan if not plan.can_run: @@ -122,52 +111,23 @@ def _extract_archive( ) -def _decompress_one_file( - source: Path, - output: Path | None, - list_only: bool, - overwrite: OverwriteMode, - threads: int, - restore_name: bool, - quiet: bool, -) -> None: +def _decompress_one_file(job: DecompressionJob, list_only: bool, quiet: bool) -> None: """Unpack one single-file container: a `.comp` or a standalone codec. Args: - source: The path to the single-file container. - output: The output path. + job: The planned job. list_only: Whether to list entries only. - overwrite: What to do when the destination file already exists. - threads: Worker threads for xz, or 0 for one per core. - restore_name: Whether to name and date a gzip file's output as its - header does. quiet: Whether to suppress output. """ + plan = job.plan + dest: Path = plan.dest + if list_only: fail( - f"Error: {source.name} holds a single file, so there is nothing to list", + f"Error: {plan.src.name} holds a single file, so there is nothing to list", EXIT_USAGE, ) - job = DecompressionJob.from_file( - src=source, - dest=None, - overwrite=overwrite, - threads=threads, - restore_name=restore_name, - ) - dest: Path = resolve_single_output(output, job.plan.dest) - if dest != job.plan.dest: - job = DecompressionJob.from_file( - src=source, - dest=dest, - overwrite=overwrite, - threads=threads, - restore_name=restore_name, - ) - - plan = job.plan - if not plan.can_run: fail(f"Error: {plan.reason_if_unavailable}", EXIT_USAGE) @@ -352,15 +312,17 @@ def extract( threads=threads, ) + request = ExtractRequest( + inputs=inputs, output=output, options=options, restore_name=name + ) + try: - for source in inputs: - if looks_like_archive(source): - _extract_archive(source, output, list_only, options, quiet) + for job in plan_extract(request): + if isinstance(job, ExtractJob): + _extract_archive(job, list_only, quiet) else: - _decompress_one_file( - source, output, list_only, mode, threads, name, quiet - ) + _decompress_one_file(job, list_only, quiet) except KeyboardInterrupt: cancelled("Extraction") diff --git a/src/compresso/frontend/dispatch.py b/src/compresso/frontend/dispatch.py index 0d9c3ab..16a13af 100644 --- a/src/compresso/frontend/dispatch.py +++ b/src/compresso/frontend/dispatch.py @@ -2,10 +2,25 @@ from __future__ import annotations +from dataclasses import dataclass, field +from functools import partial from pathlib import Path from .._core import detect_format, format_is_archive -from .api import STANDALONE_FORMATS, canonical_standalone_format +from .api import ( + STANDALONE_FORMATS, + CompressionJob, + CompressionOptions, + DecompressionJob, + canonical_standalone_format, +) +from .archive_api import ( + ArchiveJob, + ArchiveOptions, + ExtractJob, + ExtractOptions, + OverwriteMode, +) # What `compress` reaches for when given several inputs and no format DEFAULT_ARCHIVE_FORMAT = "zip" @@ -79,6 +94,7 @@ def default_archive_output(first_input: Path, fmt: str) -> Path: """ # A trailing separator would otherwise make the name empty stem: str = first_input.resolve().name + return first_input.resolve().parent / f"{stem}.{fmt}" @@ -114,3 +130,141 @@ def format_is_single_file(fmt: str | None) -> bool: True if `fmt` names a single-file format, False otherwise. """ return canonical_standalone_format(fmt) is not None + + +@dataclass(frozen=True) +class CompressRequest: + """What `compress` was asked to do, already parsed. + + Attributes: + inputs: The files and directories to compress. + output: The `-o` value, or None to name the output after the first input. + format: The container asked for, or None to infer it. + algo: The `.comp` algorithm, or None for the strategy's default. + strategy: The `.comp` strategy. + level: The compression level, or None for the format's own. + overwrite: What to do when the output already exists. + threads: Worker threads, or 0 for one per core. + """ + + inputs: list[Path] + output: Path | None = None + format: str | None = None + algo: str | None = None + strategy: str = "balanced" + level: int | None = None + overwrite: OverwriteMode = OverwriteMode.RENAME + threads: int = 0 + + +@dataclass(frozen=True) +class ExtractRequest: + """What `extract` was asked to do, already parsed. + + Attributes: + inputs: The archives and single-file containers to unpack. + output: The `-o` value, or None to write beside each input. + options: The extraction policy; single-file inputs use only its + `overwrite` and `threads`. + restore_name: Whether to name and date a gzip file's output as its + header does. + """ + + inputs: list[Path] + output: Path | None = None + options: ExtractOptions = field(default_factory=ExtractOptions) + restore_name: bool = False + + +def plan_compress(request: CompressRequest) -> CompressionJob | ArchiveJob: + """Turn a compress request into a planned job. + + One file becomes a single-file container; a directory, several inputs, or + an archive format becomes an archive. + + Args: + request: The parsed request. + + Returns: + The planned job, which may still be unable to run (see its plan). + + Raises: + ValueError: If a single-file format was asked for with several inputs + or a directory. + """ + fmt: str | None = request.format or infer_format(request.output) + many: bool = len(request.inputs) > 1 or any(p.is_dir() for p in request.inputs) + + if format_is_single_file(fmt) and many: + raise ValueError( + f"{fmt} holds a single file; " + f"use an archive format such as tar.{fmt} for several" + ) + + if is_archive_format(fmt) or (many and fmt is None): + fmt = fmt or DEFAULT_ARCHIVE_FORMAT + output: Path = request.output or default_archive_output(request.inputs[0], fmt) + archive_options = ArchiveOptions( + format=fmt.lower(), + compression_level=request.level, + overwrite=request.overwrite, + threads=request.threads, + ) + + return ArchiveJob.from_paths( + sources=request.inputs, output=output, options=archive_options + ) + + source: Path = request.inputs[0] + options = CompressionOptions( + algo=request.algo, + strategy=request.strategy, + level=request.level, + format=fmt, + overwrite=request.overwrite, + threads=request.threads, + ) + + job = CompressionJob.from_file(src=source, dest=None, options=options) + # Planned once to learn the default name, then repointed if `-o` moves it + dest: Path = resolve_single_output(request.output, job.plan.dest) + if dest != job.plan.dest: + job = CompressionJob.from_file(src=source, dest=dest, options=options) + + return job + + +def plan_extract(request: ExtractRequest) -> list[ExtractJob | DecompressionJob]: + """Turn an extract request into one planned job per input. + + Each input is dispatched on its own bytes, not its name: an archive is + unpacked into a directory, anything else is decompressed to one file. + + Args: + request: The parsed request. + + Returns: + The planned jobs, in input order; each may still be unable to run. + """ + jobs: list[ExtractJob | DecompressionJob] = [] + for source in request.inputs: + if looks_like_archive(source): + jobs.append( + ExtractJob.from_archive( + archive=source, output_dir=request.output, options=request.options + ) + ) + continue + + plan = partial( + DecompressionJob.from_file, + src=source, + overwrite=request.options.overwrite, + threads=request.options.threads, + restore_name=request.restore_name, + ) + job: DecompressionJob = plan(dest=None) + dest: Path = resolve_single_output(request.output, job.plan.dest) + jobs.append(job if dest == job.plan.dest else plan(dest=dest)) + + return jobs diff --git a/tests/python/cli/test_compress.py b/tests/python/cli/test_compress.py index 7a060f7..956a7c2 100644 --- a/tests/python/cli/test_compress.py +++ b/tests/python/cli/test_compress.py @@ -2,9 +2,6 @@ from __future__ import annotations -import gzip -import os -import tarfile from pathlib import Path import pytest @@ -78,99 +75,26 @@ def test_existing_output( class TestSmartCompress: - """Test `compress` picks the job from its inputs and the format asked for.""" + """Test `compress` runs the job dispatch plans; the rules are in frontend/test_dispatch.py.""" - def test_one_file_becomes_a_compresso_container(self, payload: Path) -> None: - """Test that with no format, a single file gets the .comp container.""" - invoke("compress", payload, "-q") - - assert payload.with_suffix(payload.suffix + ".comp").exists() - - @pytest.mark.parametrize("fmt", ["gz", "bz2", "xz", "zst", "lz4", "sz"]) - def test_one_file_into_a_standalone_container( - self, payload: Path, tmp_path: Path, fmt: str - ) -> None: - """Test that -f names a single-file container, the suffix follows from it, - and it comes back.""" - restored = tmp_path / "restored.bin" - - invoke("compress", payload, "-f", fmt, "-q") - invoke("extract", f"{payload}.{fmt}", "-o", restored, "-q") - - assert restored.read_bytes() == payload.read_bytes() - - def test_standalone_output_is_a_real_gzip( - self, payload: Path, tmp_path: Path + def test_a_directory_round_trips_as_an_archive( + self, source_tree: Path, tmp_path: Path ) -> None: - """Test that the .gz written is the format, not merely the name.""" - output = tmp_path / "out.gz" - invoke("compress", payload, "-o", output, "-q") - - with gzip.open(output, "rb") as f: - assert f.read() == payload.read_bytes() - - def test_a_directory_becomes_an_archive(self, source_tree: Path) -> None: - """Test that a directory needs a container that holds many entries.""" + """Test that a directory is written as the default archive and comes back.""" invoke("compress", source_tree, "-q") - assert (source_tree.parent / f"{source_tree.name}.zip").exists() - - def test_several_files_become_an_archive( - self, payload: Path, tmp_path: Path - ) -> None: - """Test that multiple files become an archive.""" - second = tmp_path / "second.bin" - second.write_bytes(os.urandom(1024)) - output = tmp_path / "both.tar" - - invoke("compress", payload, second, "-o", output, "-q") - - with tarfile.open(output) as tar: - assert sorted(tar.getnames()) == [payload.name, second.name] - - @pytest.mark.parametrize( - "name,expected_magic", - [ - ("out.gz", b"\x1f\x8b"), - ("out.zst", b"\x28\xb5\x2f\xfd"), - ("out.zip", b"PK"), - ], - ) - def test_format_inferred_from_the_output_name( - self, payload: Path, tmp_path: Path, name: str, expected_magic: bytes - ) -> None: - """Test that a recognisable extension on -o chooses the format by itself.""" - output = tmp_path / name - - invoke("compress", payload, "-o", output, "-q") - - assert output.read_bytes().startswith(expected_magic) - - def test_explicit_format_beats_the_output_name( - self, payload: Path, tmp_path: Path - ) -> None: - """Test that -f overrides whatever the extension suggests.""" - output = tmp_path / "misleading.gz" - - invoke("compress", payload, "-o", output, "-f", "zst", "-q") - - assert output.read_bytes().startswith(b"\x28\xb5\x2f\xfd") - - @pytest.mark.parametrize( - "name", ["out.tar.zst", "out.tar.lz4", "out.tlz4", "out.tar.sz"] - ) - def test_tar_inside_a_codec_is_an_archive( - self, source_tree: Path, tmp_path: Path, name: str - ) -> None: - """Test that two-part extensions name an archive, not their codec alone.""" - output = tmp_path / name - - invoke("compress", source_tree, "-o", output, "-q") - - # Round-trips as an archive rather than as a lone compressed file dest = tmp_path / "back" - invoke("extract", output, "-o", dest, "-q") - assert (dest / source_tree.name).is_dir() + invoke( + "extract", + source_tree.with_name(f"{source_tree.name}.zip"), + "-o", + dest, + "-q", + ) + + for original in source_tree.iterdir(): + restored = dest / source_tree.name / original.name + assert restored.read_bytes() == original.read_bytes() def test_single_file_format_refuses_several_inputs( self, source_tree: Path, tmp_path: Path diff --git a/tests/python/cli/test_extract.py b/tests/python/cli/test_extract.py index 6aeaa59..cd75403 100644 --- a/tests/python/cli/test_extract.py +++ b/tests/python/cli/test_extract.py @@ -201,70 +201,7 @@ def test_name_flag( class TestSmartExtract: - """Test `extract` picks the job from each file's own bytes.""" - - def test_dispatch_follows_the_bytes_not_the_name( - self, payload: Path, tmp_path: Path - ) -> None: - """Test that a renamed .gz is still recognised as one.""" - packed = tmp_path / "p.gz" - invoke("compress", payload, "-o", packed, "-q") - - disguised = tmp_path / "mystery.dat" - disguised.write_bytes(packed.read_bytes()) - restored = tmp_path / "back.bin" - - invoke("extract", disguised, "-o", restored, "-q") - - assert restored.read_bytes() == payload.read_bytes() - - def test_an_archive_named_like_anything_still_extracts( - self, - source_tree: Path, - tmp_path: Path, - make_archive: Callable[[str], Path], - ) -> None: - """Test that a renamed archive still extracts.""" - disguised = tmp_path / "riddle.bin" - disguised.write_bytes(make_archive("tar").read_bytes()) - dest = tmp_path / "out" - - invoke("extract", disguised, "-o", dest, "-q") - - assert (dest / source_tree.name).is_dir() - - def test_an_archive_unpacks_beside_itself( - self, - source_tree: Path, - tmp_path: Path, - make_archive: Callable[[str], Path], - ) -> None: - """Test that with no -o, entries land next to the archive, not in the cwd.""" - packed = make_archive("tar") - - # The sources are beside the archive, so they would be in the way - for f in source_tree.iterdir(): - f.unlink() - source_tree.rmdir() - - invoke("extract", packed, "-q") - - assert (tmp_path / source_tree.name).is_dir() - assert not (Path.cwd() / source_tree.name).exists() - - def test_output_directory_receives_a_single_file( - self, payload: Path, tmp_path: Path - ) -> None: - """Test that -o naming an existing directory puts the file inside it.""" - packed = tmp_path / "p.gz" - invoke("compress", payload, "-o", packed, "-q") - - into = tmp_path / "into" - into.mkdir() - - invoke("extract", packed, "-o", into, "-q") - - assert (into / "p").read_bytes() == payload.read_bytes() + """Test `extract` runs the jobs dispatch plans; the rules are in frontend/test_dispatch.py.""" def test_several_inputs_in_one_go(self, payload: Path, tmp_path: Path) -> None: """Test that each input is dispatched on its own.""" diff --git a/tests/python/frontend/test_dispatch.py b/tests/python/frontend/test_dispatch.py index 5391a98..1b813e8 100644 --- a/tests/python/frontend/test_dispatch.py +++ b/tests/python/frontend/test_dispatch.py @@ -7,16 +7,32 @@ import pytest +from compresso.frontend.api import CompressionJob, DecompressionJob +from compresso.frontend.archive_api import ArchiveJob, ExtractJob from compresso.frontend.dispatch import ( + CompressRequest, + ExtractRequest, default_archive_output, format_is_single_file, infer_format, is_archive_format, looks_like_archive, + plan_compress, + plan_extract, resolve_single_output, ) +@pytest.fixture +def tree(tmp_path: Path) -> Path: + """A directory holding one file.""" + root = tmp_path / "tree" + root.mkdir() + (root / "a.txt").write_text("a") + + return root + + class TestInferFormat: """Test reading a format off an output filename.""" @@ -135,3 +151,184 @@ def test_anything_else_is_the_file(self, tmp_path: Path) -> None: out = tmp_path / "out" / "y.gz" assert resolve_single_output(out, tmp_path / "x.gz") == out + + +class TestPlanCompress: + """Test `compress` picks the job from its inputs and the format asked for.""" + + def test_one_file_becomes_a_compresso_container(self, small_file: Path) -> None: + """Test that one file with no format defaults to a `.comp` beside it.""" + job = plan_compress(CompressRequest([small_file])) + + assert isinstance(job, CompressionJob) + assert job.plan.options.format is None + assert job.plan.dest == small_file.with_name("small.txt.comp") + + @pytest.mark.parametrize("fmt", ["gz", "bz2", "xz", "zst", "lz4", "sz"]) + def test_one_file_into_a_standalone_container( + self, small_file: Path, fmt: str + ) -> None: + """Test that each single-file format compresses one file beside it.""" + job = plan_compress(CompressRequest([small_file], format=fmt)) + + assert isinstance(job, CompressionJob) + assert job.plan.can_compress + assert job.plan.dest == small_file.with_name(f"small.txt.{fmt}") + + def test_a_directory_becomes_an_archive(self, tree: Path) -> None: + """Test that a directory with no format defaults to a zip beside it.""" + job = plan_compress(CompressRequest([tree])) + + assert isinstance(job, ArchiveJob) + assert job.plan.options.format == "zip" + assert job.plan.output == tree.with_name("tree.zip") + + def test_several_files_become_an_archive( + self, small_file: Path, tmp_path: Path + ) -> None: + """Test that several files are archived in the output's format.""" + second = tmp_path / "second.txt" + second.write_text("b") + output = tmp_path / "both.tar" + + job = plan_compress(CompressRequest([small_file, second], output=output)) + + assert isinstance(job, ArchiveJob) + assert job.plan.options.format == "tar" + assert job.plan.output == output + + @pytest.mark.parametrize( + ("name", "kind", "fmt"), + [ + ("out.gz", CompressionJob, "gz"), + ("out.zst", CompressionJob, "zst"), + ("out.zip", ArchiveJob, "zip"), + ], + ) + def test_format_inferred_from_the_output_name( + self, small_file: Path, tmp_path: Path, name: str, kind: type, fmt: str + ) -> None: + """Test that the output's suffix picks both the format and the job kind.""" + output = tmp_path / name + + job = plan_compress(CompressRequest([small_file], output=output)) + + assert isinstance(job, kind) + assert job.plan.options.format == fmt + + def test_explicit_format_beats_the_output_name( + self, small_file: Path, tmp_path: Path + ) -> None: + """Test that `--format` wins over the output's suffix, keeping its name.""" + output = tmp_path / "misleading.gz" + + job = plan_compress(CompressRequest([small_file], output=output, format="zst")) + + assert isinstance(job, CompressionJob) + assert job.plan.options.format == "zst" + assert job.plan.dest == output + + @pytest.mark.parametrize( + "name", ["out.tar.zst", "out.tar.lz4", "out.tlz4", "out.tar.sz"] + ) + def test_tar_inside_a_codec_is_an_archive( + self, tree: Path, tmp_path: Path, name: str + ) -> None: + """Test that compressed tar suffixes, short forms included, archive.""" + job = plan_compress(CompressRequest([tree], output=tmp_path / name)) + + assert isinstance(job, ArchiveJob) + assert job.plan.can_run + assert job.plan.options.format == name.removeprefix("out.") + + def test_existing_directory_receives_the_file( + self, small_file: Path, tmp_path: Path + ) -> None: + """Test that an existing `-o` directory gets the default filename.""" + into = tmp_path / "into" + into.mkdir() + + job = plan_compress(CompressRequest([small_file], output=into, format="gz")) + + assert isinstance(job, CompressionJob) + assert job.plan.dest == into / "small.txt.gz" + + def test_single_file_format_refuses_several_inputs(self, tree: Path) -> None: + """Test that a single-file format on a directory errors, suggesting tar.""" + with pytest.raises(ValueError, match="tar.gz"): + plan_compress(CompressRequest([tree], format="gz")) + + def test_unknown_format_cannot_run(self, small_file: Path) -> None: + """Test that an unknown format plans a job that refuses to run.""" + job = plan_compress(CompressRequest([small_file], format="bogus")) + + assert isinstance(job, CompressionJob) + assert not job.plan.can_compress + + +class TestPlanExtract: + """Test `extract` picks each input's job from its own bytes.""" + + def test_dispatch_follows_the_bytes_not_the_name(self, tmp_path: Path) -> None: + """Test that a gzip stream with an unknown suffix still decompresses.""" + disguised = tmp_path / "mystery.dat" + disguised.write_bytes(gzip.compress(b"a")) + restored = tmp_path / "back.bin" + + [job] = plan_extract(ExtractRequest([disguised], output=restored)) + + assert isinstance(job, DecompressionJob) + assert job.plan.format == "gzip" + assert job.plan.dest == restored + + def test_an_archive_named_like_anything_still_extracts( + self, tmp_path: Path + ) -> None: + """Test that a zip without a `.zip` suffix plans a runnable extraction.""" + disguised = tmp_path / "riddle.bin" + with zipfile.ZipFile(disguised, "w") as zf: + zf.writestr("a.txt", "a") + dest = tmp_path / "out" + + [job] = plan_extract(ExtractRequest([disguised], output=dest)) + + assert isinstance(job, ExtractJob) + assert job.plan.can_run + assert job.plan.output_dir == dest + + def test_an_archive_unpacks_beside_itself(self, tmp_path: Path) -> None: + """Test that without `-o` an archive extracts into its own directory.""" + archive = tmp_path / "x.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr("a.txt", "a") + + [job] = plan_extract(ExtractRequest([archive])) + + assert isinstance(job, ExtractJob) + assert job.plan.output_dir == tmp_path + + def test_output_directory_receives_a_single_file(self, tmp_path: Path) -> None: + """Test that an existing `-o` directory gets the suffix-stripped name.""" + packed = tmp_path / "p.gz" + packed.write_bytes(gzip.compress(b"a")) + into = tmp_path / "into" + into.mkdir() + + [job] = plan_extract(ExtractRequest([packed], output=into)) + + assert isinstance(job, DecompressionJob) + assert job.plan.dest == into / "p" + + def test_each_input_is_dispatched_on_its_own(self, tmp_path: Path) -> None: + """Test that a mixed list plans one job per input, in order.""" + packed = tmp_path / "a.gz" + packed.write_bytes(gzip.compress(b"a")) + archive = tmp_path / "b.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr("b.txt", "b") + + first, second = plan_extract(ExtractRequest([packed, archive])) + + assert isinstance(first, DecompressionJob) + assert first.plan.dest == tmp_path / "a" + assert isinstance(second, ExtractJob) From 140190cf930225d5b52923c1dba67395ce9c5b19 Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Thu, 22 Oct 2026 22:34:22 +0100 Subject: [PATCH 04/31] feat(frontend): infer an archive's format from its output name, defaulting to zip - plan_archive() now reads an unset format off the output's suffix, falling back to zip - 'ArchiveOptions.format' defaults to None, and a caller passing a named output gets the format they named - 'ArchiveJob.run' asserts that the planner resolved it - Format tables, canonical_standalone_format(), infer_format(), and 'DEFAULT_ARCHIVE_FORMAT' move to a new frontend/_formats.py --- src/compresso/frontend/_formats.py | 67 +++++++++++++++++++++++ src/compresso/frontend/api.py | 35 ++---------- src/compresso/frontend/archive_api.py | 18 ++++-- src/compresso/frontend/dispatch.py | 39 ++----------- src/compresso/meson.build | 1 + tests/python/frontend/test_archive_api.py | 25 ++++++++- tests/python/frontend/test_dispatch.py | 2 +- 7 files changed, 115 insertions(+), 72 deletions(-) create mode 100644 src/compresso/frontend/_formats.py diff --git a/src/compresso/frontend/_formats.py b/src/compresso/frontend/_formats.py new file mode 100644 index 0000000..45645cc --- /dev/null +++ b/src/compresso/frontend/_formats.py @@ -0,0 +1,67 @@ +"""Format names and the rules for reading one from an output path.""" + +from __future__ import annotations + +from pathlib import Path + +from .._core import format_is_archive, standalone_formats + +# What an archive is written as when neither the caller nor its name says +DEFAULT_ARCHIVE_FORMAT = "zip" + +_STANDALONE = standalone_formats() + +# Single-file container formats, as opposed to the Compresso container or an +# archive, including aliases +STANDALONE_FORMATS: dict[str, str] = { + alias: f["name"] for f in _STANDALONE for alias in (f["name"], *f["aliases"]) +} + +# Suffix each standalone format conventionally uses, for naming an output the +# caller did not name themselves +STANDALONE_SUFFIXES: dict[str, str] = { + f["name"]: "." + f["exts"][0] for f in _STANDALONE +} + + +def canonical_standalone_format(name: str | None) -> str | None: + """Return the canonical name for a single-file format, or None. + + Args: + name: A format as the user wrote it, e.g. "gz". + + Returns: + The canonical name (e.g. "gzip"), or None if it is not a single-file + format; includes None, archive formats and unknown names. + """ + if not name: + return None + + return STANDALONE_FORMATS.get(name.lower()) + + +def infer_format(output: Path | None) -> str | None: + """Infer a format from an output filename. + + Args: + output: The path the user asked to write, or None. + + Returns: + The format named by the suffix, or None if it names none; includes + `.comp`, whose container is the default anyway. + """ + if output is None: + return None + + # Mirrors the core (`x.tar.lz4` is tar in lz4); unwritable formats + # still match, so the default is never written under their suffix + suffix: str = output.suffix.lower().lstrip(".") + if Path(output.stem).suffix.lower() == ".tar" and format_is_archive( + f"tar.{suffix}" + ): + return f"tar.{suffix}" + + if suffix in STANDALONE_FORMATS or format_is_archive(suffix): + return suffix + + return None diff --git a/src/compresso/frontend/api.py b/src/compresso/frontend/api.py index 94e5ba1..5dcdeb3 100644 --- a/src/compresso/frontend/api.py +++ b/src/compresso/frontend/api.py @@ -18,47 +18,22 @@ detect_format, format_is_archive, gzip_stored_name, - standalone_formats, ) from .._core import get_default_backend_for_strategy as default_backend from .._levels import to_core_level from ..introspect.file_inspect import InspectResult from ..introspect.file_inspect import inspect as inspect_file from ..introspect.speeds import get_estimated_speeds +from ._formats import ( + STANDALONE_FORMATS, + STANDALONE_SUFFIXES, + canonical_standalone_format, +) from ._job import JobResult, ProgressCallback, ThreadedJob, to_core_progress from .archive_api import _OVERWRITE_CODES, OverwriteMode MB = 1024 * 1024 -_STANDALONE = standalone_formats() - -# Single-file container formats, as opposed to the Compresso container or an -# archive, including aliases -STANDALONE_FORMATS: dict[str, str] = { - alias: f["name"] for f in _STANDALONE for alias in (f["name"], *f["aliases"]) -} - -# Suffix each standalone format conventionally uses, for naming an output the -# caller did not name themselves -STANDALONE_SUFFIXES: dict[str, str] = { - f["name"]: "." + f["exts"][0] for f in _STANDALONE -} - - -def canonical_standalone_format(name: str | None) -> str | None: - """Return the canonical name for a single-file format, or None. - - Args: - name: A format as the user wrote it, e.g. "gz". - - Returns: - The canonical name (e.g. "gzip"), or None if it is not a single-file - format — which includes None, archive formats and unknown names. - """ - if not name: - return None - return STANDALONE_FORMATS.get(name.lower()) - @dataclass(frozen=True) class CompressionOptions: diff --git a/src/compresso/frontend/archive_api.py b/src/compresso/frontend/archive_api.py index 77e47e4..bf11fa0 100644 --- a/src/compresso/frontend/archive_api.py +++ b/src/compresso/frontend/archive_api.py @@ -18,6 +18,7 @@ list_archive_contents, ) from .._levels import to_core_level +from ._formats import DEFAULT_ARCHIVE_FORMAT, infer_format from ._job import JobResult, ProgressCallback, ThreadedJob, to_core_progress @@ -109,7 +110,8 @@ class ArchiveOptions: """Represents the options for an archive operation. Attributes: - format: The archive/compression format to write. + format: The archive/compression format to write, or None to take it + from the output's suffix, falling back to zip. compression_level: The compression level to use, or None for the format's default. overwrite: What to do when the destination archive already exists. @@ -117,7 +119,7 @@ class ArchiveOptions: the output is the same for any count. """ - format: str = "tar.zst" # Default to tar with zstd + format: str | None = None compression_level: int | None = None overwrite: OverwriteMode = OverwriteMode.RENAME threads: int = 0 @@ -224,6 +226,9 @@ def plan_archive( source_paths: list[Path] = [Path(s) for s in sources] output_path = Path(output) + fmt: str = options.format or infer_format(output_path) or DEFAULT_ARCHIVE_FORMAT + options = replace(options, format=fmt) + try: # Plain strings still work at runtime options = replace(options, overwrite=OverwriteMode(options.overwrite)) @@ -265,7 +270,7 @@ def plan_archive( reason_if_unavailable=f"Source path does not exist: {missing[0]}", ) - if not format_is_archive(options.format): + if not format_is_archive(fmt): return ArchivePlan( sources=source_paths, output=output_path, @@ -273,13 +278,13 @@ def plan_archive( total_input_size=0, entry_count=len(source_paths), can_run=False, - reason_if_unavailable=f"Format does not support archives: {options.format}", + reason_if_unavailable=f"Format does not support archives: {fmt}", ) if options.compression_level is not None: try: # The level belongs to the codec stage, or to the container itself - check_level(options.compression_level, format=options.format) + check_level(options.compression_level, format=fmt) except ValueError as e: return ArchivePlan( @@ -481,6 +486,9 @@ def run( plan=self.plan, ) + # plan_archive resolves an unset format before a plan can run + assert self.plan.options.format is not None + total: int = self.plan.total_input_size try: actual_path = create_archive( diff --git a/src/compresso/frontend/dispatch.py b/src/compresso/frontend/dispatch.py index 16a13af..e750c75 100644 --- a/src/compresso/frontend/dispatch.py +++ b/src/compresso/frontend/dispatch.py @@ -7,13 +7,12 @@ from pathlib import Path from .._core import detect_format, format_is_archive -from .api import ( - STANDALONE_FORMATS, - CompressionJob, - CompressionOptions, - DecompressionJob, +from ._formats import ( + DEFAULT_ARCHIVE_FORMAT, canonical_standalone_format, + infer_format, ) +from .api import CompressionJob, CompressionOptions, DecompressionJob from .archive_api import ( ArchiveJob, ArchiveOptions, @@ -22,36 +21,6 @@ OverwriteMode, ) -# What `compress` reaches for when given several inputs and no format -DEFAULT_ARCHIVE_FORMAT = "zip" - - -def infer_format(output: Path | None) -> str | None: - """Infer a format from an output filename. - - Args: - output: The path the user asked to write, or None. - - Returns: - The format named by the suffix, or None if it names none — which - includes `.comp`, whose container is the default anyway. - """ - if output is None: - return None - - # Mirrors the core (`x.tar.lz4` is tar in lz4); unwritable formats - # still match, so the default is never written under their suffix - suffix: str = output.suffix.lower().lstrip(".") - if Path(output.stem).suffix.lower() == ".tar" and format_is_archive( - f"tar.{suffix}" - ): - return f"tar.{suffix}" - - if suffix in STANDALONE_FORMATS or format_is_archive(suffix): - return suffix - - return None - def looks_like_archive(path: Path) -> bool: """Return whether `path`'s own bytes say it holds many entries. diff --git a/src/compresso/meson.build b/src/compresso/meson.build index 50ae493..7ed94f8 100644 --- a/src/compresso/meson.build +++ b/src/compresso/meson.build @@ -63,6 +63,7 @@ py.install_sources( py.install_sources( 'frontend/__init__.py', + 'frontend/_formats.py', 'frontend/_job.py', 'frontend/api.py', 'frontend/archive_api.py', diff --git a/tests/python/frontend/test_archive_api.py b/tests/python/frontend/test_archive_api.py index 595aa13..eb83085 100644 --- a/tests/python/frontend/test_archive_api.py +++ b/tests/python/frontend/test_archive_api.py @@ -36,10 +36,33 @@ def test_archive_options_default(self) -> None: """Test creating ArchiveOptions with defaults.""" opts = ArchiveOptions() - assert opts.format == "tar.zst" + assert opts.format is None assert opts.compression_level is None assert opts.overwrite is OverwriteMode.RENAME + @pytest.mark.parametrize( + ("name", "fmt", "expected"), + [ + ("out.tar.zst", None, "tar.zst"), + ("out.bin", None, "zip"), + ("out.tar.zst", "tar", "tar"), + ], + ids=["suffix", "unrecognised-suffix", "explicit-wins"], + ) + def test_format_comes_from_the_output_name( + self, + small_file: Path, + tmp_path: Path, + name: str, + fmt: str | None, + expected: str, + ) -> None: + """Test that an unset format is read from the output's suffix, else zip.""" + plan = plan_archive([small_file], tmp_path / name, ArchiveOptions(format=fmt)) + + assert plan.can_run, plan.reason_if_unavailable + assert plan.options.format == expected + def test_plan_archive_valid(self, sample_text_file: Path, tmp_path: Path) -> None: """Test that a plan over existing sources can run and sums input size.""" out = tmp_path / "out.tar.zst" diff --git a/tests/python/frontend/test_dispatch.py b/tests/python/frontend/test_dispatch.py index 1b813e8..15a00c7 100644 --- a/tests/python/frontend/test_dispatch.py +++ b/tests/python/frontend/test_dispatch.py @@ -7,6 +7,7 @@ import pytest +from compresso.frontend._formats import infer_format from compresso.frontend.api import CompressionJob, DecompressionJob from compresso.frontend.archive_api import ArchiveJob, ExtractJob from compresso.frontend.dispatch import ( @@ -14,7 +15,6 @@ ExtractRequest, default_archive_output, format_is_single_file, - infer_format, is_archive_format, looks_like_archive, plan_compress, From 5aeb7490047654ecf91a95cbcef73e2e48b03e53 Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Thu, 22 Oct 2026 23:24:56 +0100 Subject: [PATCH 05/31] refactor(cli): move all CLI printing into _render - '_render' becomes a package holding all CLi printing and output logic - New jobs.py holds the before-and-after report for each job kind - New info.py holds the entry listing and the archive and single-file metadata reports - The package root holds formatting, progress and exit codes - Commands are simplified to the plan check, timing, progress bar, and the exit, with each report now one call taking a plan --- .../cli/{_render.py => _render/__init__.py} | 10 +- src/compresso/cli/_render/info.py | 229 ++++++++++++++++++ src/compresso/cli/_render/jobs.py | 169 +++++++++++++ src/compresso/cli/compress.py | 69 +----- src/compresso/cli/extract.py | 87 +------ src/compresso/cli/inspect.py | 182 +------------- src/compresso/meson.build | 8 +- tests/python/cli/test_extract.py | 2 +- 8 files changed, 445 insertions(+), 311 deletions(-) rename src/compresso/cli/{_render.py => _render/__init__.py} (94%) create mode 100644 src/compresso/cli/_render/info.py create mode 100644 src/compresso/cli/_render/jobs.py diff --git a/src/compresso/cli/_render.py b/src/compresso/cli/_render/__init__.py similarity index 94% rename from src/compresso/cli/_render.py rename to src/compresso/cli/_render/__init__.py index 1a598d4..ba33141 100644 --- a/src/compresso/cli/_render.py +++ b/src/compresso/cli/_render/__init__.py @@ -1,4 +1,8 @@ -"""Shared rendering for the CLI: formatting, progress and exit codes.""" +"""Everything the CLI prints. + +The package root holds formatting, progress and exit codes; `jobs` and `info` hold +each command's reports. +""" from __future__ import annotations @@ -7,8 +11,8 @@ from contextlib import contextmanager from typing import NoReturn -from ..frontend._job import JobResult, ProgressCallback -from ._app import app +from ...frontend._job import JobResult, ProgressCallback +from .._app import app # Exit codes EXIT_OK = 0 diff --git a/src/compresso/cli/_render/info.py b/src/compresso/cli/_render/info.py new file mode 100644 index 0000000..46689c5 --- /dev/null +++ b/src/compresso/cli/_render/info.py @@ -0,0 +1,229 @@ +"""What describes an archive or a single-file container: its entries and metadata.""" + +from __future__ import annotations + +import stat +import time +from pathlib import Path + +import orjson + +from ...frontend.archive_api import ArchiveEntry +from ...introspect.file_inspect import InspectResult +from .._app import app +from . import EXIT_USAGE, fail, format_size, format_time, succeed + + +def list_entries(entries: list[ArchiveEntry]) -> None: + """Print one line per entry: its size, then its path. + + Directories and symlinks leave the size blank, a symlink is followed by + its target, and each entry is indented once for every directory above it + that was already listed. + + Args: + entries: The archive's entries, in archive order. + """ + listed_dirs: set[str] = set() + + for entry in entries: + parts = entry.path.rstrip("/").split("/") + depth = sum("/".join(parts[:i]) in listed_dirs for i in range(1, len(parts))) + + if entry.is_dir: + listed_dirs.add("/".join(parts)) + + # A symlink's stored size is the length of its target; hardlink's have no data + size = ( + "" + if entry.is_dir or entry.is_symlink or entry.is_hardlink + else format_size(size_bytes=entry.size) + ) + target = f" -> {entry.link_target}" if entry.is_symlink else "" + + # "1023.99 KB" is the widest format_size gives + print(f"{size:>10} {' ' * depth}{entry.path}{target}") + + +def _entry_detail_line(entry: ArchiveEntry) -> str: + """Format one entry's full detail: mode, mtime, size, path, and target. + + Args: + entry: The archive entry to format. + + Returns: + A single formatted line for the entry. + """ + if entry.is_dir: + type_bit = stat.S_IFDIR + elif entry.is_symlink: + type_bit = stat.S_IFLNK + else: + type_bit = stat.S_IFREG + + mode = stat.filemode(type_bit | entry.mode) + mtime = time.strftime("%Y-%m-%d %H:%M", time.localtime(entry.mtime)) + has_no_data = entry.is_dir or entry.is_symlink or entry.is_hardlink + size = "" if has_no_data else format_size(size_bytes=entry.size) + + if entry.is_symlink: + target = f" -> {entry.link_target}" + elif entry.is_hardlink: + target = f" link to {entry.link_target}" + else: + target = "" + + detail = "" + if entry.compressed_size is not None: + detail = f" (compressed {format_size(size_bytes=entry.compressed_size)}" + if entry.crc is not None: + detail += f", crc {entry.crc:08x}" + if entry.method is not None: + detail += f", method {entry.method}" + detail += ")" + + return f"{mode} {mtime} {size:>10} {entry.path}{target}{detail}" + + +def archive_info( + file: Path, + fmt: str | None, + entries: list[ArchiveEntry], + *, + as_json: bool, + show_entries: bool, +) -> None: + """Print an archive's metadata, as text or JSON. + + Args: + file: The archive's path. + fmt: The archive's detected format. + entries: The archive's entries. + as_json: If True, print JSON instead of text. + show_entries: If True, also print each entry's own detail. + """ + total_size = sum(entry.size for entry in entries) + archive_size: int = file.stat().st_size + + if as_json: + data: dict[str, object] = { + "path": str(file), + "format": fmt, + "is_archive": True, + "entry_count": len(entries), + "total_size": total_size, + "archive_size": archive_size, + } + if show_entries: + data["entries"] = [ + { + "path": entry.path, + "size": entry.size, + "is_dir": entry.is_dir, + "is_symlink": entry.is_symlink, + "is_hardlink": entry.is_hardlink, + "mtime": entry.mtime, + "mode": entry.mode, + "link_target": entry.link_target, + "compressed_size": entry.compressed_size, + "crc": entry.crc, + "method": entry.method, + } + for entry in entries + ] + print(orjson.dumps(data, option=orjson.OPT_INDENT_2).decode()) + return + + lines: list[str] = [ + f"File: {file}", + f"Format: {fmt}", + f"Entries: {len(entries)}", + f"Total size: {format_size(size_bytes=total_size)}", + f"Archive size: {format_size(size_bytes=archive_size)}", + ] + + if total_size: + ratio: int | float = (archive_size / total_size) * 100 + lines.append(f"Compression: {ratio:.1f}% of original") + + print("\n".join(lines)) + + if show_entries: + print("\n" + "\n".join(_entry_detail_line(entry) for entry in entries)) + + +def file_info(result: InspectResult, *, as_json: bool) -> None: + """Print a single-file container's metadata, as text or JSON. + + Args: + result: The container's inspection. + as_json: If True, print JSON instead of text. + """ + if as_json: + data: dict[str, str | int | None | float] = { + "path": str(object=result.path), + "is_compresso": result.is_compresso, + "header_ok": result.header_ok, + "version": result.version, + "algo_id": result.algo_id, + "algo_name": result.algo_name, + "level": result.level, + "flags": result.flags, + "orig_size": result.orig_size, + "backend_available": result.backend_available, + "can_decompress": result.can_decompress, + "estimated_decomp_s": result.estimated_decomp_s, + "reason": result.reason, + } + print(orjson.dumps(data, option=orjson.OPT_INDENT_2).decode()) + return + + print(f"File: {result.path}\n") + + if not result.is_compresso: + reason = f"\n Reason: {result.reason}" if result.reason else "" + fail(f"Not a valid Compresso file{reason}", EXIT_USAGE) + + if not result.header_ok: + reason = f"\n Reason: {result.reason}" if result.reason else "" + fail(f"Invalid file header{reason}", EXIT_USAGE) + + succeed("Valid Compresso file\n") + + lines: list[str] = [ + f"Algorithm: {result.algo_name or 'Unknown'} (ID: {result.algo_id})", + f"Version: {result.version}", + "", + ] + + if result.level is not None: + lines.append(f"Level: {result.level}") + else: + lines.append("Level: auto") + + if result.orig_size: + lines.append(f"Original size: {format_size(size_bytes=result.orig_size)}") + + compressed_size: int = result.path.stat().st_size + lines.append(f"Compressed size: {format_size(size_bytes=compressed_size)}") + + if result.orig_size: + ratio: int | float = (compressed_size / result.orig_size) * 100 + lines.append(f"Compression: {ratio:.1f}% of original") + + lines += [ + "", + f"Backend available: {'Yes' if result.backend_available else 'No'}", + f"Can decompress: {'Yes' if result.can_decompress else 'No'}", + ] + + if result.estimated_decomp_s: + lines.append( + f"Est. decomp time: {format_time(seconds=result.estimated_decomp_s)}" + ) + lines.append("") + + print("\n".join(lines)) + + if not result.can_decompress and result.reason: + app.echo(message=app.style(text=f"\n⚠ {result.reason}", fg="yellow")) diff --git a/src/compresso/cli/_render/jobs.py b/src/compresso/cli/_render/jobs.py new file mode 100644 index 0000000..5a57409 --- /dev/null +++ b/src/compresso/cli/_render/jobs.py @@ -0,0 +1,169 @@ +"""What a compress, archive, extract or decompress job prints before and after.""" + +from __future__ import annotations + +from pathlib import Path + +from ...frontend.api import CompressionPlan, DecompressionPlan +from ...frontend.archive_api import ArchivePlan, ExtractPlan +from . import format_size, format_time, succeed + + +def compress_plan(plan: CompressionPlan) -> None: + """Print what a single-file compression is about to do. + + Args: + plan: The planned compression. + """ + lines: list[str] = [ + f"Compressing: {plan.src}", + f"Output: {plan.dest}", + f"Algorithm: {plan.backend_name}", + ] + + if not plan.options.format: + lines.append(f"Strategy: {plan.options.strategy}") + + if plan.options.level is not None: + lines.append(f"Level: {plan.options.level}") + + print("\n".join(lines) + "\n") + + +def compress_result(plan: CompressionPlan, written: Path, elapsed: float) -> None: + """Print a finished single-file compression's stats. + + Args: + plan: The plan as it was before the job ran. + written: The path actually written, which a rename may have moved. + elapsed: Seconds the job took. + """ + compressed_size: int = written.stat().st_size + ratio: int | float = ( + (compressed_size / plan.input_size) * 100 if plan.input_size > 0 else 0 + ) + speed_mbs: int | float = ( + (plan.input_size / (1024 * 1024)) / elapsed if elapsed > 0 else 0 + ) + + print() + succeed("Compression successful!\n") + if written != plan.dest: + print(f" Renamed to: {written}") + + print( + f" Original size: {format_size(size_bytes=plan.input_size)}\n" + f" Compressed size: {format_size(size_bytes=compressed_size)}\n" + f" Ratio: {ratio:.1f}% of original\n" + f" Time: {format_time(seconds=elapsed)}\n" + f" Speed: {speed_mbs:.2f} MB/s\n" + f" Saved: {format_size(size_bytes=plan.input_size - compressed_size)}\n" + ) + + +def archive_plan(plan: ArchivePlan) -> None: + """Print what an archive job is about to do. + + Args: + plan: The planned archive. + """ + print( + f"Archiving: {plan.entry_count} source(s)\n" + f"Output: {plan.output}\n" + f"Format: {plan.options.format}\n" + f"Input size: {format_size(size_bytes=plan.total_input_size)}\n" + ) + + +def archive_result(plan: ArchivePlan, written: Path, elapsed: float) -> None: + """Print a finished archive's stats. + + Args: + plan: The plan as it was before the job ran. + written: The path actually written, which a rename may have moved. + elapsed: Seconds the job took. + """ + print() + succeed("Archive created!\n") + if written != plan.output: + print(f" Renamed to: {written}") + + print( + f" Input size: {format_size(size_bytes=plan.total_input_size)}\n" + f" Archive size: {format_size(size_bytes=written.stat().st_size)}\n" + f" Time: {format_time(seconds=elapsed)}\n" + ) + + +def extract_plan(plan: ExtractPlan) -> None: + """Print what an extraction is about to do. + + Args: + plan: The planned extraction. + """ + print( + f"Extracting: {plan.archive}\n" + f"Output dir: {plan.output_dir}\n" + f"Entries: {len(plan.entries)}\n" + ) + + +def extract_result(plan: ExtractPlan, elapsed: float) -> None: + """Print a finished extraction's stats. + + Args: + plan: The extraction's plan. + elapsed: Seconds the job took. + """ + print() + succeed("Extraction successful!\n") + print( + f" Entries: {len(plan.entries)}\n Time: {format_time(seconds=elapsed)}\n" + ) + + +def decompress_plan(plan: DecompressionPlan) -> None: + """Print what a single-file decompression is about to do. + + Args: + plan: The planned decompression. + """ + lines: list[str] = [ + f"Decompressing: {plan.src}", + f"Output: {plan.dest}", + f"Format: {plan.format or plan.inspection.algo_name}", + ] + if plan.inspection.orig_size: + lines.append( + f"Original size: {format_size(size_bytes=plan.inspection.orig_size)}" + ) + + print("\n".join(lines) + "\n") + + +def decompress_result( + plan: DecompressionPlan, written: Path, compressed_size: int, elapsed: float +) -> None: + """Print a finished single-file decompression's stats. + + Args: + plan: The plan as it was before the job ran. + written: The path actually written, which a rename may have moved. + compressed_size: The source's size in bytes. + elapsed: Seconds the job took. + """ + decompressed_size: int = written.stat().st_size + speed_mbs: int | float = ( + (decompressed_size / (1024 * 1024)) / elapsed if elapsed > 0 else 0 + ) + + succeed("Decompression successful!\n") + if written != plan.dest: + print(f" Renamed to: {written}") + + print( + f" Compressed size: {format_size(size_bytes=compressed_size)}\n" + f" Decompressed size: {format_size(size_bytes=decompressed_size)}\n" + f" Time: {format_time(seconds=elapsed)}\n" + f" Speed: {speed_mbs:.2f} MB/s\n" + ) diff --git a/src/compresso/cli/compress.py b/src/compresso/cli/compress.py index ee69bc0..a188e0a 100644 --- a/src/compresso/cli/compress.py +++ b/src/compresso/cli/compress.py @@ -17,11 +17,9 @@ cancelled, exit_for_result, fail, - format_size, - format_time, progress_bar, - succeed, ) +from ._render.jobs import archive_plan, archive_result, compress_plan, compress_result def _compress_one_file(job: CompressionJob, quiet: bool) -> None: @@ -32,24 +30,12 @@ def _compress_one_file(job: CompressionJob, quiet: bool) -> None: quiet: Whether to suppress progress output. """ plan = job.plan - dest: Path = plan.dest - fmt: str | None = plan.options.format - level: int | None = plan.options.level if not plan.can_compress: fail(f"Error: {plan.reason_if_unavailable}", EXIT_USAGE) if not quiet: - lines: list[str] = [ - f"Compressing: {plan.src}", - f"Output: {plan.dest}", - f"Algorithm: {plan.backend_name}", - ] - if not fmt: - lines.append(f"Strategy: {plan.options.strategy}") - if level is not None: - lines.append(f"Level: {level}") - print("\n".join(lines) + "\n") + compress_plan(plan) start_time: float = time.time() @@ -64,30 +50,10 @@ def _compress_one_file(job: CompressionJob, quiet: bool) -> None: exit_for_result(result, "Compression") - # RENAME may have written somewhere other than `dest`; job.plan is updated - # in place on success to reflect the path actually written - actual_dest: Path = job.plan.dest - compressed_size: int = actual_dest.stat().st_size - ratio: int | float = ( - (compressed_size / plan.input_size) * 100 if plan.input_size > 0 else 0 - ) - speed_mbs: int | float = ( - (plan.input_size / (1024 * 1024)) / elapsed if elapsed > 0 else 0 - ) - + # RENAME may have written somewhere other than `plan.dest`; job.plan is + # replaced on success with the path actually written if not quiet: - print() - succeed("Compression successful!\n") - if actual_dest != dest: - print(f" Renamed to: {actual_dest}") - print( - f" Original size: {format_size(size_bytes=plan.input_size)}\n" - f" Compressed size: {format_size(size_bytes=compressed_size)}\n" - f" Ratio: {ratio:.1f}% of original\n" - f" Time: {format_time(seconds=elapsed)}\n" - f" Speed: {speed_mbs:.2f} MB/s\n" - f" Saved: {format_size(size_bytes=plan.input_size - compressed_size)}\n" - ) + compress_result(plan, job.plan.dest, elapsed) def _archive_many(job: ArchiveJob, quiet: bool) -> None: @@ -98,18 +64,12 @@ def _archive_many(job: ArchiveJob, quiet: bool) -> None: quiet: Whether to suppress progress output. """ plan = job.plan - destination: Path = plan.output if not plan.can_run: fail(f"Error: {plan.reason_if_unavailable}", EXIT_USAGE) if not quiet: - print( - f"Archiving: {plan.entry_count} source(s)\n" - f"Output: {plan.output}\n" - f"Format: {plan.options.format}\n" - f"Input size: {format_size(size_bytes=plan.total_input_size)}\n" - ) + archive_plan(plan) start_time: float = time.time() @@ -126,21 +86,10 @@ def _archive_many(job: ArchiveJob, quiet: bool) -> None: exit_for_result(result, "Archive") - # RENAME may have written somewhere other than `destination`; job.plan is - # updated in place on success to reflect the path actually written - actual_output: Path = job.plan.output - archive_size: int = actual_output.stat().st_size - + # RENAME may have written somewhere other than `plan.output`; job.plan is + # replaced on success with the path actually written if not quiet: - print() - succeed("Archive created!") - if actual_output != destination: - print(f" Renamed to: {actual_output}") - print( - f" Input size: {format_size(size_bytes=plan.total_input_size)}\n" - f" Archive size: {format_size(size_bytes=archive_size)}\n" - f" Time: {format_time(seconds=elapsed)}\n" - ) + archive_result(plan, job.plan.output, elapsed) def compress( diff --git a/src/compresso/cli/extract.py b/src/compresso/cli/extract.py index 6cec6e4..3f93d4e 100644 --- a/src/compresso/cli/extract.py +++ b/src/compresso/cli/extract.py @@ -9,7 +9,6 @@ from .._core import BackendError, Error, HeaderError from ..frontend.api import DecompressionJob from ..frontend.archive_api import ( - ArchiveEntry, ExtractJob, ExtractOptions, OverwriteMode, @@ -23,42 +22,15 @@ cancelled, exit_for_result, fail, - format_size, - format_time, progress_bar, - succeed, ) - - -def list_entries(entries: list[ArchiveEntry]) -> None: - """Print one line per entry: its size, then its path. - - Directories and symlinks leave the size blank, a symlink is followed by - its target, and each entry is indented once for every directory above it - that was already listed. - - Args: - entries: The archive's entries, in archive order. - """ - listed_dirs: set[str] = set() - - for entry in entries: - parts = entry.path.rstrip("/").split("/") - depth = sum("/".join(parts[:i]) in listed_dirs for i in range(1, len(parts))) - - if entry.is_dir: - listed_dirs.add("/".join(parts)) - - # A symlink's stored size is the length of its target; hardlink's have no data - size = ( - "" - if entry.is_dir or entry.is_symlink or entry.is_hardlink - else format_size(size_bytes=entry.size) - ) - target = f" -> {entry.link_target}" if entry.is_symlink else "" - - # "1023.99 KB" is the widest format_size gives - print(f"{size:>10} {' ' * depth}{entry.path}{target}") +from ._render.info import list_entries +from ._render.jobs import ( + decompress_plan, + decompress_result, + extract_plan, + extract_result, +) def _extract_archive(job: ExtractJob, list_only: bool, quiet: bool) -> None: @@ -79,11 +51,7 @@ def _extract_archive(job: ExtractJob, list_only: bool, quiet: bool) -> None: return if not quiet: - print( - f"Extracting: {plan.archive}\n" - f"Output dir: {plan.output_dir}\n" - f"Entries: {len(plan.entries)}\n" - ) + extract_plan(plan) start_time: float = time.time() @@ -103,12 +71,7 @@ def _extract_archive(job: ExtractJob, list_only: bool, quiet: bool) -> None: exit_for_result(result, "Extraction") if not quiet: - print() - succeed("Extraction successful!") - print( - f" Entries: {len(plan.entries)}\n" - f" Time: {format_time(seconds=elapsed)}\n" - ) + extract_result(plan, elapsed) def _decompress_one_file(job: DecompressionJob, list_only: bool, quiet: bool) -> None: @@ -120,7 +83,6 @@ def _decompress_one_file(job: DecompressionJob, list_only: bool, quiet: bool) -> quiet: Whether to suppress output. """ plan = job.plan - dest: Path = plan.dest if list_only: fail( @@ -131,17 +93,8 @@ def _decompress_one_file(job: DecompressionJob, list_only: bool, quiet: bool) -> if not plan.can_run: fail(f"Error: {plan.reason_if_unavailable}", EXIT_USAGE) - insp = plan.inspection - if not quiet: - lines: list[str] = [ - f"Decompressing: {plan.src}", - f"Output: {plan.dest}", - f"Format: {plan.format or insp.algo_name}", - ] - if insp.orig_size: - lines.append(f"Original size: {format_size(size_bytes=insp.orig_size)}") - print("\n".join(lines) + "\n") + decompress_plan(plan) start_time: float = time.time() @@ -160,24 +113,10 @@ def _decompress_one_file(job: DecompressionJob, list_only: bool, quiet: bool) -> exit_for_result(result, "Decompression") - # RENAME may have written somewhere other than `dest`; job.plan is updated - # in place on success to reflect the path actually written - actual_dest: Path = job.plan.dest - decompressed_size: int = actual_dest.stat().st_size - speed_mbs: int | float = ( - (decompressed_size / (1024 * 1024)) / elapsed if elapsed > 0 else 0 - ) - + # RENAME may have written somewhere other than `plan.dest`; job.plan is + # replaced on success with the path actually written if not quiet: - succeed("Decompression successful!\n") - if actual_dest != dest: - print(f" Renamed to: {actual_dest}") - print( - f" Compressed size: {format_size(size_bytes=compressed_size)}\n" - f" Decompressed size: {format_size(size_bytes=decompressed_size)}\n" - f" Time: {format_time(seconds=elapsed)}\n" - f" Speed: {speed_mbs:.2f} MB/s\n" - ) + decompress_result(plan, job.plan.dest, compressed_size, elapsed) def extract( diff --git a/src/compresso/cli/inspect.py b/src/compresso/cli/inspect.py index ff41d5d..835354c 100644 --- a/src/compresso/cli/inspect.py +++ b/src/compresso/cli/inspect.py @@ -2,66 +2,16 @@ from __future__ import annotations -import stat -import time from pathlib import Path from typing import Annotated -import orjson - from .._core import detect_format -from ..frontend.archive_api import ArchiveEntry from ..frontend.archive_api import plan_extraction as plan_archive_extraction from ..frontend.dispatch import looks_like_archive from ..introspect.file_inspect import inspect as inspect_file from ._app import app -from ._render import ( - EXIT_USAGE, - cancelled, - fail, - format_size, - format_time, - succeed, -) - - -def _entry_detail_line(entry: ArchiveEntry) -> str: - """Format one entry's full detail: mode, mtime, size, path, and target. - - Args: - entry: The archive entry to format. - - Returns: - A single formatted line for the entry. - """ - if entry.is_dir: - type_bit = stat.S_IFDIR - elif entry.is_symlink: - type_bit = stat.S_IFLNK - else: - type_bit = stat.S_IFREG - - mode = stat.filemode(type_bit | entry.mode) - mtime = time.strftime("%Y-%m-%d %H:%M", time.localtime(entry.mtime)) - has_no_data = entry.is_dir or entry.is_symlink or entry.is_hardlink - size = "" if has_no_data else format_size(size_bytes=entry.size) - if entry.is_symlink: - target = f" -> {entry.link_target}" - elif entry.is_hardlink: - target = f" link to {entry.link_target}" - else: - target = "" - - detail = "" - if entry.compressed_size is not None: - detail = f" (compressed {format_size(size_bytes=entry.compressed_size)}" - if entry.crc is not None: - detail += f", crc {entry.crc:08x}" - if entry.method is not None: - detail += f", method {entry.method}" - detail += ")" - - return f"{mode} {mtime} {size:>10} {entry.path}{target}{detail}" +from ._render import EXIT_USAGE, cancelled, fail +from ._render.info import archive_info, file_info def _inspect_archive(file: Path, output_json: bool, show_entries: bool) -> None: @@ -77,55 +27,13 @@ def _inspect_archive(file: Path, output_json: bool, show_entries: bool) -> None: if not plan.can_run: fail(f"Error: {plan.reason_if_unavailable}", EXIT_USAGE) - fmt = detect_format(file) - total_size = sum(entry.size for entry in plan.entries) - archive_size: int = file.stat().st_size - - if output_json: - data: dict[str, object] = { - "path": str(file), - "format": fmt, - "is_archive": True, - "entry_count": len(plan.entries), - "total_size": total_size, - "archive_size": archive_size, - } - if show_entries: - data["entries"] = [ - { - "path": entry.path, - "size": entry.size, - "is_dir": entry.is_dir, - "is_symlink": entry.is_symlink, - "is_hardlink": entry.is_hardlink, - "mtime": entry.mtime, - "mode": entry.mode, - "link_target": entry.link_target, - "compressed_size": entry.compressed_size, - "crc": entry.crc, - "method": entry.method, - } - for entry in plan.entries - ] - print(orjson.dumps(data, option=orjson.OPT_INDENT_2).decode()) - return - - lines: list[str] = [ - f"File: {file}", - f"Format: {fmt}", - f"Entries: {len(plan.entries)}", - f"Total size: {format_size(size_bytes=total_size)}", - f"Archive size: {format_size(size_bytes=archive_size)}", - ] - - if total_size: - ratio: int | float = (archive_size / total_size) * 100 - lines.append(f"Compression: {ratio:.1f}% of original") - - print("\n".join(lines)) - - if show_entries: - print("\n" + "\n".join(_entry_detail_line(entry) for entry in plan.entries)) + archive_info( + file, + detect_format(file), + plan.entries, + as_json=output_json, + show_entries=show_entries, + ) def inspect( @@ -160,77 +68,7 @@ def inspect( _inspect_archive(file, output_json, show_entries) return - result = inspect_file(path=file) - - if output_json: - data: dict[str, str | int | None | float] = { - "path": str(object=result.path), - "is_compresso": result.is_compresso, - "header_ok": result.header_ok, - "version": result.version, - "algo_id": result.algo_id, - "algo_name": result.algo_name, - "level": result.level, - "flags": result.flags, - "orig_size": result.orig_size, - "backend_available": result.backend_available, - "can_decompress": result.can_decompress, - "estimated_decomp_s": result.estimated_decomp_s, - "reason": result.reason, - } - print(orjson.dumps(data, option=orjson.OPT_INDENT_2).decode()) - return - - print(f"File: {result.path}\n") - - if not result.is_compresso: - reason = f"\n Reason: {result.reason}" if result.reason else "" - fail(f"Not a valid Compresso file{reason}", EXIT_USAGE) - - if not result.header_ok: - reason = f"\n Reason: {result.reason}" if result.reason else "" - fail(f"Invalid file header{reason}", EXIT_USAGE) - - succeed("Valid Compresso file\n") - - lines: list[str] = [ - f"Algorithm: {result.algo_name or 'Unknown'} (ID: {result.algo_id})", - f"Version: {result.version}", - "", - ] - - if result.level is not None: - lines.append(f"Level: {result.level}") - - else: - lines.append("Level: auto") - - if result.orig_size: - lines.append(f"Original size: {format_size(size_bytes=result.orig_size)}") - - compressed_size: int = file.stat().st_size - lines.append(f"Compressed size: {format_size(size_bytes=compressed_size)}") - - if result.orig_size: - ratio: int | float = (compressed_size / result.orig_size) * 100 - lines.append(f"Compression: {ratio:.1f}% of original") - - lines += [ - "", - f"Backend available: {'Yes' if result.backend_available else 'No'}", - f"Can decompress: {'Yes' if result.can_decompress else 'No'}", - ] - - if result.estimated_decomp_s: - lines.append( - f"Est. decomp time: {format_time(seconds=result.estimated_decomp_s)}" - ) - lines.append("") - - print("\n".join(lines)) - - if not result.can_decompress and result.reason: - app.echo(message=app.style(text=f"\n⚠ {result.reason}", fg="yellow")) + file_info(inspect_file(path=file), as_json=output_json) except KeyboardInterrupt: cancelled("Inspection") diff --git a/src/compresso/meson.build b/src/compresso/meson.build index 7ed94f8..b197aea 100644 --- a/src/compresso/meson.build +++ b/src/compresso/meson.build @@ -52,7 +52,6 @@ py.install_sources( 'cli/__init__.py', 'cli/__main__.py', 'cli/_app.py', - 'cli/_render.py', 'cli/algos.py', 'cli/benchmark.py', 'cli/compress.py', @@ -61,6 +60,13 @@ py.install_sources( subdir: 'compresso/cli', ) +py.install_sources( + 'cli/_render/__init__.py', + 'cli/_render/info.py', + 'cli/_render/jobs.py', + subdir: 'compresso/cli/_render', +) + py.install_sources( 'frontend/__init__.py', 'frontend/_formats.py', diff --git a/tests/python/cli/test_extract.py b/tests/python/cli/test_extract.py index cd75403..d4b5352 100644 --- a/tests/python/cli/test_extract.py +++ b/tests/python/cli/test_extract.py @@ -16,7 +16,7 @@ EXIT_USAGE, format_size, ) -from compresso.cli.extract import list_entries +from compresso.cli._render.info import list_entries from compresso.frontend.archive_api import ( ArchiveEntry, ) From 8d66d5aff714dd008ef8f448d93826944bebfe82 Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Fri, 23 Oct 2026 01:35:47 +0100 Subject: [PATCH 06/31] feat(cli): carry on past a failed input instead of stopping - New cli/_run.py holds run_job(), which times one job under its bar and raises 'InputFailed' - run_all() runs each in turn, reports a failure with the input's name and continues, returning the exit code - report_error() splits out of fail(), so a failure can be printed without exiting - Drops exit_for_result(), as its cancel-and-fail logic is now included in run_job() - extract() now stats its inputs up front, so a path that isn't a file is still a usage error with nothing run - An unrecognised format or a truncated stream now fails that input alone - Drops the timing, bar and exit handling from the four command bodies - compress() now runs through the same path with its single job --- src/compresso/cli/_render/__init__.py | 30 ++++----- src/compresso/cli/_run.py | 95 +++++++++++++++++++++++++++ src/compresso/cli/compress.py | 54 ++++++--------- src/compresso/cli/extract.py | 72 +++++++++----------- src/compresso/meson.build | 1 + tests/python/cli/conftest.py | 7 +- tests/python/cli/test_app.py | 6 +- tests/python/cli/test_extract.py | 44 +++++++++++++ 8 files changed, 207 insertions(+), 102 deletions(-) create mode 100644 src/compresso/cli/_run.py diff --git a/src/compresso/cli/_render/__init__.py b/src/compresso/cli/_render/__init__.py index ba33141..85dcd95 100644 --- a/src/compresso/cli/_render/__init__.py +++ b/src/compresso/cli/_render/__init__.py @@ -11,7 +11,7 @@ from contextlib import contextmanager from typing import NoReturn -from ...frontend._job import JobResult, ProgressCallback +from ...frontend._job import ProgressCallback from .._app import app # Exit codes @@ -64,6 +64,15 @@ def format_time(seconds: float) -> str: return f"{mins}m {secs:.1f}s" +def report_error(message: str) -> None: + """Print an error to stderr. + + Args: + message: Text to show, without the leading marker. + """ + app.echo(message=app.style(text=f"✗ {message}", fg="red"), err=True) + + def fail(message: str, code: int = EXIT_FAILED) -> NoReturn: """Print an error to stderr and exit. @@ -71,7 +80,7 @@ def fail(message: str, code: int = EXIT_FAILED) -> NoReturn: message: Text to show, without the leading marker. code: Process exit code; see the EXIT_* constants. """ - app.echo(message=app.style(text=f"✗ {message}", fg="red"), err=True) + report_error(message) sys.exit(code) @@ -94,23 +103,6 @@ def cancelled(what: str) -> NoReturn: sys.exit(EXIT_CANCELLED) -def exit_for_result(result: JobResult, what: str) -> None: - """Exit with the right code if `result` did not succeed. - - A cancelled job is not a failure, so it gets the conventional SIGINT - code rather than the generic failure one. - - Args: - result: The outcome of a frontend job. - what: Operation name for the message, e.g. "Compression". - """ - if result.cancelled: - cancelled(what) - - if not result.ok: - fail(f"\n{what} failed: {result.error}") - - @contextmanager def progress_bar( label: str, total: int, *, enabled: bool diff --git a/src/compresso/cli/_run.py b/src/compresso/cli/_run.py new file mode 100644 index 0000000..1246ad1 --- /dev/null +++ b/src/compresso/cli/_run.py @@ -0,0 +1,95 @@ +"""Running a command's jobs one input at a time, carrying on past a failure.""" + +from __future__ import annotations + +import time +from collections.abc import Callable, Iterable +from pathlib import Path +from typing import Any, TypeVar + +from .._core import Error +from ..frontend._job import ThreadedJob +from ._render import ( + BAR_THRESHOLD, + EXIT_FAILED, + EXIT_OK, + cancelled, + progress_bar, + report_error, +) + +J = TypeVar("J") + + +class InputFailed(Exception): + """One input failed; the runner reports it and goes on to the next.""" + + +def run_job( + job: ThreadedJob[Any], + source: Path, + label: str, + total: int, + *, + quiet: bool, + what: str, +) -> float: + """Run `job` under a progress bar. + + Args: + job: The planned job, already checked as runnable. + source: The input, named in the error if the job fails. + label: Text shown beside the bar. + total: Expected byte count; small jobs get no bar. + quiet: Whether to suppress the bar. + what: Operation name for messages, e.g. "Compression". + + Returns: + Seconds the job took. + + Raises: + InputFailed: If the job failed. + """ + start_time: float = time.time() + + with progress_bar( + label, total, enabled=not quiet and total > BAR_THRESHOLD + ) as on_progress: + result = job.run(progress=on_progress) + + elapsed: float = time.time() - start_time + + if result.cancelled: + cancelled(what) + + if not result.ok: + raise InputFailed(f"{source}: {what} failed: {result.error}") + + return elapsed + + +def run_all(jobs: Iterable[J], run_one: Callable[[J], None], what: str) -> int: + """Run each job in turn, reporting a failed one and going on to the next. + + Args: + jobs: The planned jobs, one per input. + run_one: Runs one job, raising `InputFailed` or `Error` if it fails. + what: Operation name for messages, e.g. "Compression". + + Returns: + EXIT_FAILED if any job failed, else EXIT_OK. + """ + failed = False + for job in jobs: + try: + run_one(job) + + except InputFailed as e: + report_error(str(e)) + failed = True + + except Error as e: + report_error(f"{what} error: {e}") + failed = True + + return EXIT_FAILED if failed else EXIT_OK diff --git a/src/compresso/cli/compress.py b/src/compresso/cli/compress.py index a188e0a..f7965db 100644 --- a/src/compresso/cli/compress.py +++ b/src/compresso/cli/compress.py @@ -2,7 +2,7 @@ from __future__ import annotations -import time +import sys from pathlib import Path from typing import Annotated @@ -11,15 +11,9 @@ from ..frontend.archive_api import ArchiveJob, OverwriteMode from ..frontend.dispatch import CompressRequest, plan_compress from ._app import app -from ._render import ( - BAR_THRESHOLD, - EXIT_USAGE, - cancelled, - exit_for_result, - fail, - progress_bar, -) +from ._render import EXIT_USAGE, cancelled, fail from ._render.jobs import archive_plan, archive_result, compress_plan, compress_result +from ._run import run_all, run_job def _compress_one_file(job: CompressionJob, quiet: bool) -> None: @@ -37,18 +31,9 @@ def _compress_one_file(job: CompressionJob, quiet: bool) -> None: if not quiet: compress_plan(plan) - start_time: float = time.time() - - with progress_bar( - "Compressing", - plan.input_size, - enabled=not quiet and plan.input_size > BAR_THRESHOLD, - ) as on_progress: - result = job.run(progress=on_progress) - - elapsed: float = time.time() - start_time - - exit_for_result(result, "Compression") + elapsed: float = run_job( + job, plan.src, "Compressing", plan.input_size, quiet=quiet, what="Compression" + ) # RENAME may have written somewhere other than `plan.dest`; job.plan is # replaced on success with the path actually written @@ -71,20 +56,16 @@ def _archive_many(job: ArchiveJob, quiet: bool) -> None: if not quiet: archive_plan(plan) - start_time: float = time.time() - # A format with a codec stage reads the data twice, so the job reports a # larger total than this once it starts; the bar follows it - with progress_bar( + elapsed: float = run_job( + job, + plan.output, "Archiving", plan.total_input_size, - enabled=not quiet and plan.total_input_size > BAR_THRESHOLD, - ) as on_progress: - result = job.run(progress=on_progress) - - elapsed: float = time.time() - start_time - - exit_for_result(result, "Archive") + quiet=quiet, + what="Archive", + ) # RENAME may have written somewhere other than `plan.output`; job.plan is # replaced on success with the path actually written @@ -231,10 +212,13 @@ def compress( except ValueError as e: fail(f"Error: {e}", EXIT_USAGE) - if isinstance(job, ArchiveJob): - _archive_many(job, quiet) - else: - _compress_one_file(job, quiet) + def run_one(job: CompressionJob | ArchiveJob) -> None: + if isinstance(job, ArchiveJob): + _archive_many(job, quiet) + else: + _compress_one_file(job, quiet) + + sys.exit(run_all([job], run_one, "Compression")) except KeyboardInterrupt: cancelled("Compression") diff --git a/src/compresso/cli/extract.py b/src/compresso/cli/extract.py index 3f93d4e..0a7175a 100644 --- a/src/compresso/cli/extract.py +++ b/src/compresso/cli/extract.py @@ -2,7 +2,7 @@ from __future__ import annotations -import time +import sys from pathlib import Path from typing import Annotated @@ -16,14 +16,7 @@ ) from ..frontend.dispatch import ExtractRequest, plan_extract from ._app import app -from ._render import ( - BAR_THRESHOLD, - EXIT_USAGE, - cancelled, - exit_for_result, - fail, - progress_bar, -) +from ._render import EXIT_USAGE, cancelled, fail from ._render.info import list_entries from ._render.jobs import ( decompress_plan, @@ -31,6 +24,7 @@ extract_plan, extract_result, ) +from ._run import InputFailed, run_all, run_job def _extract_archive(job: ExtractJob, list_only: bool, quiet: bool) -> None: @@ -44,7 +38,7 @@ def _extract_archive(job: ExtractJob, list_only: bool, quiet: bool) -> None: plan = job.plan if not plan.can_run: - fail(f"Error: {plan.reason_if_unavailable}", EXIT_USAGE) + raise InputFailed(f"{plan.archive}: {plan.reason_if_unavailable}") if list_only: list_entries(job.list_contents()) @@ -53,22 +47,16 @@ def _extract_archive(job: ExtractJob, list_only: bool, quiet: bool) -> None: if not quiet: extract_plan(plan) - start_time: float = time.time() - # Progress covers the bytes written out; a compressed archive also reports # the pass that decompresses it, so the job's own total wins - extracted_size: int = sum(entry.size for entry in plan.entries) - - with progress_bar( + elapsed: float = run_job( + job, + plan.archive, "Extracting", - extracted_size, - enabled=not quiet and extracted_size > BAR_THRESHOLD, - ) as on_progress: - result = job.run(progress=on_progress) - - elapsed: float = time.time() - start_time - - exit_for_result(result, "Extraction") + sum(entry.size for entry in plan.entries), + quiet=quiet, + what="Extraction", + ) if not quiet: extract_result(plan, elapsed) @@ -91,27 +79,23 @@ def _decompress_one_file(job: DecompressionJob, list_only: bool, quiet: bool) -> ) if not plan.can_run: - fail(f"Error: {plan.reason_if_unavailable}", EXIT_USAGE) + raise InputFailed(f"{plan.src}: {plan.reason_if_unavailable}") if not quiet: decompress_plan(plan) - start_time: float = time.time() - # Progress counts the compressed bytes read, so the source's size is the # denominator rather than the size it expands to compressed_size: int = plan.src.stat().st_size - with progress_bar( + elapsed: float = run_job( + job, + plan.src, "Decompressing", compressed_size, - enabled=not quiet and compressed_size > BAR_THRESHOLD, - ) as on_progress: - result = job.run(progress=on_progress) - - elapsed: float = time.time() - start_time - - exit_for_result(result, "Decompression") + quiet=quiet, + what="Decompression", + ) # RENAME may have written somewhere other than `plan.dest`; job.plan is # replaced on success with the path actually written @@ -234,6 +218,12 @@ def extract( if not inputs: fail("Error: No input files given", EXIT_USAGE) + # A stat shows these, so nothing runs; what only an input's bytes show + # fails that input alone + for path in inputs: + if not path.is_file(): + fail(f"Error: {path} does not exist or is not a file", EXIT_USAGE) + if overwrite: mode = OverwriteMode.OVERWRITE elif skip_existing: @@ -255,13 +245,15 @@ def extract( inputs=inputs, output=output, options=options, restore_name=name ) - try: - for job in plan_extract(request): - if isinstance(job, ExtractJob): - _extract_archive(job, list_only, quiet) + def run_one(job: ExtractJob | DecompressionJob) -> None: + if isinstance(job, ExtractJob): + _extract_archive(job, list_only, quiet) + + else: + _decompress_one_file(job, list_only, quiet) - else: - _decompress_one_file(job, list_only, quiet) + try: + sys.exit(run_all(plan_extract(request), run_one, "Extraction")) except KeyboardInterrupt: cancelled("Extraction") diff --git a/src/compresso/meson.build b/src/compresso/meson.build index b197aea..92c7e16 100644 --- a/src/compresso/meson.build +++ b/src/compresso/meson.build @@ -52,6 +52,7 @@ py.install_sources( 'cli/__init__.py', 'cli/__main__.py', 'cli/_app.py', + 'cli/_run.py', 'cli/algos.py', 'cli/benchmark.py', 'cli/compress.py', diff --git a/tests/python/cli/conftest.py b/tests/python/cli/conftest.py index d600291..30e931d 100644 --- a/tests/python/cli/conftest.py +++ b/tests/python/cli/conftest.py @@ -11,8 +11,7 @@ import pytest from typer.testing import CliRunner, Result -import compresso.cli.compress as compress_cmd -import compresso.cli.extract as extract_cmd +import compresso.cli._run as run_mod from compresso.cli import app from compresso.cli._render import ( BAR_THRESHOLD, @@ -145,9 +144,7 @@ def counting(fraction: float, done: int, reported_total: int) -> None: yield counting - # Each command imports progress_bar by name, so each binding is patched - monkeypatch.setattr(compress_cmd, "progress_bar", spy) - monkeypatch.setattr(extract_cmd, "progress_bar", spy) + monkeypatch.setattr(run_mod, "progress_bar", spy) return opened diff --git a/tests/python/cli/test_app.py b/tests/python/cli/test_app.py index 5206d77..93a454f 100644 --- a/tests/python/cli/test_app.py +++ b/tests/python/cli/test_app.py @@ -194,12 +194,12 @@ class TestExitCodes: [ ("compress", "nope.bin"), ("inspect", "payload.bin"), - ("extract", "payload.bin"), + ("extract", "nope.gz"), ], ) def test_bad_input_is_usage(self, payload: Path, command: str, name: str) -> None: - """Test that a missing or unrecognised input is wrong, rather than the - operation having failed.""" + """Test that a missing input, or one inspect can't read, is wrong, rather + than the operation having failed.""" invoke(command, payload.parent / name, expect=EXIT_USAGE) @pytest.mark.parametrize("command", ["compress", "extract"]) diff --git a/tests/python/cli/test_extract.py b/tests/python/cli/test_extract.py index d4b5352..f829e87 100644 --- a/tests/python/cli/test_extract.py +++ b/tests/python/cli/test_extract.py @@ -215,6 +215,50 @@ def test_several_inputs_in_one_go(self, payload: Path, tmp_path: Path) -> None: assert (tmp_path / "a").read_bytes() == payload.read_bytes() assert (tmp_path / "b").read_bytes() == payload.read_bytes() + # Unrecognised fails while planning, truncated while running; random bytes + # named .gz would be tried as gzip, so the unrecognised one has no codec suffix + @pytest.mark.parametrize("bad", ["unrecognised", "truncated"]) + def test_a_failed_input_does_not_stop_the_rest( + self, + payload: Path, + source_tree: Path, + tmp_path: Path, + make_archive: Callable[[str], Path], + bad: str, + ) -> None: + """Test that the inputs either side of a bad one are still extracted, and + the bad one is named and exits 1.""" + good = tmp_path / "good.gz" + good.write_bytes(gzip.compress(payload.read_bytes())) + + if bad == "unrecognised": + broken = tmp_path / "bad.bin" + broken.write_bytes(payload.read_bytes()) + else: + broken = tmp_path / "bad.gz" + broken.write_bytes(good.read_bytes()[: good.stat().st_size // 2]) + + dest = tmp_path / "dest" + dest.mkdir() + + result = invoke( + "extract", + good, + broken, + make_archive("zip"), + "-o", + dest, + "-q", + expect=EXIT_FAILED, + ) + + assert str(broken) in result.output + assert (dest / "good").read_bytes() == payload.read_bytes() + assert not (dest / "bad").exists() + for original in source_tree.iterdir(): + restored = dest / source_tree.name / original.name + assert restored.read_bytes() == original.read_bytes() + def test_listing_a_single_file_container_is_refused( self, payload: Path, tmp_path: Path ) -> None: From 196febdf2921dd9a7c548f4e42fff4a62162bbe2 Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Fri, 23 Oct 2026 13:18:30 +0100 Subject: [PATCH 07/31] feat(cli): rename 'list' command to 'formats', and show the formats too - 'formats' now prints backends, standalone containers, and archive formats - Includes algorithms with level ranges, standalone formats with aliases/extensions, and archive formats with compressed variants - The variants are dervied from the core rather than assumed, so a codec that isn't compiled doesn't get listed - Moves rendering to _render/formats.py alongside the other reports, with the heading and item layout shared by the three sections --- src/compresso/cli/__init__.py | 4 +- src/compresso/cli/_render/formats.py | 57 ++++++++++++++++++++++++++++ src/compresso/cli/algos.py | 37 ------------------ src/compresso/cli/formats.py | 43 +++++++++++++++++++++ src/compresso/meson.build | 3 +- tests/python/cli/test_app.py | 19 +++++----- tests/python/cli/test_formats.py | 54 ++++++++++++++++++++++++++ tests/python/cli/test_list.py | 37 ------------------ 8 files changed, 167 insertions(+), 87 deletions(-) create mode 100644 src/compresso/cli/_render/formats.py delete mode 100644 src/compresso/cli/algos.py create mode 100644 src/compresso/cli/formats.py create mode 100644 tests/python/cli/test_formats.py delete mode 100644 tests/python/cli/test_list.py diff --git a/src/compresso/cli/__init__.py b/src/compresso/cli/__init__.py index 84816e8..95b7601 100644 --- a/src/compresso/cli/__init__.py +++ b/src/compresso/cli/__init__.py @@ -2,7 +2,7 @@ from __future__ import annotations -from . import algos, benchmark, compress, extract, inspect +from . import benchmark, compress, extract, formats, inspect from ._app import app __all__ = ["app", "main"] @@ -13,7 +13,7 @@ extract, inspect, benchmark, - algos, + formats, ) for _module in _COMMAND_MODULES: diff --git a/src/compresso/cli/_render/formats.py b/src/compresso/cli/_render/formats.py new file mode 100644 index 0000000..ec70b16 --- /dev/null +++ b/src/compresso/cli/_render/formats.py @@ -0,0 +1,57 @@ +"""What `formats` prints: the algorithms and formats this build can write.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from ...introspect.capabilities import BackendCapabilities +from .._app import app + +if TYPE_CHECKING: + from ..._core import _StandaloneFormatDict + + +def _heading(text: str, count: int) -> None: + app.echo(message=f"{text}: {count}\n") + + +def _item(name: str, details: dict[str, str]) -> None: + app.echo(message=app.style(text=f"- {name}", fg="green", bold=True)) + width = max(map(len, details), default=0) + 2 + for label, value in details.items(): + app.echo(message=f" {label + ':':<{width}}{value}") + + app.echo() + + +def list_formats( + algos: list[BackendCapabilities], + standalone: list[_StandaloneFormatDict], + archives: dict[str, list[str]], +) -> None: + """Print the .comp algorithms, then the standalone and archive formats. + + Args: + algos: The backends a .comp container can use. + standalone: The single-file container formats. + archives: Each archive format, mapped to the compressed variants + written as `.`, e.g. `tar.zst`. + """ + _heading("Available compression algorithms", len(algos)) + for cap in algos: + levels = "none" if cap.min_level is None else f"{cap.min_level}-{cap.max_level}" + _item(cap.name, {"ID": str(cap.id), "Levels": levels}) + + _heading("Standalone formats", len(standalone)) + for fmt in standalone: + _item( + fmt["name"], + { + "Aliases": ", ".join(fmt["aliases"]) or "none", + "Extensions": ", ".join(f".{ext}" for ext in fmt["exts"]), + }, + ) + + _heading("Archive formats", len(archives)) + for name, compressed in archives.items(): + _item(name, {"Compressed": ", ".join(compressed)} if compressed else {}) diff --git a/src/compresso/cli/algos.py b/src/compresso/cli/algos.py deleted file mode 100644 index 68e81de..0000000 --- a/src/compresso/cli/algos.py +++ /dev/null @@ -1,37 +0,0 @@ -"""The `list` command, showing the available compression backends.""" - -from __future__ import annotations - -from ..introspect.capabilities import list_capabilities -from ._app import app -from ._render import cancelled, fail - - -def list_algos() -> None: - """List all available compression algorithms.""" - try: - caps = list_capabilities() - - if not caps: - fail("No compression backends available") - - print(f"Available compression algorithms: {len(caps)}\n") - - for cap in caps: - app.echo(message=app.style(text=f"- {cap.name}", fg="green", bold=True)) - levels: str = ( - "none" if cap.min_level is None else f"{cap.min_level}-{cap.max_level}" - ) - app.echo(message=app.style(text=f" ID: {cap.id}")) - app.echo(message=app.style(text=f" Levels: {levels}\n")) - - except KeyboardInterrupt: - cancelled("Listing") - - except Exception as e: - fail(f"Error listing algorithms: {e}") - - -def register() -> None: - """Attach this command to the app.""" - app.command(name="list", aliases=["l", "ls"])(list_algos) diff --git a/src/compresso/cli/formats.py b/src/compresso/cli/formats.py new file mode 100644 index 0000000..e4bd666 --- /dev/null +++ b/src/compresso/cli/formats.py @@ -0,0 +1,43 @@ +"""The `formats` command, showing the algorithms and formats available.""" + +from __future__ import annotations + +from .._core import archive_capabilities, format_is_archive, standalone_formats +from ..introspect.capabilities import list_capabilities +from ._app import app +from ._render import cancelled, fail +from ._render.formats import list_formats + + +def formats() -> None: + """List the available compression algorithms and file formats.""" + try: + caps = list_capabilities() + + if not caps: + fail("No compression backends available") + + standalone = standalone_formats() + # Asked of the core, which names a variant by any of the codec's + # names; listed by the extension an output is given + archives = { + archive["name"]: [ + f"{archive['name']}.{fmt['exts'][0]}" + for fmt in standalone + if format_is_archive(f"{archive['name']}.{fmt['exts'][0]}") + ] + for archive in archive_capabilities() + } + + list_formats(caps, standalone, archives) + + except KeyboardInterrupt: + cancelled("Listing") + + except Exception as e: + fail(f"Error listing formats: {e}") + + +def register() -> None: + """Attach this command to the app.""" + app.command(name="formats")(formats) diff --git a/src/compresso/meson.build b/src/compresso/meson.build index 92c7e16..6141ca1 100644 --- a/src/compresso/meson.build +++ b/src/compresso/meson.build @@ -53,16 +53,17 @@ py.install_sources( 'cli/__main__.py', 'cli/_app.py', 'cli/_run.py', - 'cli/algos.py', 'cli/benchmark.py', 'cli/compress.py', 'cli/extract.py', + 'cli/formats.py', 'cli/inspect.py', subdir: 'compresso/cli', ) py.install_sources( 'cli/_render/__init__.py', + 'cli/_render/formats.py', 'cli/_render/info.py', 'cli/_render/jobs.py', subdir: 'compresso/cli/_render', diff --git a/tests/python/cli/test_app.py b/tests/python/cli/test_app.py index 93a454f..1311e03 100644 --- a/tests/python/cli/test_app.py +++ b/tests/python/cli/test_app.py @@ -9,7 +9,7 @@ import pytest -import compresso.cli.algos as algos_cmd +import compresso.cli.formats as formats_cmd import compresso.cli.inspect as inspect_cmd from compresso.cli._render import ( EXIT_CANCELLED, @@ -71,12 +71,13 @@ class TestAppStructure: def test_commands_are_listed_in_workflow_order(self) -> None: """Test that the order is chosen in `__init__`, not inherited from import sorting. - Alphabetical imports would put `list` first. + Alphabetical imports would put `benchmark` first. """ output = invoke("--help").output - # Matched on the alias column, which is unique per row; a bare command - # name also appears inside other commands' descriptions + # Matched on the alias column, which is unique per row, or the + # description where there is none; a bare command name also appears + # inside other commands' descriptions positions = [ output.index(aliases) for aliases in [ @@ -84,7 +85,7 @@ def test_commands_are_listed_in_workflow_order(self) -> None: "(x, ex", "(i, info)", "(b, bench)", - "(l, ls)", + "List the available", ] ] assert positions == sorted(positions) @@ -106,8 +107,6 @@ def test_commands_are_listed_in_workflow_order(self) -> None: ("info", "inspect"), ("b", "benchmark"), ("bench", "benchmark"), - ("l", "list"), - ("ls", "list"), ], ) def test_aliases_still_resolve(self, alias: str, command: str) -> None: @@ -282,7 +281,7 @@ class TestInterruption: [ (["compress"], "Compression cancelled"), (["inspect"], "Inspection cancelled"), - (["list"], "Listing cancelled"), + (["formats"], "Listing cancelled"), ], ) def test_interrupt_is_reported( @@ -300,10 +299,10 @@ def interrupt(*args: object, **kwargs: object) -> None: monkeypatch.setattr(CompressionJob, "from_file", interrupt) monkeypatch.setattr(inspect_cmd, "inspect_file", interrupt) - monkeypatch.setattr(algos_cmd, "list_capabilities", interrupt) + monkeypatch.setattr(formats_cmd, "list_capabilities", interrupt) args: list[str | Path] = [*command] - if command[0] != "list": + if command[0] != "formats": args.append(payload) result = invoke(*args, expect=EXIT_CANCELLED) diff --git a/tests/python/cli/test_formats.py b/tests/python/cli/test_formats.py new file mode 100644 index 0000000..319b248 --- /dev/null +++ b/tests/python/cli/test_formats.py @@ -0,0 +1,54 @@ +"""Tests for `compresso formats`.""" + +from __future__ import annotations + +import subprocess +import sys + +from compresso.cli._render import ( + EXIT_OK, +) + +from .conftest import invoke + + +def section(output: str, name: str) -> str: + """Return one `- name` block of the output, up to the blank line after it.""" + return output[output.index(f"- {name}\n") :].split("\n\n")[0] + + +class TestFormats: + """Test the read-only `formats` command.""" + + def test_runs_as_a_module(self) -> None: + """Test that `python -m compresso.cli` works, which a package needs __main__ for.""" + result = subprocess.run( + [sys.executable, "-m", "compresso.cli", "formats"], + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == EXIT_OK + assert "zstd" in result.stdout + + def test_formats_shows_level_ranges(self) -> None: + """Test that formats shows each backend's levels, and none for snappy.""" + output = invoke("formats").output + + assert "Levels: 1-22" in section(output, "zstd") + assert "Levels: none" in section(output, "snappy") + + def test_formats_shows_standalone_and_archive_formats(self) -> None: + """Test that each standalone format shows its aliases and extensions, and + tar its compressed variants, named as an output would be.""" + output = invoke("formats").output + standalone = output[output.index("Standalone formats") :] + + assert "Aliases: zst" in section(standalone, "zstd") + assert "Extensions: .zst, .zstd" in section(standalone, "zstd") + assert "Aliases: none" in section(standalone, "lz4") + assert "tar.gz, tar.bz2, tar.xz, tar.zst, tar.lz4, tar.sz" in section( + output, "tar" + ) + assert section(output, "zip") == "- zip" diff --git a/tests/python/cli/test_list.py b/tests/python/cli/test_list.py deleted file mode 100644 index 2d7e671..0000000 --- a/tests/python/cli/test_list.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Tests for `compresso list`.""" - -from __future__ import annotations - -import subprocess -import sys - -from compresso.cli._render import ( - EXIT_OK, -) - -from .conftest import invoke - - -class TestList: - """Test the read-only `list` command.""" - - def test_runs_as_a_module(self) -> None: - """Test that `python -m compresso.cli` works, which a package needs __main__ for.""" - result = subprocess.run( - [sys.executable, "-m", "compresso.cli", "list"], - capture_output=True, - text=True, - check=False, - ) - - assert result.returncode == EXIT_OK - assert "zstd" in result.stdout - - def test_list_shows_level_ranges(self) -> None: - """Test that list shows each backend's levels, and none for snappy.""" - output = invoke("list").output - - zstd = output[output.index("- zstd") :].split("\n\n")[0] - snappy = output[output.index("- snappy") :].split("\n\n")[0] - assert "Levels: 1-22" in zstd - assert "Levels: none" in snappy From 856546ae99d29720e0dc116c7e8095de0c67b6ea Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Fri, 23 Oct 2026 17:03:53 +0100 Subject: [PATCH 08/31] perf(cli): defer the asyncio and tabulate imports - 'tabulate' now imports when needed, instead of for every command - 'asyncio' is now only imported when run_async() is called, rather than for sync commands as well --- src/compresso/frontend/_job.py | 4 +++- src/compresso/introspect/benchmark.py | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/compresso/frontend/_job.py b/src/compresso/frontend/_job.py index 847c585..ea725a6 100644 --- a/src/compresso/frontend/_job.py +++ b/src/compresso/frontend/_job.py @@ -2,7 +2,6 @@ from __future__ import annotations -import asyncio import threading from collections.abc import Callable from dataclasses import dataclass @@ -229,6 +228,9 @@ async def run_async( Returns: JobResult: The outcome of the job. """ + # Deferred so sync callers skip asyncio's import cost + import asyncio + token: CancelToken = cancel if cancel is not None else CancelToken() loop = asyncio.get_running_loop() future = loop.run_in_executor(None, self.run, progress, token) diff --git a/src/compresso/introspect/benchmark.py b/src/compresso/introspect/benchmark.py index 2af603a..69430d3 100644 --- a/src/compresso/introspect/benchmark.py +++ b/src/compresso/introspect/benchmark.py @@ -8,8 +8,6 @@ from dataclasses import dataclass from pathlib import Path -from tabulate import tabulate - from .._core import compress_file, decompress_file from .._levels import to_core_level from .capabilities import get_by_name @@ -312,4 +310,8 @@ def print_results(results: list[BenchmarkResult]) -> None: ] table_data.append(row) + # Deferred: tabulate pulls in wcwidth, which every CLI command would + # otherwise pay for via the introspect package import + from tabulate import tabulate + print(tabulate(tabular_data=table_data, headers=headers, tablefmt="grid")) From 1a700150634c5bc8329c0466299c8e16f2c1a7a9 Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Fri, 23 Oct 2026 23:14:10 +0100 Subject: [PATCH 09/31] feat(cli): add a 'list' command for an archive's entries - Merges 'extract --list' and 'inspect --entries' into a single 'list' command - 'list' takes several archives, prints each under a '==> path <==' header as head does, and goes through run_all() - A single-file container or an unreadable input fails alone and the rest still list - New '--long' adds each entry's mode, mtime and, where the format tracks them, its compressed size and CRC - '--json' is one array item per archive, carrying the entries in full - Drops the '--list' and '--entries' flags from 'extract' and 'inspect' - Updates 'formats' interrupted message --- src/compresso/cli/__init__.py | 2 + src/compresso/cli/_render/__init__.py | 14 ++++ src/compresso/cli/_render/formats.py | 16 +++- src/compresso/cli/_render/info.py | 44 ++++++----- src/compresso/cli/extract.py | 25 +------ src/compresso/cli/formats.py | 2 +- src/compresso/cli/inspect.py | 29 ++------ src/compresso/cli/list.py | 103 ++++++++++++++++++++++++++ src/compresso/meson.build | 1 + tests/python/cli/test_app.py | 8 +- tests/python/cli/test_extract.py | 64 +--------------- tests/python/cli/test_inspect.py | 16 +--- tests/python/cli/test_list.py | 95 ++++++++++++++++++++++++ 13 files changed, 274 insertions(+), 145 deletions(-) create mode 100644 src/compresso/cli/list.py create mode 100644 tests/python/cli/test_list.py diff --git a/src/compresso/cli/__init__.py b/src/compresso/cli/__init__.py index 95b7601..c4501ef 100644 --- a/src/compresso/cli/__init__.py +++ b/src/compresso/cli/__init__.py @@ -3,6 +3,7 @@ from __future__ import annotations from . import benchmark, compress, extract, formats, inspect +from . import list as list_ from ._app import app __all__ = ["app", "main"] @@ -11,6 +12,7 @@ _COMMAND_MODULES = ( compress, extract, + list_, inspect, benchmark, formats, diff --git a/src/compresso/cli/_render/__init__.py b/src/compresso/cli/_render/__init__.py index 85dcd95..fd4a5aa 100644 --- a/src/compresso/cli/_render/__init__.py +++ b/src/compresso/cli/_render/__init__.py @@ -9,6 +9,7 @@ import sys from collections.abc import Generator from contextlib import contextmanager +from pathlib import Path from typing import NoReturn from ...frontend._job import ProgressCallback @@ -93,6 +94,19 @@ def succeed(message: str) -> None: app.echo(message=app.style(text=f"✓ {message}", fg="green")) +def input_header(path: Path, *, first: bool) -> None: + """Print the `==> name <==` line that head puts before each of several inputs. + + Args: + path: The input, as given. + first: False puts a blank line above, to separate it from the last. + """ + if not first: + print() + + print(f"==> {path} <==") + + def cancelled(what: str) -> NoReturn: """Report an interrupted operation and exit with the SIGINT code. diff --git a/src/compresso/cli/_render/formats.py b/src/compresso/cli/_render/formats.py index ec70b16..432c94d 100644 --- a/src/compresso/cli/_render/formats.py +++ b/src/compresso/cli/_render/formats.py @@ -12,10 +12,24 @@ def _heading(text: str, count: int) -> None: + """Print a section title with its item count, then a blank line. + + Args: + text: The section title, e.g. `Native algorithms`. + count: How many items the section lists. + """ app.echo(message=f"{text}: {count}\n") def _item(name: str, details: dict[str, str]) -> None: + """Print a bulleted name, then its details as indented `label: value` lines. + + The values line up in one column, and empty `details` prints only the name. + + Args: + name: The algorithm or format name, shown in bold green. + details: Labels mapped to their values, printed in order. + """ app.echo(message=app.style(text=f"- {name}", fg="green", bold=True)) width = max(map(len, details), default=0) + 2 for label, value in details.items(): @@ -37,7 +51,7 @@ def list_formats( archives: Each archive format, mapped to the compressed variants written as `.`, e.g. `tar.zst`. """ - _heading("Available compression algorithms", len(algos)) + _heading("Native algorithms", len(algos)) for cap in algos: levels = "none" if cap.min_level is None else f"{cap.min_level}-{cap.max_level}" _item(cap.name, {"ID": str(cap.id), "Levels": levels}) diff --git a/src/compresso/cli/_render/info.py b/src/compresso/cli/_render/info.py index 46689c5..8fafbf3 100644 --- a/src/compresso/cli/_render/info.py +++ b/src/compresso/cli/_render/info.py @@ -14,7 +14,7 @@ from . import EXIT_USAGE, fail, format_size, format_time, succeed -def list_entries(entries: list[ArchiveEntry]) -> None: +def list_entries(entries: list[ArchiveEntry], *, long: bool = False) -> None: """Print one line per entry: its size, then its path. Directories and symlinks leave the size blank, a symlink is followed by @@ -23,7 +23,14 @@ def list_entries(entries: list[ArchiveEntry]) -> None: Args: entries: The archive's entries, in archive order. + long: If True, print each entry's full detail line instead, unindented. """ + if long: + for entry in entries: + print(_entry_detail_line(entry)) + + return + listed_dirs: set[str] = set() for entry in entries: @@ -85,13 +92,25 @@ def _entry_detail_line(entry: ArchiveEntry) -> str: return f"{mode} {mtime} {size:>10} {entry.path}{target}{detail}" +def listings_json(listings: list[tuple[Path, str, list[ArchiveEntry]]]) -> None: + """Print archives' entries as a JSON array of `{path, format, entries}`. + + Args: + listings: Each archive's path, detected format and entries. + """ + data = [ + {"path": str(path), "format": fmt, "entries": entries} + for path, fmt, entries in listings + ] + print(orjson.dumps(data, option=orjson.OPT_INDENT_2).decode()) + + def archive_info( file: Path, fmt: str | None, entries: list[ArchiveEntry], *, as_json: bool, - show_entries: bool, ) -> None: """Print an archive's metadata, as text or JSON. @@ -100,7 +119,6 @@ def archive_info( fmt: The archive's detected format. entries: The archive's entries. as_json: If True, print JSON instead of text. - show_entries: If True, also print each entry's own detail. """ total_size = sum(entry.size for entry in entries) archive_size: int = file.stat().st_size @@ -114,23 +132,6 @@ def archive_info( "total_size": total_size, "archive_size": archive_size, } - if show_entries: - data["entries"] = [ - { - "path": entry.path, - "size": entry.size, - "is_dir": entry.is_dir, - "is_symlink": entry.is_symlink, - "is_hardlink": entry.is_hardlink, - "mtime": entry.mtime, - "mode": entry.mode, - "link_target": entry.link_target, - "compressed_size": entry.compressed_size, - "crc": entry.crc, - "method": entry.method, - } - for entry in entries - ] print(orjson.dumps(data, option=orjson.OPT_INDENT_2).decode()) return @@ -148,9 +149,6 @@ def archive_info( print("\n".join(lines)) - if show_entries: - print("\n" + "\n".join(_entry_detail_line(entry) for entry in entries)) - def file_info(result: InspectResult, *, as_json: bool) -> None: """Print a single-file container's metadata, as text or JSON. diff --git a/src/compresso/cli/extract.py b/src/compresso/cli/extract.py index 0a7175a..c1993bf 100644 --- a/src/compresso/cli/extract.py +++ b/src/compresso/cli/extract.py @@ -17,7 +17,6 @@ from ..frontend.dispatch import ExtractRequest, plan_extract from ._app import app from ._render import EXIT_USAGE, cancelled, fail -from ._render.info import list_entries from ._render.jobs import ( decompress_plan, decompress_result, @@ -27,12 +26,11 @@ from ._run import InputFailed, run_all, run_job -def _extract_archive(job: ExtractJob, list_only: bool, quiet: bool) -> None: +def _extract_archive(job: ExtractJob, quiet: bool) -> None: """Unpack one archive. Args: job: The planned job. - list_only: Whether to list entries only. quiet: Whether to suppress output. """ plan = job.plan @@ -40,10 +38,6 @@ def _extract_archive(job: ExtractJob, list_only: bool, quiet: bool) -> None: if not plan.can_run: raise InputFailed(f"{plan.archive}: {plan.reason_if_unavailable}") - if list_only: - list_entries(job.list_contents()) - return - if not quiet: extract_plan(plan) @@ -62,22 +56,15 @@ def _extract_archive(job: ExtractJob, list_only: bool, quiet: bool) -> None: extract_result(plan, elapsed) -def _decompress_one_file(job: DecompressionJob, list_only: bool, quiet: bool) -> None: +def _decompress_one_file(job: DecompressionJob, quiet: bool) -> None: """Unpack one single-file container: a `.comp` or a standalone codec. Args: job: The planned job. - list_only: Whether to list entries only. quiet: Whether to suppress output. """ plan = job.plan - if list_only: - fail( - f"Error: {plan.src.name} holds a single file, so there is nothing to list", - EXIT_USAGE, - ) - if not plan.can_run: raise InputFailed(f"{plan.src}: {plan.reason_if_unavailable}") @@ -114,9 +101,6 @@ def extract( help="Where to write (default: beside the input)", ), ] = None, - list_only: Annotated[ - bool, app.Option("--list", help="List archive contents without extracting") - ] = False, overwrite: Annotated[ bool, app.Option("--overwrite", help="Replace files that already exist") ] = False, @@ -194,7 +178,6 @@ def extract( output: Where to write (default: beside the input); a directory for archives, and for one file either a directory to put it in or the path to write. - list_only: If True, list archive contents without extracting. overwrite: If True, replace files that already exist. skip_existing: If True, leave files that already exist untouched. error_on_conflict: If True, fail instead of renaming a clashing file. @@ -247,10 +230,10 @@ def extract( def run_one(job: ExtractJob | DecompressionJob) -> None: if isinstance(job, ExtractJob): - _extract_archive(job, list_only, quiet) + _extract_archive(job, quiet) else: - _decompress_one_file(job, list_only, quiet) + _decompress_one_file(job, quiet) try: sys.exit(run_all(plan_extract(request), run_one, "Extraction")) diff --git a/src/compresso/cli/formats.py b/src/compresso/cli/formats.py index e4bd666..0d9dd17 100644 --- a/src/compresso/cli/formats.py +++ b/src/compresso/cli/formats.py @@ -32,7 +32,7 @@ def formats() -> None: list_formats(caps, standalone, archives) except KeyboardInterrupt: - cancelled("Listing") + cancelled("Format listing") except Exception as e: fail(f"Error listing formats: {e}") diff --git a/src/compresso/cli/inspect.py b/src/compresso/cli/inspect.py index 835354c..0651399 100644 --- a/src/compresso/cli/inspect.py +++ b/src/compresso/cli/inspect.py @@ -14,26 +14,19 @@ from ._render.info import archive_info, file_info -def _inspect_archive(file: Path, output_json: bool, show_entries: bool) -> None: +def _inspect_archive(file: Path, output_json: bool) -> None: """Inspect an archive: a container that can hold many entries. Args: file: The path to the archive. output_json: If True, output metadata in JSON format. - show_entries: If True, also report each entry's own detail. """ plan = plan_archive_extraction(archive=file) if not plan.can_run: fail(f"Error: {plan.reason_if_unavailable}", EXIT_USAGE) - archive_info( - file, - detect_format(file), - plan.entries, - as_json=output_json, - show_entries=show_entries, - ) + archive_info(file, detect_format(file), plan.entries, as_json=output_json) def inspect( @@ -41,31 +34,21 @@ def inspect( output_json: Annotated[ bool, app.Option("--json", help="Output in JSON format") ] = False, - show_entries: Annotated[ - bool, - app.Option( - "--entries", "-e", help="For an archive, also list each entry's detail" - ), - ] = False, ) -> None: """Inspect a compressed file or archive and show metadata. \f - An archive reports its format, entry count, and total size; pass - `--entries` to also list each entry's own mtime, permissions, and (for a - format like zip that tracks it) per-entry compressed size and CRC. A - single-file container, a Compresso `.comp` or a standalone codec, reports - its algorithm and decompression info as before. + An archive reports its format, entry count, and total size; `list` shows + its entries. A single-file container, a Compresso `.comp` or a standalone + codec, reports its algorithm and decompression info as before. Args: file: The path to the compressed file or archive. output_json: If True, output metadata in JSON format. - show_entries: If True and the file is an archive, also list each - entry's own detail. """ try: if looks_like_archive(file): - _inspect_archive(file, output_json, show_entries) + _inspect_archive(file, output_json) return file_info(inspect_file(path=file), as_json=output_json) diff --git a/src/compresso/cli/list.py b/src/compresso/cli/list.py new file mode 100644 index 0000000..17e2900 --- /dev/null +++ b/src/compresso/cli/list.py @@ -0,0 +1,103 @@ +"""The `list` command, showing an archive's entries.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Annotated + +from .._core import detect_format +from ..frontend.archive_api import ArchiveEntry +from ..frontend.archive_api import plan_extraction as plan_archive_extraction +from ..frontend.dispatch import is_archive_format +from ._app import app +from ._render import EXIT_USAGE, cancelled, fail, input_header +from ._render.info import list_entries, listings_json +from ._run import InputFailed, run_all + + +def _read_entries(path: Path) -> tuple[str, list[ArchiveEntry]]: + """Read one archive's format and entries. + + Args: + path: The archive. + + Returns: + The detected format and the entries, in archive order. + + Raises: + InputFailed: If the input is not an archive or can't be read. + """ + fmt = detect_format(path) + + if not is_archive_format(fmt): + if fmt == "unknown": + raise InputFailed(f"{path}: not a recognised archive") + + raise InputFailed(f"{path}: holds a single file, so there is nothing to list") + + plan = plan_archive_extraction(archive=path) + + if not plan.can_run: + raise InputFailed(f"{path}: {plan.reason_if_unavailable}") + + return fmt, plan.entries + + +def list_archives( + inputs: Annotated[list[Path], app.Argument(help="Archives to list")], + long: Annotated[ + bool, + app.Option("--long", "-l", help="Show each entry's mode, mtime and detail"), + ] = False, + output_json: Annotated[ + bool, app.Option("--json", help="Output a JSON array, one item per archive") + ] = False, +) -> None: + """List the entries of archives. + \f + + Each entry is shown with its size and path, indented under the directories + listed above it; `--long` adds its mode and mtime, and (for a format like + zip that tracks it) its compressed size and CRC. + + Args: + inputs: The archives to list. + long: If True, show each entry's full detail. + output_json: If True, print a JSON array of `{path, format, entries}`. + """ + for path in inputs: + if not path.is_file(): + fail(f"Error: {path} does not exist or is not a file", EXIT_USAGE) + + listings: list[tuple[Path, str, list[ArchiveEntry]]] = [] + + def run_one(path: Path) -> None: + fmt, entries = _read_entries(path) + + if not output_json: + if len(inputs) > 1: + input_header(path, first=not listings) + + list_entries(entries, long=long) + + listings.append((path, fmt, entries)) + + try: + code = run_all(inputs, run_one, "Listing") + + if output_json: + listings_json(listings) + + sys.exit(code) + + except KeyboardInterrupt: + cancelled("Listing") + + except Exception as e: + fail(f"Unexpected error: {e}") + + +def register() -> None: + """Attach this command to the app.""" + app.command(name="list", aliases=["l"])(list_archives) diff --git a/src/compresso/meson.build b/src/compresso/meson.build index 6141ca1..784cf96 100644 --- a/src/compresso/meson.build +++ b/src/compresso/meson.build @@ -58,6 +58,7 @@ py.install_sources( 'cli/extract.py', 'cli/formats.py', 'cli/inspect.py', + 'cli/list.py', subdir: 'compresso/cli', ) diff --git a/tests/python/cli/test_app.py b/tests/python/cli/test_app.py index 1311e03..fd4e399 100644 --- a/tests/python/cli/test_app.py +++ b/tests/python/cli/test_app.py @@ -11,6 +11,7 @@ import compresso.cli.formats as formats_cmd import compresso.cli.inspect as inspect_cmd +import compresso.cli.list as list_cmd from compresso.cli._render import ( EXIT_CANCELLED, EXIT_FAILED, @@ -83,6 +84,7 @@ def test_commands_are_listed_in_workflow_order(self) -> None: for aliases in [ "(c, comp", "(x, ex", + "(l)", "(i, info)", "(b, bench)", "List the available", @@ -103,6 +105,7 @@ def test_commands_are_listed_in_workflow_order(self) -> None: ("decompress", "extract"), ("d", "extract"), ("decomp", "extract"), + ("l", "list"), ("i", "inspect"), ("info", "inspect"), ("b", "benchmark"), @@ -194,6 +197,7 @@ class TestExitCodes: ("compress", "nope.bin"), ("inspect", "payload.bin"), ("extract", "nope.gz"), + ("list", "nope.zip"), ], ) def test_bad_input_is_usage(self, payload: Path, command: str, name: str) -> None: @@ -281,7 +285,8 @@ class TestInterruption: [ (["compress"], "Compression cancelled"), (["inspect"], "Inspection cancelled"), - (["formats"], "Listing cancelled"), + (["list"], "Listing cancelled"), + (["formats"], "Format listing cancelled"), ], ) def test_interrupt_is_reported( @@ -299,6 +304,7 @@ def interrupt(*args: object, **kwargs: object) -> None: monkeypatch.setattr(CompressionJob, "from_file", interrupt) monkeypatch.setattr(inspect_cmd, "inspect_file", interrupt) + monkeypatch.setattr(list_cmd, "detect_format", interrupt) monkeypatch.setattr(formats_cmd, "list_capabilities", interrupt) args: list[str | Path] = [*command] diff --git a/tests/python/cli/test_extract.py b/tests/python/cli/test_extract.py index f829e87..19b128f 100644 --- a/tests/python/cli/test_extract.py +++ b/tests/python/cli/test_extract.py @@ -13,16 +13,10 @@ from compresso.cli._render import ( EXIT_FAILED, EXIT_OK, - EXIT_USAGE, - format_size, -) -from compresso.cli._render.info import list_entries -from compresso.frontend.archive_api import ( - ArchiveEntry, ) from ..helpers import file_entry, link_entry, make_tar, renamed, umask -from .conftest import CONFLICT_MODES, TREE_FILE_SIZE, invoke +from .conftest import CONFLICT_MODES, invoke class TestArchiveRoundTrip: @@ -46,51 +40,6 @@ def test_archive_then_extract( restored = dest / source_tree.name / original.name assert restored.read_bytes() == original.read_bytes() - def test_extract_list_only( - self, - tmp_path: Path, - make_archive: Callable[[str], Path], - ) -> None: - """Test that --list prints each entry's size and path without writing anything.""" - archive = make_archive("tar") - dest = tmp_path / "dest" - - result = invoke("extract", archive, "--list", "-o", dest) - - lines = result.output.splitlines() - f0_line = next(line for line in lines if "f0.bin" in line) - assert f0_line.split()[:2] == format_size(TREE_FILE_SIZE).split() - assert not dest.exists() - - # Directories have a blank size; their contents are indented beneath them - dir_line = next(line for line in lines if line.rstrip("/").endswith("tree")) - assert dir_line == f"{'':>10} {dir_line.strip()}" - assert f0_line.endswith(" tree/f0.bin") - - def test_extract_list_indents_by_depth( - self, capsys: pytest.CaptureFixture[str] - ) -> None: - """Test that --list indents only under directories it has listed.""" - entries = [ - ArchiveEntry(path="a/", is_dir=True), - ArchiveEntry(path="a/b/", is_dir=True), - ArchiveEntry(path="a/b/c.txt", size=1024), - ArchiveEntry(path="a/d.txt", size=10), - ArchiveEntry(path="a/e", size=5, is_symlink=True, link_target="d.txt"), - ArchiveEntry(path="x/y.txt", size=0), # no "x/" entry was listed - ] - - list_entries(entries) - - assert capsys.readouterr().out.splitlines() == [ - " " * 12 + "a/", - " " * 14 + "a/b/", - " 1.00 KB" + " " * 6 + "a/b/c.txt", - " 10.00 B" + " " * 4 + "a/d.txt", - " " * 14 + "a/e -> d.txt", - " 0.00 B" + " " * 2 + "x/y.txt", - ] - @pytest.mark.parametrize( "flags,expect,renames", [ @@ -259,17 +208,6 @@ def test_a_failed_input_does_not_stop_the_rest( restored = dest / source_tree.name / original.name assert restored.read_bytes() == original.read_bytes() - def test_listing_a_single_file_container_is_refused( - self, payload: Path, tmp_path: Path - ) -> None: - """Test that there is nothing to list in a container holding one file.""" - packed = tmp_path / "p.gz" - invoke("compress", payload, "-o", packed, "-q") - - result = invoke("extract", packed, "--list", expect=EXIT_USAGE) - - assert "nothing to list" in result.output - class TestMergedVerbCompatibility: """Test that the old names still work, and the old argument order says so.""" diff --git a/tests/python/cli/test_inspect.py b/tests/python/cli/test_inspect.py index 4f73c5f..902b045 100644 --- a/tests/python/cli/test_inspect.py +++ b/tests/python/cli/test_inspect.py @@ -32,23 +32,15 @@ def test_inspect_json(self, payload: Path, tmp_path: Path) -> None: assert orjson.loads(result.output)["is_compresso"] is True def test_inspect_archive_text(self, make_archive: Callable[[str], Path]) -> None: - """Test that an archive shows only a summary unless --entries is given.""" - archive = make_archive("zip") - - summary = invoke("inspect", archive).output - listing = invoke("inspect", archive, "--entries").output + """Test that an archive shows a summary, not its entries.""" + summary = invoke("inspect", make_archive("zip")).output assert "Entries: 4" in summary assert "f0.bin" not in summary - assert "f0.bin" in listing def test_inspect_archive_json(self, make_archive: Callable[[str], Path]) -> None: - """Test that --json carries entries only when --entries is given.""" - archive = make_archive("zip") - - summary = orjson.loads(invoke("inspect", archive, "--json").output) - listing = orjson.loads(invoke("inspect", archive, "--entries", "--json").output) + """Test that --json carries the summary, not the entries.""" + summary = orjson.loads(invoke("inspect", make_archive("zip"), "--json").output) assert summary["entry_count"] == 4 assert "entries" not in summary - assert sum("f0.bin" in e["path"] for e in listing["entries"]) == 1 diff --git a/tests/python/cli/test_list.py b/tests/python/cli/test_list.py new file mode 100644 index 0000000..16ab252 --- /dev/null +++ b/tests/python/cli/test_list.py @@ -0,0 +1,95 @@ +"""Tests for `compresso list`.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import orjson +import pytest + +from compresso.cli._render import EXIT_FAILED, format_size +from compresso.cli._render.info import list_entries +from compresso.frontend.archive_api import ArchiveEntry + +from .conftest import TREE_FILE_SIZE, invoke + + +class TestList: + """Test the read-only `list` command.""" + + def test_list_shows_sizes_and_paths( + self, make_archive: Callable[[str], Path] + ) -> None: + """Test that list prints each entry's size and path.""" + lines = invoke("list", make_archive("tar")).output.splitlines() + + f0_line = next(line for line in lines if "f0.bin" in line) + assert f0_line.split()[:2] == format_size(TREE_FILE_SIZE).split() + + # Directories have a blank size; their contents are indented beneath them + dir_line = next(line for line in lines if line.rstrip("/").endswith("tree")) + assert dir_line == f"{'':>10} {dir_line.strip()}" + assert f0_line.endswith(" tree/f0.bin") + + def test_list_indents_by_depth(self, capsys: pytest.CaptureFixture[str]) -> None: + """Test that list indents only under directories it has listed.""" + entries = [ + ArchiveEntry(path="a/", is_dir=True), + ArchiveEntry(path="a/b/", is_dir=True), + ArchiveEntry(path="a/b/c.txt", size=1024), + ArchiveEntry(path="a/d.txt", size=10), + ArchiveEntry(path="a/e", size=5, is_symlink=True, link_target="d.txt"), + ArchiveEntry(path="x/y.txt", size=0), # no "x/" entry was listed + ] + + list_entries(entries) + + assert capsys.readouterr().out.splitlines() == [ + " " * 12 + "a/", + " " * 14 + "a/b/", + " 1.00 KB" + " " * 6 + "a/b/c.txt", + " 10.00 B" + " " * 4 + "a/d.txt", + " " * 14 + "a/e -> d.txt", + " 0.00 B" + " " * 2 + "x/y.txt", + ] + + def test_long_shows_mode_and_zip_detail( + self, make_archive: Callable[[str], Path] + ) -> None: + """Test that -l shows each entry's mode, and a zip entry's CRC.""" + lines = invoke("list", "-l", make_archive("zip")).output.splitlines() + + f0_line = next(line for line in lines if "f0.bin" in line) + assert f0_line.startswith("-rw") + assert ", crc " in f0_line + + def test_several_inputs_carry_on_past_a_single_file( + self, payload: Path, tmp_path: Path, make_archive: Callable[[str], Path] + ) -> None: + """Test that each archive gets a header, and a single-file input fails + alone, exiting 1 once both archives are listed.""" + tar, zip_ = make_archive("tar"), make_archive("zip") + packed = tmp_path / "p.gz" + invoke("compress", payload, "-o", packed, "-q") + + result = invoke("list", tar, packed, zip_, expect=EXIT_FAILED) + + assert f"{packed}: holds a single file" in result.output + assert result.stdout.startswith(f"==> {tar} <==\n") + assert f"\n\n==> {zip_} <==\n" in result.stdout + assert f"==> {packed} <==" not in result.stdout + + def test_json_is_an_array_per_archive( + self, make_archive: Callable[[str], Path] + ) -> None: + """Test that --json is an array even for one input, with every entry field.""" + archive = make_archive("zip") + + (listing,) = orjson.loads(invoke("list", archive, "--json").output) + + assert listing["path"] == str(archive) + assert listing["format"] == "zip" + f0 = next(e for e in listing["entries"] if e["path"].endswith("f0.bin")) + assert f0["size"] == TREE_FILE_SIZE + assert f0["crc"] is not None From d1641cee3ba71bf46efce56452a5884d76680f99 Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Sat, 24 Oct 2026 12:11:26 +0100 Subject: [PATCH 10/31] feat(cli): rename 'inspect' to 'info' and accept several inputs - Renames the 'inspect' command to 'info' - 'info' now dispatches on the detected format - An archive gets its summary, a '.comp' its header detail, and a standalone codec file its format and size - Now takes several inputs, each headed with '==> path <==', and runs through run_all() - An unrecognised or unreadable input fails alone, while with the rest still described - '--json' is one array item per input, each carrying its format - The render calls now return their JSON item and print only the text, so one print_json() serialises what both commands collected - require_files() moves into _run.py for the shared up-front stat - Adds an 'OSError' branch to run_all(), so a file made unreadable since that stat is reported against itself - Drops 'ExtractJob.list_contents' along with 'extract --list' --- src/compresso/cli/__init__.py | 4 +- src/compresso/cli/_render/info.py | 132 +++++++++++++--------- src/compresso/cli/_run.py | 23 +++- src/compresso/cli/extract.py | 13 +-- src/compresso/cli/info.py | 104 +++++++++++++++++ src/compresso/cli/inspect.py | 65 ----------- src/compresso/cli/list.py | 20 ++-- src/compresso/frontend/archive_api.py | 8 -- src/compresso/meson.build | 2 +- tests/python/cli/test_app.py | 42 +++++-- tests/python/cli/test_info.py | 77 +++++++++++++ tests/python/cli/test_inspect.py | 46 -------- tests/python/frontend/test_archive_api.py | 1 - 13 files changed, 334 insertions(+), 203 deletions(-) create mode 100644 src/compresso/cli/info.py delete mode 100644 src/compresso/cli/inspect.py create mode 100644 tests/python/cli/test_info.py delete mode 100644 tests/python/cli/test_inspect.py diff --git a/src/compresso/cli/__init__.py b/src/compresso/cli/__init__.py index c4501ef..78c1e9d 100644 --- a/src/compresso/cli/__init__.py +++ b/src/compresso/cli/__init__.py @@ -2,7 +2,7 @@ from __future__ import annotations -from . import benchmark, compress, extract, formats, inspect +from . import benchmark, compress, extract, formats, info from . import list as list_ from ._app import app @@ -13,7 +13,7 @@ compress, extract, list_, - inspect, + info, benchmark, formats, ) diff --git a/src/compresso/cli/_render/info.py b/src/compresso/cli/_render/info.py index 8fafbf3..3a85e43 100644 --- a/src/compresso/cli/_render/info.py +++ b/src/compresso/cli/_render/info.py @@ -11,7 +11,7 @@ from ...frontend.archive_api import ArchiveEntry from ...introspect.file_inspect import InspectResult from .._app import app -from . import EXIT_USAGE, fail, format_size, format_time, succeed +from . import format_size, format_time, succeed def list_entries(entries: list[ArchiveEntry], *, long: bool = False) -> None: @@ -92,50 +92,49 @@ def _entry_detail_line(entry: ArchiveEntry) -> str: return f"{mode} {mtime} {size:>10} {entry.path}{target}{detail}" -def listings_json(listings: list[tuple[Path, str, list[ArchiveEntry]]]) -> None: - """Print archives' entries as a JSON array of `{path, format, entries}`. +def print_json(items: list[dict[str, object]]) -> None: + """Print a JSON array, one item per input. Args: - listings: Each archive's path, detected format and entries. + items: Each input's description. """ - data = [ - {"path": str(path), "format": fmt, "entries": entries} - for path, fmt, entries in listings - ] - print(orjson.dumps(data, option=orjson.OPT_INDENT_2).decode()) + print(orjson.dumps(items, option=orjson.OPT_INDENT_2).decode()) def archive_info( file: Path, - fmt: str | None, + fmt: str, entries: list[ArchiveEntry], *, as_json: bool, -) -> None: - """Print an archive's metadata, as text or JSON. +) -> dict[str, object]: + """Describe an archive, printing the text form unless JSON is wanted. Args: file: The archive's path. fmt: The archive's detected format. entries: The archive's entries. - as_json: If True, print JSON instead of text. + as_json: If True, print nothing; the caller prints the returned item. + + Returns: + The archive's JSON item. """ total_size = sum(entry.size for entry in entries) archive_size: int = file.stat().st_size + data: dict[str, object] = { + "path": str(file), + "format": fmt, + "is_archive": True, + "entry_count": len(entries), + "total_size": total_size, + "archive_size": archive_size, + } if as_json: - data: dict[str, object] = { - "path": str(file), - "format": fmt, - "is_archive": True, - "entry_count": len(entries), - "total_size": total_size, - "archive_size": archive_size, - } - print(orjson.dumps(data, option=orjson.OPT_INDENT_2).decode()) - return + return data lines: list[str] = [ + "", f"File: {file}", f"Format: {fmt}", f"Entries: {len(entries)}", @@ -148,44 +147,70 @@ def archive_info( lines.append(f"Compression: {ratio:.1f}% of original") print("\n".join(lines)) + print() + + return data -def file_info(result: InspectResult, *, as_json: bool) -> None: - """Print a single-file container's metadata, as text or JSON. +def codec_info(file: Path, fmt: str, *, as_json: bool) -> dict[str, object]: + """Describe a standalone codec file, e.g. a `.gz`, by its format and size. Args: - result: The container's inspection. - as_json: If True, print JSON instead of text. + file: The file's path. + fmt: The file's detected format. + as_json: If True, print nothing; the caller prints the returned item. + + Returns: + The file's JSON item. """ - if as_json: - data: dict[str, str | int | None | float] = { - "path": str(object=result.path), - "is_compresso": result.is_compresso, - "header_ok": result.header_ok, - "version": result.version, - "algo_id": result.algo_id, - "algo_name": result.algo_name, - "level": result.level, - "flags": result.flags, - "orig_size": result.orig_size, - "backend_available": result.backend_available, - "can_decompress": result.can_decompress, - "estimated_decomp_s": result.estimated_decomp_s, - "reason": result.reason, - } - print(orjson.dumps(data, option=orjson.OPT_INDENT_2).decode()) - return + size: int = file.stat().st_size + data: dict[str, object] = {"path": str(file), "format": fmt, "size": size} - print(f"File: {result.path}\n") + lines: list[str] = [ + "", + f"File: {file}", + f"Format: {fmt}", + f"Size: {format_size(size_bytes=size)}", + "", + ] + + if not as_json: + print("\n".join(lines)) - if not result.is_compresso: - reason = f"\n Reason: {result.reason}" if result.reason else "" - fail(f"Not a valid Compresso file{reason}", EXIT_USAGE) + return data - if not result.header_ok: - reason = f"\n Reason: {result.reason}" if result.reason else "" - fail(f"Invalid file header{reason}", EXIT_USAGE) +def file_info(result: InspectResult, *, as_json: bool) -> dict[str, object]: + """Describe a `.comp` file with a valid header from its inspection. + + Args: + result: The container's inspection. + as_json: If True, print nothing; the caller prints the returned item. + + Returns: + The file's JSON item. + """ + data: dict[str, object] = { + "path": str(object=result.path), + "format": "compresso", + "is_compresso": result.is_compresso, + "header_ok": result.header_ok, + "version": result.version, + "algo_id": result.algo_id, + "algo_name": result.algo_name, + "level": result.level, + "flags": result.flags, + "orig_size": result.orig_size, + "backend_available": result.backend_available, + "can_decompress": result.can_decompress, + "estimated_decomp_s": result.estimated_decomp_s, + "reason": result.reason, + } + + if as_json: + return data + + print(f"File: {result.path}\n") succeed("Valid Compresso file\n") lines: list[str] = [ @@ -219,9 +244,10 @@ def file_info(result: InspectResult, *, as_json: bool) -> None: lines.append( f"Est. decomp time: {format_time(seconds=result.estimated_decomp_s)}" ) - lines.append("") print("\n".join(lines)) if not result.can_decompress and result.reason: app.echo(message=app.style(text=f"\n⚠ {result.reason}", fg="yellow")) + + return data diff --git a/src/compresso/cli/_run.py b/src/compresso/cli/_run.py index 1246ad1..35f84da 100644 --- a/src/compresso/cli/_run.py +++ b/src/compresso/cli/_run.py @@ -13,7 +13,9 @@ BAR_THRESHOLD, EXIT_FAILED, EXIT_OK, + EXIT_USAGE, cancelled, + fail, progress_bar, report_error, ) @@ -25,6 +27,19 @@ class InputFailed(Exception): """One input failed; the runner reports it and goes on to the next.""" +def require_files(paths: Iterable[Path]) -> None: + """Exit 2 before any input runs if one is missing or not a file. + + A stat shows these; what only an input's bytes show fails that input alone. + + Args: + paths: The inputs, as given. + """ + for path in paths: + if not path.is_file(): + fail(f"Error: {path} does not exist or is not a file", EXIT_USAGE) + + def run_job( job: ThreadedJob[Any], source: Path, @@ -73,7 +88,8 @@ def run_all(jobs: Iterable[J], run_one: Callable[[J], None], what: str) -> int: Args: jobs: The planned jobs, one per input. - run_one: Runs one job, raising `InputFailed` or `Error` if it fails. + run_one: Runs one job, raising `InputFailed`, `Error` or `OSError` if + it fails; anything else is a bug and stops the run. what: Operation name for messages, e.g. "Compression". Returns: @@ -92,4 +108,9 @@ def run_all(jobs: Iterable[J], run_one: Callable[[J], None], what: str) -> int: report_error(f"{what} error: {e}") failed = True + # As gzip does, e.g. an input removed or made unreadable since the stat + except OSError as e: + report_error(f"{e.filename}: {e.strerror}" if e.filename else str(e)) + failed = True + return EXIT_FAILED if failed else EXIT_OK diff --git a/src/compresso/cli/extract.py b/src/compresso/cli/extract.py index c1993bf..c08102d 100644 --- a/src/compresso/cli/extract.py +++ b/src/compresso/cli/extract.py @@ -23,7 +23,7 @@ extract_plan, extract_result, ) -from ._run import InputFailed, run_all, run_job +from ._run import InputFailed, require_files, run_all, run_job def _extract_archive(job: ExtractJob, quiet: bool) -> None: @@ -91,7 +91,10 @@ def _decompress_one_file(job: DecompressionJob, quiet: bool) -> None: def extract( - inputs: Annotated[list[Path], app.Argument(help="Archives or compressed files")], + inputs: Annotated[ + list[Path], + app.Argument(help="Archives or compressed files", readable=False), + ], output: Annotated[ Path | None, app.Option( @@ -201,11 +204,7 @@ def extract( if not inputs: fail("Error: No input files given", EXIT_USAGE) - # A stat shows these, so nothing runs; what only an input's bytes show - # fails that input alone - for path in inputs: - if not path.is_file(): - fail(f"Error: {path} does not exist or is not a file", EXIT_USAGE) + require_files(inputs) if overwrite: mode = OverwriteMode.OVERWRITE diff --git a/src/compresso/cli/info.py b/src/compresso/cli/info.py new file mode 100644 index 0000000..c075db1 --- /dev/null +++ b/src/compresso/cli/info.py @@ -0,0 +1,104 @@ +"""The `info` command, describing archives and compressed files.""" + +from __future__ import annotations + +import sys +from collections.abc import Callable +from functools import partial +from pathlib import Path +from typing import Annotated + +from .._core import detect_format +from ..frontend.archive_api import plan_extraction as plan_archive_extraction +from ..frontend.dispatch import is_archive_format +from ..introspect.file_inspect import inspect as inspect_file +from ._app import app +from ._render import cancelled, fail, input_header +from ._render.info import archive_info, codec_info, file_info, print_json +from ._run import InputFailed, require_files, run_all + + +def _describer(path: Path) -> Callable[..., dict[str, object]]: + """Read one input, returning the call that describes it. + + Args: + path: The input. + + Returns: + A render call taking `as_json`, which prints the input unless JSON is + wanted and returns its JSON item. + + Raises: + InputFailed: If the input is unrecognised or can't be read. + """ + fmt = detect_format(path) + + if is_archive_format(fmt): + plan = plan_archive_extraction(archive=path) + if not plan.can_run: + raise InputFailed(f"{path}: {plan.reason_if_unavailable}") + + return partial(archive_info, path, fmt, plan.entries) + + if fmt == "compresso": + result = inspect_file(path=path) + if not result.header_ok: + raise InputFailed(f"{path}: {result.reason}") + + return partial(file_info, result) + + if fmt == "unknown": + raise InputFailed(f"{path}: not a recognised archive or compressed file") + + return partial(codec_info, path, fmt) + + +def info( + inputs: Annotated[ + list[Path], app.Argument(help="Files to describe", readable=False) + ], + output_json: Annotated[ + bool, app.Option("--json", help="Output a JSON array, one item per input") + ] = False, +) -> None: + """Describe archives and compressed files. + \f + + An archive reports its format, entry count, and total size; `list` shows + its entries. A Compresso `.comp` reports its algorithm and decompression + info, and a standalone codec file its format and size. + + Args: + inputs: The archives or compressed files to describe. + output_json: If True, print a JSON array, one item per input. + """ + require_files(inputs) + + items: list[dict[str, object]] = [] + + def run_one(path: Path) -> None: + describe = _describer(path) + + if len(inputs) > 1 and not output_json: + input_header(path, first=not items) + + items.append(describe(as_json=output_json)) + + try: + code = run_all(inputs, run_one, "Inspection") + + if output_json: + print_json(items) + + sys.exit(code) + + except KeyboardInterrupt: + cancelled("Inspection") + + except Exception as e: + fail(f"Unexpected error: {e}") + + +def register() -> None: + """Attach this command to the app.""" + app.command(aliases=["i"])(info) diff --git a/src/compresso/cli/inspect.py b/src/compresso/cli/inspect.py deleted file mode 100644 index 0651399..0000000 --- a/src/compresso/cli/inspect.py +++ /dev/null @@ -1,65 +0,0 @@ -"""The `inspect` command.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Annotated - -from .._core import detect_format -from ..frontend.archive_api import plan_extraction as plan_archive_extraction -from ..frontend.dispatch import looks_like_archive -from ..introspect.file_inspect import inspect as inspect_file -from ._app import app -from ._render import EXIT_USAGE, cancelled, fail -from ._render.info import archive_info, file_info - - -def _inspect_archive(file: Path, output_json: bool) -> None: - """Inspect an archive: a container that can hold many entries. - - Args: - file: The path to the archive. - output_json: If True, output metadata in JSON format. - """ - plan = plan_archive_extraction(archive=file) - - if not plan.can_run: - fail(f"Error: {plan.reason_if_unavailable}", EXIT_USAGE) - - archive_info(file, detect_format(file), plan.entries, as_json=output_json) - - -def inspect( - file: Annotated[Path, app.Argument(help="File to inspect")], - output_json: Annotated[ - bool, app.Option("--json", help="Output in JSON format") - ] = False, -) -> None: - """Inspect a compressed file or archive and show metadata. - \f - - An archive reports its format, entry count, and total size; `list` shows - its entries. A single-file container, a Compresso `.comp` or a standalone - codec, reports its algorithm and decompression info as before. - - Args: - file: The path to the compressed file or archive. - output_json: If True, output metadata in JSON format. - """ - try: - if looks_like_archive(file): - _inspect_archive(file, output_json) - return - - file_info(inspect_file(path=file), as_json=output_json) - - except KeyboardInterrupt: - cancelled("Inspection") - - except Exception as e: - fail(f"Error inspecting file: {e}") - - -def register() -> None: - """Attach this command to the app.""" - app.command(aliases=["i", "info"])(inspect) diff --git a/src/compresso/cli/list.py b/src/compresso/cli/list.py index 17e2900..a470b0e 100644 --- a/src/compresso/cli/list.py +++ b/src/compresso/cli/list.py @@ -11,9 +11,9 @@ from ..frontend.archive_api import plan_extraction as plan_archive_extraction from ..frontend.dispatch import is_archive_format from ._app import app -from ._render import EXIT_USAGE, cancelled, fail, input_header -from ._render.info import list_entries, listings_json -from ._run import InputFailed, run_all +from ._render import cancelled, fail, input_header +from ._render.info import list_entries, print_json +from ._run import InputFailed, require_files, run_all def _read_entries(path: Path) -> tuple[str, list[ArchiveEntry]]: @@ -45,7 +45,9 @@ def _read_entries(path: Path) -> tuple[str, list[ArchiveEntry]]: def list_archives( - inputs: Annotated[list[Path], app.Argument(help="Archives to list")], + inputs: Annotated[ + list[Path], app.Argument(help="Archives to list", readable=False) + ], long: Annotated[ bool, app.Option("--long", "-l", help="Show each entry's mode, mtime and detail"), @@ -66,11 +68,9 @@ def list_archives( long: If True, show each entry's full detail. output_json: If True, print a JSON array of `{path, format, entries}`. """ - for path in inputs: - if not path.is_file(): - fail(f"Error: {path} does not exist or is not a file", EXIT_USAGE) + require_files(inputs) - listings: list[tuple[Path, str, list[ArchiveEntry]]] = [] + listings: list[dict[str, object]] = [] def run_one(path: Path) -> None: fmt, entries = _read_entries(path) @@ -81,13 +81,13 @@ def run_one(path: Path) -> None: list_entries(entries, long=long) - listings.append((path, fmt, entries)) + listings.append({"path": str(path), "format": fmt, "entries": entries}) try: code = run_all(inputs, run_one, "Listing") if output_json: - listings_json(listings) + print_json(listings) sys.exit(code) diff --git a/src/compresso/frontend/archive_api.py b/src/compresso/frontend/archive_api.py index bf11fa0..530e00a 100644 --- a/src/compresso/frontend/archive_api.py +++ b/src/compresso/frontend/archive_api.py @@ -551,14 +551,6 @@ def from_archive( """ return cls(plan=plan_extraction(archive, output_dir, files, options)) - def list_contents(self) -> list[ArchiveEntry]: - """List archive entry paths without extracting. - - Returns: - List of ArchiveEntry instances. - """ - return list(self.plan.entries) - def run( self, progress: ProgressCallback | None = None, diff --git a/src/compresso/meson.build b/src/compresso/meson.build index 784cf96..66c2f7e 100644 --- a/src/compresso/meson.build +++ b/src/compresso/meson.build @@ -57,7 +57,7 @@ py.install_sources( 'cli/compress.py', 'cli/extract.py', 'cli/formats.py', - 'cli/inspect.py', + 'cli/info.py', 'cli/list.py', subdir: 'compresso/cli', ) diff --git a/tests/python/cli/test_app.py b/tests/python/cli/test_app.py index fd4e399..e62e022 100644 --- a/tests/python/cli/test_app.py +++ b/tests/python/cli/test_app.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +import sys from collections.abc import Callable from pathlib import Path from typing import Any @@ -10,7 +11,7 @@ import pytest import compresso.cli.formats as formats_cmd -import compresso.cli.inspect as inspect_cmd +import compresso.cli.info as info_cmd import compresso.cli.list as list_cmd from compresso.cli._render import ( EXIT_CANCELLED, @@ -85,7 +86,7 @@ def test_commands_are_listed_in_workflow_order(self) -> None: "(c, comp", "(x, ex", "(l)", - "(i, info)", + "(i)", "(b, bench)", "List the available", ] @@ -106,8 +107,7 @@ def test_commands_are_listed_in_workflow_order(self) -> None: ("d", "extract"), ("decomp", "extract"), ("l", "list"), - ("i", "inspect"), - ("info", "inspect"), + ("i", "info"), ("b", "benchmark"), ("bench", "benchmark"), ], @@ -195,16 +195,40 @@ class TestExitCodes: "command,name", [ ("compress", "nope.bin"), - ("inspect", "payload.bin"), + ("info", "nope.comp"), ("extract", "nope.gz"), ("list", "nope.zip"), ], ) def test_bad_input_is_usage(self, payload: Path, command: str, name: str) -> None: - """Test that a missing input, or one inspect can't read, is wrong, rather - than the operation having failed.""" + """Test that a missing input is wrong, rather than the operation having + failed.""" invoke(command, payload.parent / name, expect=EXIT_USAGE) + @pytest.mark.skipif( + sys.platform == "win32" or os.geteuid() == 0, reason="needs POSIX, not root" + ) + @pytest.mark.parametrize( + "command,shown", [("list", "f0.bin"), ("info", "Entries:")] + ) + def test_unreadable_input_fails_alone( + self, + tmp_path: Path, + make_archive: Callable[[str], Path], + command: str, + shown: str, + ) -> None: + """Test that an OSError from one input is reported against it, as gzip + does, and the next input still runs.""" + locked = tmp_path / "locked.zip" + locked.write_bytes(b"") + locked.chmod(0) + + result = invoke(command, locked, make_archive("zip"), expect=EXIT_FAILED) + + assert f"{locked}: Permission denied" in result.output + assert shown in result.stdout + @pytest.mark.parametrize("command", ["compress", "extract"]) @pytest.mark.parametrize( "flags", @@ -284,7 +308,7 @@ class TestInterruption: "command,expected", [ (["compress"], "Compression cancelled"), - (["inspect"], "Inspection cancelled"), + (["info"], "Inspection cancelled"), (["list"], "Listing cancelled"), (["formats"], "Format listing cancelled"), ], @@ -303,7 +327,7 @@ def interrupt(*args: object, **kwargs: object) -> None: raise KeyboardInterrupt monkeypatch.setattr(CompressionJob, "from_file", interrupt) - monkeypatch.setattr(inspect_cmd, "inspect_file", interrupt) + monkeypatch.setattr(info_cmd, "detect_format", interrupt) monkeypatch.setattr(list_cmd, "detect_format", interrupt) monkeypatch.setattr(formats_cmd, "list_capabilities", interrupt) diff --git a/tests/python/cli/test_info.py b/tests/python/cli/test_info.py new file mode 100644 index 0000000..fb96449 --- /dev/null +++ b/tests/python/cli/test_info.py @@ -0,0 +1,77 @@ +"""Tests for `compresso info`.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import orjson + +from compresso.cli._render import EXIT_FAILED + +from .conftest import invoke + + +class TestInfo: + """Test the read-only `info` command.""" + + def test_info_reports_a_valid_file(self, payload: Path, tmp_path: Path) -> None: + """Test that info describes a file the CLI just wrote.""" + archive = tmp_path / "out.comp" + invoke("compress", payload, "-o", archive, "-q") + + result = invoke("info", archive) + + assert "Valid Compresso file" in result.output + + def test_info_archive_text(self, make_archive: Callable[[str], Path]) -> None: + """Test that an archive shows a summary, not its entries.""" + summary = invoke("info", make_archive("zip")).output + + assert "Entries: 4" in summary + assert "f0.bin" not in summary + + def test_standalone_codec_shows_format_and_size( + self, payload: Path, tmp_path: Path + ) -> None: + """Test that a .gz is described by its format, not refused as a bad .comp.""" + packed = tmp_path / "p.gz" + invoke("compress", payload, "-o", packed, "-q") + + output = invoke("info", packed).output + + assert "Format: gzip" in output + assert "Size:" in output + + def test_several_inputs_carry_on_past_an_unrecognised_one( + self, payload: Path, tmp_path: Path, make_archive: Callable[[str], Path] + ) -> None: + """Test that each described input gets a header, and an unrecognised one + fails alone, exiting 1 once the rest are described.""" + archive = make_archive("tar") + packed = tmp_path / "p.gz" + invoke("compress", payload, "-o", packed, "-q") + + result = invoke("info", archive, payload, packed, expect=EXIT_FAILED) + + assert f"{payload}: not a recognised archive" in result.output + assert result.stdout.startswith(f"==> {archive} <==\n") + assert f"\n\n==> {packed} <==\n" in result.stdout + assert f"==> {payload} <==" not in result.stdout + + def test_json_is_an_array_per_input( + self, payload: Path, tmp_path: Path, make_archive: Callable[[str], Path] + ) -> None: + """Test that --json is one array item per input, whatever its kind.""" + comp, packed = tmp_path / "out.comp", tmp_path / "p.gz" + invoke("compress", payload, "-o", comp, "-q") + invoke("compress", payload, "-o", packed, "-q") + + items = orjson.loads( + invoke("info", comp, make_archive("zip"), packed, "--json").output + ) + + assert [item["format"] for item in items] == ["compresso", "zip", "gzip"] + assert items[0]["is_compresso"] is True + assert items[1]["entry_count"] == 4 + assert items[2]["size"] == packed.stat().st_size diff --git a/tests/python/cli/test_inspect.py b/tests/python/cli/test_inspect.py deleted file mode 100644 index 902b045..0000000 --- a/tests/python/cli/test_inspect.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Tests for `compresso inspect`.""" - -from __future__ import annotations - -from collections.abc import Callable -from pathlib import Path - -import orjson - -from .conftest import invoke - - -class TestInspect: - """Test the read-only `inspect` command.""" - - def test_inspect_reports_a_valid_file(self, payload: Path, tmp_path: Path) -> None: - """Test that inspect describes a file the CLI just wrote.""" - archive = tmp_path / "out.comp" - invoke("compress", payload, "-o", archive, "-q") - - result = invoke("inspect", archive) - - assert "Valid Compresso file" in result.output - - def test_inspect_json(self, payload: Path, tmp_path: Path) -> None: - """Test that --json emits parseable output.""" - archive = tmp_path / "out.comp" - invoke("compress", payload, "-o", archive, "-q") - - result = invoke("inspect", archive, "--json") - - assert orjson.loads(result.output)["is_compresso"] is True - - def test_inspect_archive_text(self, make_archive: Callable[[str], Path]) -> None: - """Test that an archive shows a summary, not its entries.""" - summary = invoke("inspect", make_archive("zip")).output - - assert "Entries: 4" in summary - assert "f0.bin" not in summary - - def test_inspect_archive_json(self, make_archive: Callable[[str], Path]) -> None: - """Test that --json carries the summary, not the entries.""" - summary = orjson.loads(invoke("inspect", make_archive("zip"), "--json").output) - - assert summary["entry_count"] == 4 - assert "entries" not in summary diff --git a/tests/python/frontend/test_archive_api.py b/tests/python/frontend/test_archive_api.py index eb83085..18aa937 100644 --- a/tests/python/frontend/test_archive_api.py +++ b/tests/python/frontend/test_archive_api.py @@ -280,7 +280,6 @@ def test_round_trip(self, tmp_path: Path, fmt: str) -> None: assert result.ok, result.error extract_job = ExtractJob.from_archive(archive_path, tmp_path / "sortie") - assert all(isinstance(e, ArchiveEntry) for e in extract_job.list_contents()) result = extract_job.run() assert result.ok, result.error From 875929422f10d5a07fdaaa4596f03e959c1e1290 Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Sat, 24 Oct 2026 13:04:28 +0100 Subject: [PATCH 11/31] feat(cli): trim aliases and drop the 'comp' executable - Each command now has a single initial alias, rather than multiple - Drops the 'comp' console script, so the name isn't claimed on PATH twice - 'formats' moves ahead of 'benchmark' in the help - Drops the now unused 'TestMergedVerbCompatibility' --- pyproject.toml | 1 - src/compresso/cli/__init__.py | 2 +- src/compresso/cli/benchmark.py | 2 +- src/compresso/cli/compress.py | 4 ++-- src/compresso/cli/extract.py | 4 ++-- tests/python/cli/test_app.py | 15 +++------------ tests/python/cli/test_compress.py | 4 ++-- tests/python/cli/test_extract.py | 21 +-------------------- 8 files changed, 12 insertions(+), 41 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f12082d..90a517d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,6 @@ dev = [ [project.scripts] compresso = "compresso.cli:main" -comp = "compresso.cli:main" [tool.meson-python.args] # Meson only activates MSVC when no other compiler is on PATH, which the MinGW diff --git a/src/compresso/cli/__init__.py b/src/compresso/cli/__init__.py index 78c1e9d..93c565c 100644 --- a/src/compresso/cli/__init__.py +++ b/src/compresso/cli/__init__.py @@ -14,8 +14,8 @@ extract, list_, info, - benchmark, formats, + benchmark, ) for _module in _COMMAND_MODULES: diff --git a/src/compresso/cli/benchmark.py b/src/compresso/cli/benchmark.py index 64449f9..daa6771 100644 --- a/src/compresso/cli/benchmark.py +++ b/src/compresso/cli/benchmark.py @@ -133,4 +133,4 @@ def benchmark( def register() -> None: """Attach this command to the app.""" - app.command(aliases=["b", "bench"])(benchmark) + app.command(aliases=["b"])(benchmark) diff --git a/src/compresso/cli/compress.py b/src/compresso/cli/compress.py index f7965db..e51f1ec 100644 --- a/src/compresso/cli/compress.py +++ b/src/compresso/cli/compress.py @@ -1,4 +1,4 @@ -"""The `compress` command, which also answers to `archive`.""" +"""The `compress` command, packing files into archives and compressed files.""" from __future__ import annotations @@ -232,4 +232,4 @@ def run_one(job: CompressionJob | ArchiveJob) -> None: def register() -> None: """Attach this command to the app.""" - app.command(aliases=["c", "comp", "archive", "a", "ar"])(compress) + app.command(aliases=["c"])(compress) diff --git a/src/compresso/cli/extract.py b/src/compresso/cli/extract.py index c08102d..ddd0090 100644 --- a/src/compresso/cli/extract.py +++ b/src/compresso/cli/extract.py @@ -1,4 +1,4 @@ -"""The `extract` command, which also answers to `decompress`.""" +"""The `extract` command, unpacking archives and compressed files.""" from __future__ import annotations @@ -249,4 +249,4 @@ def run_one(job: ExtractJob | DecompressionJob) -> None: def register() -> None: """Attach this command to the app.""" - app.command(aliases=["x", "ex", "decompress", "d", "decomp"])(extract) + app.command(aliases=["x"])(extract) diff --git a/tests/python/cli/test_app.py b/tests/python/cli/test_app.py index e62e022..8e9c283 100644 --- a/tests/python/cli/test_app.py +++ b/tests/python/cli/test_app.py @@ -83,12 +83,12 @@ def test_commands_are_listed_in_workflow_order(self) -> None: positions = [ output.index(aliases) for aliases in [ - "(c, comp", - "(x, ex", + "(c)", + "(x)", "(l)", "(i)", - "(b, bench)", "List the available", + "(b)", ] ] assert positions == sorted(positions) @@ -97,19 +97,10 @@ def test_commands_are_listed_in_workflow_order(self) -> None: "alias,command", [ ("c", "compress"), - ("comp", "compress"), - ("archive", "compress"), - ("a", "compress"), - ("ar", "compress"), ("x", "extract"), - ("ex", "extract"), - ("decompress", "extract"), - ("d", "extract"), - ("decomp", "extract"), ("l", "list"), ("i", "info"), ("b", "benchmark"), - ("bench", "benchmark"), ], ) def test_aliases_still_resolve(self, alias: str, command: str) -> None: diff --git a/tests/python/cli/test_compress.py b/tests/python/cli/test_compress.py index 956a7c2..0b33db2 100644 --- a/tests/python/cli/test_compress.py +++ b/tests/python/cli/test_compress.py @@ -25,7 +25,7 @@ def test_compress_then_decompress(self, payload: Path, tmp_path: Path) -> None: result = invoke("compress", payload, "-o", archive) assert "Compression successful" in result.output - result = invoke("decompress", archive, "-o", restored) + result = invoke("extract", archive, "-o", restored) assert "Decompression successful" in result.output assert restored.read_bytes() == payload.read_bytes() @@ -47,7 +47,7 @@ def test_algorithms_round_trip( restored = tmp_path / "restored.bin" invoke("compress", payload, "-o", archive, *args, "-q") - invoke("decompress", archive, "-o", restored, "-q") + invoke("extract", archive, "-o", restored, "-q") assert restored.read_bytes() == payload.read_bytes() diff --git a/tests/python/cli/test_extract.py b/tests/python/cli/test_extract.py index 19b128f..4fd2b6f 100644 --- a/tests/python/cli/test_extract.py +++ b/tests/python/cli/test_extract.py @@ -122,7 +122,7 @@ def test_existing_output( invoke("compress", payload, "-o", archive, "-q") restored.write_bytes(b"stale") - result = invoke("decompress", archive, "-o", restored, *flags, expect=expect) + result = invoke("extract", archive, "-o", restored, *flags, expect=expect) sibling = tmp_path / renamed(restored.name) assert (f"Renamed to: {sibling}" in result.output) == renames @@ -207,22 +207,3 @@ def test_a_failed_input_does_not_stop_the_rest( for original in source_tree.iterdir(): restored = dest / source_tree.name / original.name assert restored.read_bytes() == original.read_bytes() - - -class TestMergedVerbCompatibility: - """Test that the old names still work, and the old argument order says so.""" - - def test_decompress_now_handles_archives_too( - self, - source_tree: Path, - tmp_path: Path, - make_archive: Callable[[str], Path], - ) -> None: - """Test that the merged verb does not care which name was typed.""" - dest = tmp_path / "out" - - invoke("decompress", make_archive("tar"), "-o", dest, "-q") - - for original in source_tree.iterdir(): - restored = dest / source_tree.name / original.name - assert restored.read_bytes() == original.read_bytes() From baa83cf7e1c4ae2299026ff4b614c044e7dd8119 Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Sat, 24 Oct 2026 15:02:21 +0100 Subject: [PATCH 12/31] feat(cli): replace the overwrite flags with '--on-conflict' and '-f' - Drops the multiple overwrite flags on the 'compress' and 'extract' commands - New '--on-conflict' option takes overwrite mode by name, with '-f/--force' as shorthand for overwrite - New conflict_mode() in _run.py resolves the two for both commands, refusing '-f' alongside a mode that contradicts it - '--format' now uses a '-F' alias, rather than '-f' - 'OverwriteMode' and 'SymlinkMode' now list their members default-first, so the help shows the choices in that order --- src/compresso/cli/_run.py | 20 ++++++++++ src/compresso/cli/compress.py | 56 ++++++++------------------- src/compresso/cli/extract.py | 46 ++++++++-------------- src/compresso/frontend/archive_api.py | 20 +++++----- tests/python/cli/conftest.py | 13 ++++--- tests/python/cli/test_app.py | 28 +++++--------- tests/python/cli/test_compress.py | 6 +-- tests/python/cli/test_extract.py | 11 +++--- 8 files changed, 90 insertions(+), 110 deletions(-) diff --git a/src/compresso/cli/_run.py b/src/compresso/cli/_run.py index 35f84da..52d4435 100644 --- a/src/compresso/cli/_run.py +++ b/src/compresso/cli/_run.py @@ -9,6 +9,7 @@ from .._core import Error from ..frontend._job import ThreadedJob +from ..frontend.archive_api import OverwriteMode from ._render import ( BAR_THRESHOLD, EXIT_FAILED, @@ -40,6 +41,25 @@ def require_files(paths: Iterable[Path]) -> None: fail(f"Error: {path} does not exist or is not a file", EXIT_USAGE) +def conflict_mode(on_conflict: OverwriteMode | None, force: bool) -> OverwriteMode: + """Resolve `--on-conflict` and `-f` into one mode, exiting 2 if they disagree. + + Args: + on_conflict: The mode given, or None if it was not. + force: Whether `-f`, shorthand for `--on-conflict overwrite`, was given. + + Returns: + The mode to run with; RENAME if neither was given. + """ + if not force: + return on_conflict or OverwriteMode.RENAME + + if on_conflict not in (None, OverwriteMode.OVERWRITE): + fail(f"Error: -f contradicts --on-conflict {on_conflict}", EXIT_USAGE) + + return OverwriteMode.OVERWRITE + + def run_job( job: ThreadedJob[Any], source: Path, diff --git a/src/compresso/cli/compress.py b/src/compresso/cli/compress.py index e51f1ec..1d4ac1a 100644 --- a/src/compresso/cli/compress.py +++ b/src/compresso/cli/compress.py @@ -13,7 +13,7 @@ from ._app import app from ._render import EXIT_USAGE, cancelled, fail from ._render.jobs import archive_plan, archive_result, compress_plan, compress_result -from ._run import run_all, run_job +from ._run import conflict_mode, run_all, run_job def _compress_one_file(job: CompressionJob, quiet: bool) -> None: @@ -87,7 +87,7 @@ def compress( str | None, app.Option( "--format", - "-f", + "-F", case_sensitive=False, help="Container to write, e.g. gz, zst, tar.zst, zip", ), @@ -115,24 +115,18 @@ def compress( help="Compression level; each algorithm's range is shown by `list`", ), ] = None, - overwrite: Annotated[ - bool, - app.Option( - "--overwrite", help="Replace an archive that already exists at the output" - ), - ] = False, - skip_existing: Annotated[ - bool, + on_conflict: Annotated[ + OverwriteMode | None, app.Option( - "--skip-existing", help="Leave an existing output archive untouched" + "--on-conflict", + case_sensitive=False, + help="What to do when the output exists", + show_default="rename", ), - ] = False, - error_on_conflict: Annotated[ + ] = None, + force: Annotated[ bool, - app.Option( - "--error-on-conflict", - help="Fail instead of renaming a clashing output archive", - ), + app.Option("--force", "-f", help="Replace an existing output"), ] = False, threads: Annotated[ int, @@ -151,15 +145,14 @@ def compress( \f One file becomes a single-file container: a Compresso `.comp` by default, - or `-f gz`/`zst`/`xz`/`bz2`/`lz4` for a standalone one; a directory, or + or `-F gz`/`zst`/`xz`/`bz2`/`lz4` for a standalone one; a directory, or more than one input, becomes an archive (`zip` unless told otherwise); - a recognisable extension on `-o` picks the format on its own, and `-f` + a recognisable extension on `-o` picks the format on its own, and `-F` overrides it. A clashing output name by default gets a platform-native numbered sibling (`name 2.ext` on macOS, `name (2).ext` elsewhere), for an archive and a - single-file container alike; use `--overwrite`, `--skip-existing`, or - `--error-on-conflict` to change that. + single-file container alike; use `--on-conflict` or `-f` to change that. Args: inputs: The files and directories to compress. @@ -168,27 +161,12 @@ def compress( algo: Algorithm for the Compresso container (default: auto). strategy: Strategy for the Compresso container (default: balanced). level: Compression level (default: the format's own). - overwrite: If True, replace an existing output. - skip_existing: If True, leave an existing output untouched. - error_on_conflict: If True, fail instead of renaming a clashing output. + on_conflict: What to do when the output exists (default: rename). + force: If True, replace an existing output. threads: Worker threads for zstd and xz (default: one per core). quiet: If True, suppress all output. """ - if sum([overwrite, skip_existing, error_on_conflict]) > 1: - fail( - "Error: --overwrite, --skip-existing and --error-on-conflict " - "are mutually exclusive", - EXIT_USAGE, - ) - - if overwrite: - overwrite_mode = OverwriteMode.OVERWRITE - elif skip_existing: - overwrite_mode = OverwriteMode.SKIP - elif error_on_conflict: - overwrite_mode = OverwriteMode.ERROR - else: - overwrite_mode = OverwriteMode.RENAME + overwrite_mode = conflict_mode(on_conflict, force) try: if not inputs: diff --git a/src/compresso/cli/extract.py b/src/compresso/cli/extract.py index ddd0090..dd7cb31 100644 --- a/src/compresso/cli/extract.py +++ b/src/compresso/cli/extract.py @@ -23,7 +23,7 @@ extract_plan, extract_result, ) -from ._run import InputFailed, require_files, run_all, run_job +from ._run import InputFailed, conflict_mode, require_files, run_all, run_job def _extract_archive(job: ExtractJob, quiet: bool) -> None: @@ -104,17 +104,18 @@ def extract( help="Where to write (default: beside the input)", ), ] = None, - overwrite: Annotated[ - bool, app.Option("--overwrite", help="Replace files that already exist") - ] = False, - skip_existing: Annotated[ - bool, app.Option("--skip-existing", help="Leave files that already exist") - ] = False, - error_on_conflict: Annotated[ - bool, + on_conflict: Annotated[ + OverwriteMode | None, app.Option( - "--error-on-conflict", help="Fail instead of renaming a clashing file" + "--on-conflict", + case_sensitive=False, + help="What to do when a file exists", + show_default="rename", ), + ] = None, + force: Annotated[ + bool, + app.Option("--force", "-f", help="Replace files that already exist"), ] = False, max_total_size: Annotated[ int | None, @@ -173,17 +174,16 @@ def extract( The result is written beside its input unless `-o` is provided. By default, a clashing name gets a platform-native numbered sibling - (`name 2.ext` on macOS, `name (2).ext` elsewhere); use `--overwrite`, - `--skip-existing`, or `--error-on-conflict` to change that. + (`name 2.ext` on macOS, `name (2).ext` elsewhere); use `--on-conflict` or + `-f` to change that. Args: inputs: The archives or compressed files to unpack. output: Where to write (default: beside the input); a directory for archives, and for one file either a directory to put it in or the path to write. - overwrite: If True, replace files that already exist. - skip_existing: If True, leave files that already exist untouched. - error_on_conflict: If True, fail instead of renaming a clashing file. + on_conflict: What to do when a file exists (default: rename). + force: If True, replace files that already exist. max_total_size: Cap on total extracted bytes (default: no cap). same_permissions: If True, restore archive entries' modes exactly; single-file outputs are unaffected. @@ -194,27 +194,13 @@ def extract( stores, and give it the stored modification time, if any. quiet: If True, suppress all output. """ - if sum([overwrite, skip_existing, error_on_conflict]) > 1: - fail( - "Error: --overwrite, --skip-existing and --error-on-conflict " - "are mutually exclusive", - EXIT_USAGE, - ) + mode = conflict_mode(on_conflict, force) if not inputs: fail("Error: No input files given", EXIT_USAGE) require_files(inputs) - if overwrite: - mode = OverwriteMode.OVERWRITE - elif skip_existing: - mode = OverwriteMode.SKIP - elif error_on_conflict: - mode = OverwriteMode.ERROR - else: - mode = OverwriteMode.RENAME - options = ExtractOptions( overwrite=mode, max_total_size=max_total_size or 0, diff --git a/src/compresso/frontend/archive_api.py b/src/compresso/frontend/archive_api.py index 530e00a..62a0386 100644 --- a/src/compresso/frontend/archive_api.py +++ b/src/compresso/frontend/archive_api.py @@ -60,22 +60,22 @@ class OverwriteMode(StrEnum): """What archiving or extraction does when a destination already exists. Attributes: - ERROR: Fail the operation. - SKIP: Leave the existing file untouched. - OVERWRITE: Replace the existing file. RENAME: Write to a platform-native numbered sibling instead (e.g. "name 2.ext" on macOS, "name (2).ext" elsewhere), leaving the existing file untouched. + OVERWRITE: Replace the existing file. + SKIP: Leave the existing file untouched. + ERROR: Fail the operation. """ - ERROR = "error" - SKIP = "skip" - OVERWRITE = "overwrite" RENAME = "rename" + OVERWRITE = "overwrite" + SKIP = "skip" + ERROR = "error" -# The C policy uses ints; see `ExtractionPolicy` in csrc/archives.h. Archive -# creation applies the same codes to its single destination file. +# The C policy uses ints; see `ExtractionPolicy` in csrc/archives.h; archive +# creation applies the same codes to its single destination file _OVERWRITE_CODES: dict[OverwriteMode, int] = { OverwriteMode.ERROR: 0, OverwriteMode.SKIP: 1, @@ -89,13 +89,13 @@ class SymlinkMode(StrEnum): Attributes: DENY: Refuse the archive. - ALLOW: Create the link, if its target stays inside the output directory. FILE: Write a regular file holding the link's target. + ALLOW: Create the link, if its target stays inside the output directory. """ DENY = "deny" - ALLOW = "allow" FILE = "file" + ALLOW = "allow" _SYMLINK_CODES: dict[SymlinkMode, int] = { diff --git a/tests/python/cli/conftest.py b/tests/python/cli/conftest.py index 30e931d..6c03761 100644 --- a/tests/python/cli/conftest.py +++ b/tests/python/cli/conftest.py @@ -111,7 +111,7 @@ def compress_target( if request.param == "file": return CompressTarget(payload, tmp_path / "out.comp", ()) - return CompressTarget(source_tree, tmp_path / "out.tar", ("-f", "tar")) + return CompressTarget(source_tree, tmp_path / "out.tar", ("-F", "tar")) @dataclass @@ -155,9 +155,12 @@ def counting(fraction: float, done: int, reported_total: int) -> None: "flags,expect,kept,renames", [ ((), EXIT_OK, True, True), - (("--error-on-conflict",), EXIT_FAILED, True, False), - (("--skip-existing",), EXIT_OK, True, False), - (("--overwrite",), EXIT_OK, False, False), + (("--on-conflict", "rename"), EXIT_OK, True, True), + (("--on-conflict", "error"), EXIT_FAILED, True, False), + (("--on-conflict", "skip"), EXIT_OK, True, False), + (("--on-conflict", "overwrite"), EXIT_OK, False, False), + (("-f",), EXIT_OK, False, False), + (("-f", "--on-conflict", "overwrite"), EXIT_OK, False, False), ], - ids=["rename", "error", "skip", "overwrite"], + ids=["default", "rename", "error", "skip", "overwrite", "force", "force+overwrite"], ) diff --git a/tests/python/cli/test_app.py b/tests/python/cli/test_app.py index 8e9c283..ea122d1 100644 --- a/tests/python/cli/test_app.py +++ b/tests/python/cli/test_app.py @@ -168,7 +168,7 @@ def test_archive_round_trip_draws_both_bars( archive = tmp_path / "out.tar" dest = tmp_path / "dest" - invoke("compress", tree, "-o", archive, "-f", "tar") + invoke("compress", tree, "-o", archive, "-F", "tar") invoke("extract", archive, "-o", dest) assert [bar.enabled for bar in bars] == [True, True] @@ -221,31 +221,23 @@ def test_unreadable_input_fails_alone( assert shown in result.stdout @pytest.mark.parametrize("command", ["compress", "extract"]) - @pytest.mark.parametrize( - "flags", - [ - ["--overwrite", "--skip-existing"], - ["--overwrite", "--error-on-conflict"], - ["--skip-existing", "--error-on-conflict"], - ], - ids=["overwrite+skip", "overwrite+error", "skip+error"], - ) - def test_mutually_exclusive_flag_usage( + @pytest.mark.parametrize("mode", ["rename", "skip", "error"]) + def test_force_with_another_conflict_mode_is_usage( self, source_tree: Path, tmp_path: Path, make_archive: Callable[[str], Path], command: str, - flags: list[str], + mode: str, ) -> None: - """Test that combining any two overwrite-mode flags is contradictory.""" + """Test that -f, meaning overwrite, contradicts any other --on-conflict.""" if command == "compress": - args = [source_tree, "-o", tmp_path / "out.tar", "-f", "tar"] + args = [source_tree, "-o", tmp_path / "out.tar", "-F", "tar"] else: args = [make_archive("tar"), "-o", tmp_path / "d"] - result = invoke(command, *args, *flags, expect=EXIT_USAGE) - assert "mutually exclusive" in result.output + result = invoke(command, *args, "-f", "--on-conflict", mode, expect=EXIT_USAGE) + assert f"-f contradicts --on-conflict {mode}" in result.output # An archive's level is checked against the stage that compresses @pytest.mark.parametrize( @@ -275,11 +267,11 @@ def test_bad_benchmark_level_is_usage(self, payload: Path) -> None: """Test that a level that is not a number is a usage error.""" invoke("benchmark", payload, "--levels", "nope", expect=EXIT_USAGE) - @pytest.mark.parametrize("flags", [(), ("-f", "7z")], ids=["suffix", "flag"]) + @pytest.mark.parametrize("flags", [(), ("-F", "7z")], ids=["suffix", "flag"]) def test_recognised_but_unsupported_archive_names_itself( self, source_tree: Path, tmp_path: Path, flags: tuple[str, ...] ) -> None: - """Test that 7z, asked for by -f or by `-o x.7z` alone, reports its own name + """Test that 7z, asked for by -F or by `-o x.7z` alone, reports its own name rather than being written as a zip.""" dest = tmp_path / "o.7z" result = invoke("compress", source_tree, "-o", dest, *flags, expect=EXIT_FAILED) diff --git a/tests/python/cli/test_compress.py b/tests/python/cli/test_compress.py index 0b33db2..fe80063 100644 --- a/tests/python/cli/test_compress.py +++ b/tests/python/cli/test_compress.py @@ -103,7 +103,7 @@ def test_single_file_format_refuses_several_inputs( result = invoke( "compress", source_tree, - "-f", + "-F", "gz", "-o", tmp_path / "o.gz", @@ -113,5 +113,5 @@ def test_single_file_format_refuses_several_inputs( assert "single file" in result.output def test_unknown_format_is_refused(self, payload: Path) -> None: - """Test that an unrecognised -f names neither container.""" - invoke("compress", payload, "-f", "bogus", expect=EXIT_USAGE) + """Test that an unrecognised -F names neither container.""" + invoke("compress", payload, "-F", "bogus", expect=EXIT_USAGE) diff --git a/tests/python/cli/test_extract.py b/tests/python/cli/test_extract.py index 4fd2b6f..8f5bf98 100644 --- a/tests/python/cli/test_extract.py +++ b/tests/python/cli/test_extract.py @@ -30,7 +30,7 @@ def test_archive_then_extract( archive = tmp_path / f"out.{fmt}" dest = tmp_path / f"dest.{fmt}" - result = invoke("compress", source_tree, "-o", archive, "-f", fmt) + result = invoke("compress", source_tree, "-o", archive, "-F", fmt) assert "Archive created" in result.output result = invoke("extract", archive, "-o", dest) @@ -44,11 +44,12 @@ def test_archive_then_extract( "flags,expect,renames", [ ((), EXIT_OK, True), - (("--error-on-conflict",), EXIT_FAILED, False), - (("--skip-existing",), EXIT_OK, False), - (("--overwrite",), EXIT_OK, False), + (("--on-conflict", "error"), EXIT_FAILED, False), + (("--on-conflict", "skip"), EXIT_OK, False), + (("--on-conflict", "overwrite"), EXIT_OK, False), + (("-f",), EXIT_OK, False), ], - ids=["rename", "error", "skip", "overwrite"], + ids=["rename", "error", "skip", "overwrite", "force"], ) def test_extract_over_an_existing_tree( self, From dc600a7fd4f34ab71f223141bd62e790cb685e4c Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Mon, 26 Oct 2026 13:53:33 +0000 Subject: [PATCH 13/31] feat(cli): print one summary line per input by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Each finished input now outputs one line: 'name before → after (n%) in time' - Renames are noted where the output's name was taken, and a 'skipped, already exists' line under '--on-conflict skip' - The plan and stats blocks are now shown with a '-v/--verbose' flag - The '-q' flag still prints nothing, and both used together is a usage error - report_error() now flushes stdout first, so a failure's error doesn't appear above the summary lines of earlier inputs - plan_decompression() now reports an OSError's own 'strerror' rather than wrapping it in 'Cannot read file' --- src/compresso/cli/_render/__init__.py | 3 + src/compresso/cli/_render/jobs.py | 31 ++++++++- src/compresso/cli/compress.py | 77 +++++++++++++++++---- src/compresso/cli/extract.py | 63 +++++++++++++----- src/compresso/frontend/api.py | 4 ++ tests/python/cli/test_app.py | 96 ++++++++++++++++++++++++++- tests/python/cli/test_compress.py | 24 +++---- tests/python/cli/test_extract.py | 13 ++-- 8 files changed, 258 insertions(+), 53 deletions(-) diff --git a/src/compresso/cli/_render/__init__.py b/src/compresso/cli/_render/__init__.py index fd4a5aa..a61925f 100644 --- a/src/compresso/cli/_render/__init__.py +++ b/src/compresso/cli/_render/__init__.py @@ -71,6 +71,9 @@ def report_error(message: str) -> None: Args: message: Text to show, without the leading marker. """ + # A piped stdout is block-buffered, so without this an error could + # land before the outputs ahead of it + sys.stdout.flush() app.echo(message=app.style(text=f"✗ {message}", fg="red"), err=True) diff --git a/src/compresso/cli/_render/jobs.py b/src/compresso/cli/_render/jobs.py index 5a57409..f5e38cc 100644 --- a/src/compresso/cli/_render/jobs.py +++ b/src/compresso/cli/_render/jobs.py @@ -9,6 +9,35 @@ from . import format_size, format_time, succeed +def summary( + name: Path, before: int, after: int, elapsed: float, renamed: Path | None +) -> None: + """Print the one line a finished input gets by default. + + Args: + name: The output for compress, the input for extract. + before: Bytes read. + after: Bytes written. + elapsed: Seconds the job took. + renamed: Where the output went instead, if its name was taken. + """ + percent: float = after / before * 100 if before else 0 + note: str = f", output renamed to {renamed}" if renamed else "" + print( + f"\n{name} {format_size(before)} → {format_size(after)} ({percent:.0f}%) " + f"in {format_time(elapsed)}{note}\n" + ) + + +def skipped(existing: Path) -> None: + """Print the line an input gets when `--on-conflict skip` left its output alone. + + Args: + existing: The output that was already there. + """ + print(f"{existing} skipped, already exists") + + def compress_plan(plan: CompressionPlan) -> None: """Print what a single-file compression is about to do. @@ -83,7 +112,6 @@ def archive_result(plan: ArchivePlan, written: Path, elapsed: float) -> None: written: The path actually written, which a rename may have moved. elapsed: Seconds the job took. """ - print() succeed("Archive created!\n") if written != plan.output: print(f" Renamed to: {written}") @@ -115,7 +143,6 @@ def extract_result(plan: ExtractPlan, elapsed: float) -> None: plan: The extraction's plan. elapsed: Seconds the job took. """ - print() succeed("Extraction successful!\n") print( f" Entries: {len(plan.entries)}\n Time: {format_time(seconds=elapsed)}\n" diff --git a/src/compresso/cli/compress.py b/src/compresso/cli/compress.py index 1d4ac1a..1787d82 100644 --- a/src/compresso/cli/compress.py +++ b/src/compresso/cli/compress.py @@ -12,50 +12,74 @@ from ..frontend.dispatch import CompressRequest, plan_compress from ._app import app from ._render import EXIT_USAGE, cancelled, fail -from ._render.jobs import archive_plan, archive_result, compress_plan, compress_result +from ._render.jobs import ( + archive_plan, + archive_result, + compress_plan, + compress_result, + skipped, + summary, +) from ._run import conflict_mode, run_all, run_job -def _compress_one_file(job: CompressionJob, quiet: bool) -> None: +def _compress_one_file(job: CompressionJob, quiet: bool, verbose: bool) -> None: """Compress a single file into a one-file container. Args: job: The planned job. - quiet: Whether to suppress progress output. + quiet: Whether to suppress all output. + verbose: Whether to print the plan and stats blocks. """ plan = job.plan if not plan.can_compress: fail(f"Error: {plan.reason_if_unavailable}", EXIT_USAGE) - if not quiet: + if verbose: compress_plan(plan) + skips: bool = plan.options.overwrite is OverwriteMode.SKIP and plan.dest.exists() + elapsed: float = run_job( job, plan.src, "Compressing", plan.input_size, quiet=quiet, what="Compression" ) # RENAME may have written somewhere other than `plan.dest`; job.plan is # replaced on success with the path actually written - if not quiet: - compress_result(plan, job.plan.dest, elapsed) + written: Path = job.plan.dest + if verbose: + compress_result(plan, written, elapsed) + + elif quiet: + return + + elif skips: + skipped(plan.dest) + else: + renamed: Path | None = written if written != plan.dest else None + summary(plan.dest, plan.input_size, written.stat().st_size, elapsed, renamed) -def _archive_many(job: ArchiveJob, quiet: bool) -> None: + +def _archive_many(job: ArchiveJob, quiet: bool, verbose: bool) -> None: """Pack several paths, or a directory, into one archive. Args: job: The planned job. - quiet: Whether to suppress progress output. + quiet: Whether to suppress all output. + verbose: Whether to print the plan and stats blocks. """ plan = job.plan if not plan.can_run: fail(f"Error: {plan.reason_if_unavailable}", EXIT_USAGE) - if not quiet: + if verbose: archive_plan(plan) + skips: bool = plan.options.overwrite is OverwriteMode.SKIP and plan.output.exists() + # A format with a codec stage reads the data twice, so the job reports a # larger total than this once it starts; the bar follows it elapsed: float = run_job( @@ -69,8 +93,25 @@ def _archive_many(job: ArchiveJob, quiet: bool) -> None: # RENAME may have written somewhere other than `plan.output`; job.plan is # replaced on success with the path actually written - if not quiet: - archive_result(plan, job.plan.output, elapsed) + written: Path = job.plan.output + if verbose: + archive_result(plan, written, elapsed) + + elif quiet: + return + + elif skips: + skipped(plan.output) + + else: + renamed: Path | None = written if written != plan.output else None + summary( + plan.output, + plan.total_input_size, + written.stat().st_size, + elapsed, + renamed, + ) def compress( @@ -137,6 +178,10 @@ def compress( help="Worker threads for zstd and xz, or 0 for one per core", ), ] = 0, + verbose: Annotated[ + bool, + app.Option("--verbose", "-v", help="Show the plan and full stats"), + ] = False, quiet: Annotated[ bool, app.Option("--quiet", "-q", help="Suppress all output") ] = False, @@ -154,6 +199,9 @@ def compress( (`name 2.ext` on macOS, `name (2).ext` elsewhere), for an archive and a single-file container alike; use `--on-conflict` or `-f` to change that. + Each output gets one summary line; `-v` shows the plan and full stats + instead, and `-q` nothing at all. + Args: inputs: The files and directories to compress. output: Where to write (default: named after the first input). @@ -164,9 +212,12 @@ def compress( on_conflict: What to do when the output exists (default: rename). force: If True, replace an existing output. threads: Worker threads for zstd and xz (default: one per core). + verbose: If True, show the plan and full stats. quiet: If True, suppress all output. """ overwrite_mode = conflict_mode(on_conflict, force) + if verbose and quiet: + fail("Error: -v contradicts -q", EXIT_USAGE) try: if not inputs: @@ -192,9 +243,9 @@ def compress( def run_one(job: CompressionJob | ArchiveJob) -> None: if isinstance(job, ArchiveJob): - _archive_many(job, quiet) + _archive_many(job, quiet, verbose) else: - _compress_one_file(job, quiet) + _compress_one_file(job, quiet, verbose) sys.exit(run_all([job], run_one, "Compression")) diff --git a/src/compresso/cli/extract.py b/src/compresso/cli/extract.py index dd7cb31..5ab9e9c 100644 --- a/src/compresso/cli/extract.py +++ b/src/compresso/cli/extract.py @@ -22,58 +22,62 @@ decompress_result, extract_plan, extract_result, + skipped, + summary, ) from ._run import InputFailed, conflict_mode, require_files, run_all, run_job -def _extract_archive(job: ExtractJob, quiet: bool) -> None: +def _extract_archive(job: ExtractJob, quiet: bool, verbose: bool) -> None: """Unpack one archive. Args: job: The planned job. - quiet: Whether to suppress output. + quiet: Whether to suppress all output. + verbose: Whether to print the plan and stats blocks. """ plan = job.plan if not plan.can_run: raise InputFailed(f"{plan.archive}: {plan.reason_if_unavailable}") - if not quiet: + if verbose: extract_plan(plan) # Progress covers the bytes written out; a compressed archive also reports # the pass that decompresses it, so the job's own total wins + total: int = sum(entry.size for entry in plan.entries) elapsed: float = run_job( - job, - plan.archive, - "Extracting", - sum(entry.size for entry in plan.entries), - quiet=quiet, - what="Extraction", + job, plan.archive, "Extracting", total, quiet=quiet, what="Extraction" ) - if not quiet: + if verbose: extract_result(plan, elapsed) + elif not quiet: + summary(plan.archive, plan.archive.stat().st_size, total, elapsed, None) + -def _decompress_one_file(job: DecompressionJob, quiet: bool) -> None: +def _decompress_one_file(job: DecompressionJob, quiet: bool, verbose: bool) -> None: """Unpack one single-file container: a `.comp` or a standalone codec. Args: job: The planned job. - quiet: Whether to suppress output. + quiet: Whether to suppress all output. + verbose: Whether to print the plan and stats blocks. """ plan = job.plan if not plan.can_run: raise InputFailed(f"{plan.src}: {plan.reason_if_unavailable}") - if not quiet: + if verbose: decompress_plan(plan) # Progress counts the compressed bytes read, so the source's size is the # denominator rather than the size it expands to compressed_size: int = plan.src.stat().st_size + skips: bool = plan.overwrite is OverwriteMode.SKIP and plan.dest.exists() elapsed: float = run_job( job, @@ -86,8 +90,19 @@ def _decompress_one_file(job: DecompressionJob, quiet: bool) -> None: # RENAME may have written somewhere other than `plan.dest`; job.plan is # replaced on success with the path actually written - if not quiet: - decompress_result(plan, job.plan.dest, compressed_size, elapsed) + written: Path = job.plan.dest + if verbose: + decompress_result(plan, written, compressed_size, elapsed) + + elif quiet: + return + + elif skips: + skipped(plan.dest) + + else: + renamed: Path | None = written if written != plan.dest else None + summary(plan.src, compressed_size, written.stat().st_size, elapsed, renamed) def extract( @@ -159,6 +174,14 @@ def extract( help="Name and date a .gz file's output as its header does", ), ] = False, + verbose: Annotated[ + bool, + app.Option( + "--verbose", + "-v", + help="Show the plan and full stats", + ), + ] = False, quiet: Annotated[ bool, app.Option("--quiet", "-q", help="Suppress all output") ] = False, @@ -177,6 +200,9 @@ def extract( (`name 2.ext` on macOS, `name (2).ext` elsewhere); use `--on-conflict` or `-f` to change that. + Each input gets one summary line; `-v` shows the plan and full stats + instead, and `-q` nothing at all. + Args: inputs: The archives or compressed files to unpack. output: Where to write (default: beside the input); a directory for @@ -192,9 +218,12 @@ def extract( threads: Worker threads for xz (default: one per core). name: If True, name a gzip file's output after the name its header stores, and give it the stored modification time, if any. + verbose: If True, show the plan and full stats. quiet: If True, suppress all output. """ mode = conflict_mode(on_conflict, force) + if verbose and quiet: + fail("Error: -v contradicts -q", EXIT_USAGE) if not inputs: fail("Error: No input files given", EXIT_USAGE) @@ -215,10 +244,10 @@ def extract( def run_one(job: ExtractJob | DecompressionJob) -> None: if isinstance(job, ExtractJob): - _extract_archive(job, quiet) + _extract_archive(job, quiet, verbose) else: - _decompress_one_file(job, quiet) + _decompress_one_file(job, quiet, verbose) try: sys.exit(run_all(plan_extract(request), run_one, "Extraction")) diff --git a/src/compresso/frontend/api.py b/src/compresso/frontend/api.py index 5dcdeb3..585adf0 100644 --- a/src/compresso/frontend/api.py +++ b/src/compresso/frontend/api.py @@ -343,6 +343,10 @@ def plan( try: detected: str = detect_format(src_path) + # The caller names the file with the error + except OSError as e: + return plan(None, False, e.strerror or str(e)) + except Exception as e: # noqa: BLE001 - surface as an unavailable plan return plan(None, False, f"Cannot read file: {e}") diff --git a/tests/python/cli/test_app.py b/tests/python/cli/test_app.py index ea122d1..0a4f623 100644 --- a/tests/python/cli/test_app.py +++ b/tests/python/cli/test_app.py @@ -2,7 +2,10 @@ from __future__ import annotations +import gzip import os +import re +import subprocess import sys from collections.abc import Callable from pathlib import Path @@ -179,6 +182,96 @@ def test_archive_round_trip_draws_both_bars( ) +@pytest.fixture(params=["compress", "archive", "decompress", "extract"]) +def job_args( + request: pytest.FixtureRequest, + payload: Path, + source_tree: Path, + tmp_path: Path, + make_archive: Callable[[str], Path], +) -> tuple[list[str | Path], Path]: + """One command line for each kind of job, and the path its summary names.""" + packed = tmp_path / "out.comp" + match request.param: + case "compress": + return ["compress", payload, "-o", packed], packed + + case "archive": + return ["compress", source_tree, "-o", tmp_path / "out.tar"], ( + tmp_path / "out.tar" + ) + + case "decompress": + invoke("compress", payload, "-o", packed, "-q") + return ["extract", packed, "-o", tmp_path / "restored.bin"], packed + + case _: + archive = make_archive("tar") + return ["extract", archive, "-o", tmp_path / "dest"], archive + + +class TestOutput: + """Test one summary line per input by default, the blocks with -v, and + nothing with -q.""" + + def test_default_is_one_summary_line( + self, job_args: tuple[list[str | Path], Path] + ) -> None: + """Test that a job prints `name before → after (n%) in time` and no more.""" + args, name = job_args + output = invoke(*args).output + + assert re.fullmatch( + rf"{re.escape(str(name))} \S+ \S+ → \S+ \S+ \(\d+%\) in \S+\n", output + ), output + + def test_verbose_prints_the_blocks( + self, job_args: tuple[list[str | Path], Path] + ) -> None: + """Test that -v brings back the plan before and the stats after.""" + args, _ = job_args + output = invoke(*args, "-v").output + + assert "Output" in output + assert "Time:" in output + + def test_quiet_prints_nothing( + self, job_args: tuple[list[str | Path], Path] + ) -> None: + """Test that -q prints nothing on success.""" + args, _ = job_args + assert invoke(*args, "-q").output == "" + + def test_verbose_with_quiet_is_usage( + self, job_args: tuple[list[str | Path], Path] + ) -> None: + """Test that -v and -q together are refused before any work starts.""" + args, _ = job_args + result = invoke(*args, "-v", "-q", expect=EXIT_USAGE) + assert "-v contradicts -q" in result.output + + def test_an_error_lands_after_the_inputs_before_it( + self, payload: Path, tmp_path: Path + ) -> None: + """Test that a failed input's error follows the earlier inputs' lines + when stdout is a pipe, which block-buffers it.""" + good = tmp_path / "good.gz" + good.write_bytes(gzip.compress(payload.read_bytes())) + bad = tmp_path / "bad.bin" + bad.write_bytes(payload.read_bytes()) + + result = subprocess.run( + [sys.executable, "-m", "compresso.cli", "extract", good, bad], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + check=False, + ) + + assert result.returncode == EXIT_FAILED + assert result.stdout.index(str(good)) < result.stdout.index(str(bad)) + + class TestExitCodes: """Test 0 success, 1 the operation failed, 2 the request could not be made.""" @@ -200,7 +293,8 @@ def test_bad_input_is_usage(self, payload: Path, command: str, name: str) -> Non sys.platform == "win32" or os.geteuid() == 0, reason="needs POSIX, not root" ) @pytest.mark.parametrize( - "command,shown", [("list", "f0.bin"), ("info", "Entries:")] + "command,shown", + [("list", "f0.bin"), ("info", "Entries:"), ("extract", "out.zip ")], ) def test_unreadable_input_fails_alone( self, diff --git a/tests/python/cli/test_compress.py b/tests/python/cli/test_compress.py index fe80063..824f83d 100644 --- a/tests/python/cli/test_compress.py +++ b/tests/python/cli/test_compress.py @@ -22,19 +22,11 @@ def test_compress_then_decompress(self, payload: Path, tmp_path: Path) -> None: archive = tmp_path / "out.comp" restored = tmp_path / "restored.bin" - result = invoke("compress", payload, "-o", archive) - assert "Compression successful" in result.output - - result = invoke("extract", archive, "-o", restored) - assert "Decompression successful" in result.output + invoke("compress", payload, "-o", archive, "-q") + invoke("extract", archive, "-o", restored, "-q") assert restored.read_bytes() == payload.read_bytes() - def test_quiet_suppresses_output(self, payload: Path, tmp_path: Path) -> None: - """Test that --quiet prints nothing on success.""" - result = invoke("compress", payload, "-o", tmp_path / "q.comp", "-q") - assert result.output.strip() == "" - # The backends are covered in core and C tests; this checks the flags @pytest.mark.parametrize( "args", [("-a", "bzip2"), ("-a", "zstd", "-l", "19")], ids=["algo", "level"] @@ -64,14 +56,20 @@ def test_existing_output( kept: bool, renames: bool, ) -> None: - """Test that each overwrite mode keeps, replaces or renames around the output.""" + """Test that each overwrite mode keeps, replaces or renames around the + output, and the summary line says which.""" t = compress_target t.dest.write_bytes(b"stale") - invoke("compress", t.source, "-o", t.dest, *t.args, *flags, "-q", expect=expect) + result = invoke( + "compress", t.source, "-o", t.dest, *t.args, *flags, expect=expect + ) + sibling = t.dest.parent / renamed(t.dest.name) assert (t.dest.read_bytes() == b"stale") == kept - assert (t.dest.parent / renamed(t.dest.name)).is_file() == renames + assert sibling.is_file() == renames + assert (f"output renamed to {sibling}" in result.output) == renames + assert (f"{t.dest} skipped" in result.output) == ("skip" in flags) class TestSmartCompress: diff --git a/tests/python/cli/test_extract.py b/tests/python/cli/test_extract.py index 8f5bf98..4213a26 100644 --- a/tests/python/cli/test_extract.py +++ b/tests/python/cli/test_extract.py @@ -30,11 +30,8 @@ def test_archive_then_extract( archive = tmp_path / f"out.{fmt}" dest = tmp_path / f"dest.{fmt}" - result = invoke("compress", source_tree, "-o", archive, "-F", fmt) - assert "Archive created" in result.output - - result = invoke("extract", archive, "-o", dest) - assert "Extraction successful" in result.output + invoke("compress", source_tree, "-o", archive, "-F", fmt, "-q") + invoke("extract", archive, "-o", dest, "-q") for original in source_tree.iterdir(): restored = dest / source_tree.name / original.name @@ -117,7 +114,8 @@ def test_existing_output( kept: bool, renames: bool, ) -> None: - """Test that each overwrite mode keeps, replaces or renames around the output.""" + """Test that each overwrite mode keeps, replaces or renames around the + output, and the summary line says which.""" archive = tmp_path / "out.comp" restored = tmp_path / "restored.bin" invoke("compress", payload, "-o", archive, "-q") @@ -126,7 +124,8 @@ def test_existing_output( result = invoke("extract", archive, "-o", restored, *flags, expect=expect) sibling = tmp_path / renamed(restored.name) - assert (f"Renamed to: {sibling}" in result.output) == renames + assert (f"output renamed to {sibling}" in result.output) == renames + assert (f"{restored} skipped" in result.output) == ("skip" in flags) assert restored.read_bytes() == (b"stale" if kept else payload.read_bytes()) assert sibling.exists() == renames From b13fe6db06ac20ab250563925cfb6c01612b36eb Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Mon, 26 Oct 2026 23:45:34 +0000 Subject: [PATCH 14/31] fix(cli): refuse --algo and --strategy for formats other than '.comp' - Both flags on formats other than '.comp' now makes an unrunnable plan, so the command exits 2 with nothing written - '-a' with '-s' is a now a usage error, as they are mutually exclusive - 'strategy' now defaults to None, which lets the CLI tell a given strategy from an unspecified one - The help panel still shows 'balanced' through 'show_default', and the planner applies it - 'auto' now folds into None where the other normalising happens, rather than inside the request construction --- src/compresso/cli/compress.py | 22 ++++++++++----- src/compresso/frontend/dispatch.py | 31 ++++++++++++++++----- tests/python/cli/test_app.py | 2 +- tests/python/cli/test_compress.py | 43 ++++++++++++++++++++++++++++-- 4 files changed, 83 insertions(+), 15 deletions(-) diff --git a/src/compresso/cli/compress.py b/src/compresso/cli/compress.py index 1787d82..8104521 100644 --- a/src/compresso/cli/compress.py +++ b/src/compresso/cli/compress.py @@ -140,14 +140,15 @@ def compress( ), ] = "auto", strategy: Annotated[ - str, + str | None, app.Option( "--strategy", "-s", case_sensitive=False, help="Compression strategy to use (fast/balanced/max_ratio)", + show_default="balanced", ), - ] = "balanced", + ] = None, level: Annotated[ int | None, app.Option( @@ -206,8 +207,10 @@ def compress( inputs: The files and directories to compress. output: Where to write (default: named after the first input). format: Container to write (default: inferred, else .comp or zip). - algo: Algorithm for the Compresso container (default: auto). - strategy: Strategy for the Compresso container (default: balanced). + algo: Algorithm for the Compresso container (default: auto); any + other format refuses it. + strategy: Strategy for the Compresso container (default: balanced); + any other format refuses it, as does `-a`. level: Compression level (default: the format's own). on_conflict: What to do when the output exists (default: rename). force: If True, replace an existing output. @@ -224,12 +227,19 @@ def compress( fail("Error: No input paths given", EXIT_USAGE) algo_lower: str | None = algo.lower() if algo else None + if algo_lower == "auto": + algo_lower = None + + # -a names the algorithm that -s would otherwise pick + if algo_lower and strategy: + fail("Error: -a contradicts -s", EXIT_USAGE) + request = CompressRequest( inputs=inputs, output=output, format=format, - algo=None if algo_lower == "auto" else algo_lower, - strategy=strategy.lower(), + algo=algo_lower, + strategy=strategy.lower() if strategy else None, level=level, overwrite=overwrite_mode, threads=threads, diff --git a/src/compresso/frontend/dispatch.py b/src/compresso/frontend/dispatch.py index e750c75..c0e6eef 100644 --- a/src/compresso/frontend/dispatch.py +++ b/src/compresso/frontend/dispatch.py @@ -2,7 +2,7 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from functools import partial from pathlib import Path @@ -110,7 +110,7 @@ class CompressRequest: output: The `-o` value, or None to name the output after the first input. format: The container asked for, or None to infer it. algo: The `.comp` algorithm, or None for the strategy's default. - strategy: The `.comp` strategy. + strategy: The `.comp` strategy, or None for the default. level: The compression level, or None for the format's own. overwrite: What to do when the output already exists. threads: Worker threads, or 0 for one per core. @@ -120,7 +120,7 @@ class CompressRequest: output: Path | None = None format: str | None = None algo: str | None = None - strategy: str = "balanced" + strategy: str | None = None level: int | None = None overwrite: OverwriteMode = OverwriteMode.RENAME threads: int = 0 @@ -155,7 +155,8 @@ def plan_compress(request: CompressRequest) -> CompressionJob | ArchiveJob: request: The parsed request. Returns: - The planned job, which may still be unable to run (see its plan). + The planned job, which may still be unable to run (see its plan), + as when `algo` or `strategy` is given for anything but a `.comp`. Raises: ValueError: If a single-file format was asked for with several inputs @@ -164,6 +165,9 @@ def plan_compress(request: CompressRequest) -> CompressionJob | ArchiveJob: fmt: str | None = request.format or infer_format(request.output) many: bool = len(request.inputs) > 1 or any(p.is_dir() for p in request.inputs) + # Only the .comp container records its codec; the others fix their own + comp_flag: str | None = "-a" if request.algo else "-s" if request.strategy else None + if format_is_single_file(fmt) and many: raise ValueError( f"{fmt} holds a single file; " @@ -180,14 +184,22 @@ def plan_compress(request: CompressRequest) -> CompressionJob | ArchiveJob: threads=request.threads, ) - return ArchiveJob.from_paths( + archive_job = ArchiveJob.from_paths( sources=request.inputs, output=output, options=archive_options ) + if comp_flag: + archive_job.plan = replace( + archive_job.plan, + can_run=False, + reason_if_unavailable=f"{comp_flag} applies only to .comp, not {fmt}", + ) + + return archive_job source: Path = request.inputs[0] options = CompressionOptions( algo=request.algo, - strategy=request.strategy, + strategy=request.strategy or CompressionOptions.strategy, level=request.level, format=fmt, overwrite=request.overwrite, @@ -200,6 +212,13 @@ def plan_compress(request: CompressRequest) -> CompressionJob | ArchiveJob: if dest != job.plan.dest: job = CompressionJob.from_file(src=source, dest=dest, options=options) + if comp_flag and format_is_single_file(fmt): + job.plan = replace( + job.plan, + can_compress=False, + reason_if_unavailable=f"{comp_flag} applies only to .comp, not {fmt}", + ) + return job diff --git a/tests/python/cli/test_app.py b/tests/python/cli/test_app.py index 0a4f623..1d18c57 100644 --- a/tests/python/cli/test_app.py +++ b/tests/python/cli/test_app.py @@ -222,7 +222,7 @@ def test_default_is_one_summary_line( output = invoke(*args).output assert re.fullmatch( - rf"{re.escape(str(name))} \S+ \S+ → \S+ \S+ \(\d+%\) in \S+\n", output + rf"\n{re.escape(str(name))} \S+ \S+ → \S+ \S+ \(\d+%\) in \S+\n\n", output ), output def test_verbose_prints_the_blocks( diff --git a/tests/python/cli/test_compress.py b/tests/python/cli/test_compress.py index 824f83d..d35635b 100644 --- a/tests/python/cli/test_compress.py +++ b/tests/python/cli/test_compress.py @@ -29,12 +29,14 @@ def test_compress_then_decompress(self, payload: Path, tmp_path: Path) -> None: # The backends are covered in core and C tests; this checks the flags @pytest.mark.parametrize( - "args", [("-a", "bzip2"), ("-a", "zstd", "-l", "19")], ids=["algo", "level"] + "args", + [("-a", "bzip2"), ("-a", "zstd", "-l", "19"), ("-s", "fast")], + ids=["algo", "level", "strategy"], ) def test_algorithms_round_trip( self, payload: Path, tmp_path: Path, args: tuple[str, ...] ) -> None: - """Test that -a and -l reach the backend.""" + """Test that -a, -l and -s reach the backend.""" archive = tmp_path / "out.comp" restored = tmp_path / "restored.bin" @@ -44,6 +46,43 @@ def test_algorithms_round_trip( assert restored.read_bytes() == payload.read_bytes() +class TestCompOnlyFlags: + """Test that -a and -s, which pick the .comp codec, are refused elsewhere.""" + + def test_algo_with_strategy(self, payload: Path) -> None: + """Test that -a and -s together are a usage error.""" + result = invoke( + "compress", payload, "-a", "zstd", "-s", "fast", expect=EXIT_USAGE + ) + + assert "-a contradicts -s" in result.output + + @pytest.mark.parametrize( + ("args", "message"), + [ + (("-a", "zstd", "-F", "gz"), "-a applies only to .comp, not gz"), + (("-s", "fast", "-o", "x.zip"), "-s applies only to .comp, not zip"), + ], + ids=["algo-standalone", "strategy-archive"], + ) + def test_refused_outside_comp( + self, + payload: Path, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + args: tuple[str, ...], + message: str, + ) -> None: + """Test that a format with its own codec refuses -a and -s, writing nothing.""" + monkeypatch.chdir(tmp_path) + before = set(tmp_path.iterdir()) + + result = invoke("compress", payload, *args, expect=EXIT_USAGE) + + assert message in result.output + assert set(tmp_path.iterdir()) == before + + class TestCompressConflicts: """Test what `compress` does when its output already exists.""" From 5d4a4c6b936ff3ba20e71a2c797a2ecbc21949de Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Tue, 27 Oct 2026 00:46:41 +0000 Subject: [PATCH 15/31] feat(cli): split extract's output into -C and -o - New '-C/--directory' flag is now where to write, for any input - The '-o' flag is now the one file to write, for a single input that isn't an archive - Drops the now unused resolve_single_output() - Mutually exclusive combinations are refused: '-o' with '-C', '-o' with several inputs, and '-o' on an archive - 'compress -o' now takes the path as given, rather than treating an existing directory as the output directory - '-a' stops showing 'auto' as its default --- src/compresso/cli/compress.py | 6 ++- src/compresso/cli/extract.py | 30 +++++++++---- src/compresso/frontend/dispatch.py | 54 +++++++++--------------- tests/python/cli/test_app.py | 6 +-- tests/python/cli/test_compress.py | 2 +- tests/python/cli/test_extract.py | 49 ++++++++++++++++++---- tests/python/frontend/test_dispatch.py | 58 ++++++++------------------ 7 files changed, 111 insertions(+), 94 deletions(-) diff --git a/src/compresso/cli/compress.py b/src/compresso/cli/compress.py index 8104521..98e663e 100644 --- a/src/compresso/cli/compress.py +++ b/src/compresso/cli/compress.py @@ -136,7 +136,11 @@ def compress( algo: Annotated[ str | None, app.Option( - "--algo", "-a", case_sensitive=False, help="Compression algorithm to use" + "--algo", + "-a", + case_sensitive=False, + help="Compression algorithm to use", + show_default=False, ), ] = "auto", strategy: Annotated[ diff --git a/src/compresso/cli/extract.py b/src/compresso/cli/extract.py index 5ab9e9c..337155e 100644 --- a/src/compresso/cli/extract.py +++ b/src/compresso/cli/extract.py @@ -110,13 +110,20 @@ def extract( list[Path], app.Argument(help="Archives or compressed files", readable=False), ], + directory: Annotated[ + Path | None, + app.Option( + "--directory", + "-C", + help="Directory to write into (default: beside each input)", + ), + ] = None, output: Annotated[ Path | None, app.Option( - "--output-dir", "--output", "-o", - help="Where to write (default: beside the input)", + help="File to write, for one compressed file that isn't an archive", ), ] = None, on_conflict: Annotated[ @@ -194,7 +201,7 @@ def extract( Compresso `.comp`, or a standalone `.gz`/`.bz2`/`.xz`/`.zst`/`.lz4`/`.sz`, is decompressed to one file. - The result is written beside its input unless `-o` is provided. + The result is written beside its input unless `-C` or `-o` is provided. By default, a clashing name gets a platform-native numbered sibling (`name 2.ext` on macOS, `name (2).ext` elsewhere); use `--on-conflict` or @@ -205,9 +212,8 @@ def extract( Args: inputs: The archives or compressed files to unpack. - output: Where to write (default: beside the input); a directory for - archives, and for one file either a directory to put it in or the - path to write. + directory: The directory to write into (default: beside each input). + output: The file to write, for a single input that isn't an archive. on_conflict: What to do when a file exists (default: rename). force: If True, replace files that already exist. max_total_size: Cap on total extracted bytes (default: no cap). @@ -228,6 +234,12 @@ def extract( if not inputs: fail("Error: No input files given", EXIT_USAGE) + if output and directory: + fail("Error: -o contradicts -C", EXIT_USAGE) + + if output and len(inputs) > 1: + fail("Error: -o names one file; use -C for several inputs", EXIT_USAGE) + require_files(inputs) options = ExtractOptions( @@ -239,7 +251,11 @@ def extract( ) request = ExtractRequest( - inputs=inputs, output=output, options=options, restore_name=name + inputs=inputs, + directory=directory, + output=output, + options=options, + restore_name=name, ) def run_one(job: ExtractJob | DecompressionJob) -> None: diff --git a/src/compresso/frontend/dispatch.py b/src/compresso/frontend/dispatch.py index c0e6eef..cf707fa 100644 --- a/src/compresso/frontend/dispatch.py +++ b/src/compresso/frontend/dispatch.py @@ -67,28 +67,6 @@ def default_archive_output(first_input: Path, fmt: str) -> Path: return first_input.resolve().parent / f"{stem}.{fmt}" -def resolve_single_output(output: Path | None, default: Path) -> Path: - """Work out where a single-file job should write. - - `-o` means a directory for an archive, but for one file it is the file itself; - an existing directory receives the file, anything else is the file. - - Args: - output: The `-o` value, or None. - default: Where the job would write if not told otherwise. - - Returns: - The destination path. - """ - if output is None: - return default - - if output.is_dir(): - return output / default.name - - return output - - def format_is_single_file(fmt: str | None) -> bool: """Return whether `fmt` names a container holding exactly one file. @@ -132,7 +110,8 @@ class ExtractRequest: Attributes: inputs: The archives and single-file containers to unpack. - output: The `-o` value, or None to write beside each input. + directory: The `-C` value, or None to write beside each input. + output: The `-o` file, for a single non-archive input. options: The extraction policy; single-file inputs use only its `overwrite` and `threads`. restore_name: Whether to name and date a gzip file's output as its @@ -140,6 +119,7 @@ class ExtractRequest: """ inputs: list[Path] + directory: Path | None = None output: Path | None = None options: ExtractOptions = field(default_factory=ExtractOptions) restore_name: bool = False @@ -206,11 +186,7 @@ def plan_compress(request: CompressRequest) -> CompressionJob | ArchiveJob: threads=request.threads, ) - job = CompressionJob.from_file(src=source, dest=None, options=options) - # Planned once to learn the default name, then repointed if `-o` moves it - dest: Path = resolve_single_output(request.output, job.plan.dest) - if dest != job.plan.dest: - job = CompressionJob.from_file(src=source, dest=dest, options=options) + job = CompressionJob.from_file(src=source, dest=request.output, options=options) if comp_flag and format_is_single_file(fmt): job.plan = replace( @@ -237,11 +213,17 @@ def plan_extract(request: ExtractRequest) -> list[ExtractJob | DecompressionJob] jobs: list[ExtractJob | DecompressionJob] = [] for source in request.inputs: if looks_like_archive(source): - jobs.append( - ExtractJob.from_archive( - archive=source, output_dir=request.output, options=request.options - ) + archive_job = ExtractJob.from_archive( + archive=source, output_dir=request.directory, options=request.options ) + if request.output: + archive_job.plan = replace( + archive_job.plan, + can_run=False, + reason_if_unavailable="-o names one file; use -C for an archive", + ) + + jobs.append(archive_job) continue plan = partial( @@ -251,8 +233,12 @@ def plan_extract(request: ExtractRequest) -> list[ExtractJob | DecompressionJob] threads=request.options.threads, restore_name=request.restore_name, ) + if request.output or not request.directory: + jobs.append(plan(dest=request.output)) + continue + + # Planned once to learn the default name, then moved into `-C` job: DecompressionJob = plan(dest=None) - dest: Path = resolve_single_output(request.output, job.plan.dest) - jobs.append(job if dest == job.plan.dest else plan(dest=dest)) + jobs.append(plan(dest=request.directory / job.plan.dest.name)) return jobs diff --git a/tests/python/cli/test_app.py b/tests/python/cli/test_app.py index 1d18c57..97a1eb8 100644 --- a/tests/python/cli/test_app.py +++ b/tests/python/cli/test_app.py @@ -172,7 +172,7 @@ def test_archive_round_trip_draws_both_bars( dest = tmp_path / "dest" invoke("compress", tree, "-o", archive, "-F", "tar") - invoke("extract", archive, "-o", dest) + invoke("extract", archive, "-C", dest) assert [bar.enabled for bar in bars] == [True, True] assert all(bar.updates > 0 for bar in bars) @@ -207,7 +207,7 @@ def job_args( case _: archive = make_archive("tar") - return ["extract", archive, "-o", tmp_path / "dest"], archive + return ["extract", archive, "-C", tmp_path / "dest"], archive class TestOutput: @@ -328,7 +328,7 @@ def test_force_with_another_conflict_mode_is_usage( if command == "compress": args = [source_tree, "-o", tmp_path / "out.tar", "-F", "tar"] else: - args = [make_archive("tar"), "-o", tmp_path / "d"] + args = [make_archive("tar"), "-C", tmp_path / "d"] result = invoke(command, *args, "-f", "--on-conflict", mode, expect=EXIT_USAGE) assert f"-f contradicts --on-conflict {mode}" in result.output diff --git a/tests/python/cli/test_compress.py b/tests/python/cli/test_compress.py index d35635b..256c7c5 100644 --- a/tests/python/cli/test_compress.py +++ b/tests/python/cli/test_compress.py @@ -124,7 +124,7 @@ def test_a_directory_round_trips_as_an_archive( invoke( "extract", source_tree.with_name(f"{source_tree.name}.zip"), - "-o", + "-C", dest, "-q", ) diff --git a/tests/python/cli/test_extract.py b/tests/python/cli/test_extract.py index 4213a26..48fab3c 100644 --- a/tests/python/cli/test_extract.py +++ b/tests/python/cli/test_extract.py @@ -13,6 +13,7 @@ from compresso.cli._render import ( EXIT_FAILED, EXIT_OK, + EXIT_USAGE, ) from ..helpers import file_entry, link_entry, make_tar, renamed, umask @@ -31,7 +32,7 @@ def test_archive_then_extract( dest = tmp_path / f"dest.{fmt}" invoke("compress", source_tree, "-o", archive, "-F", fmt, "-q") - invoke("extract", archive, "-o", dest, "-q") + invoke("extract", archive, "-C", dest, "-q") for original in source_tree.iterdir(): restored = dest / source_tree.name / original.name @@ -61,9 +62,9 @@ def test_extract_over_an_existing_tree( by default, and follows the overwrite mode otherwise.""" archive = make_archive("tar") dest = tmp_path / "dest" - invoke("extract", archive, "-o", dest, "-q") - invoke("extract", archive, "-o", dest, *flags, "-q", expect=expect) + invoke("extract", archive, "-C", dest, "-q") + invoke("extract", archive, "-C", dest, *flags, "-q", expect=expect) top_level = [source_tree.name, *([renamed(source_tree.name)] * renames)] assert sorted(p.name for p in dest.iterdir()) == sorted(top_level) @@ -84,7 +85,7 @@ def test_same_permissions_skips_the_umask( make_tar(archive, file_entry("a.txt", mode=0o777)) dest = tmp_path / "dest" - invoke("extract", archive, "-o", dest, *(["-p"] if exact else []), "-q") + invoke("extract", archive, "-C", dest, *(["-p"] if exact else []), "-q") expected = 0o777 if exact else 0o777 & ~umask() assert (dest / "a.txt").stat().st_mode & 0o7777 == expected @@ -95,8 +96,8 @@ def test_symlinks_flag_reaches_the_policy(self, tmp_path: Path) -> None: make_tar(archive, link_entry("link", "target", kind=tarfile.SYMTYPE)) dest = tmp_path / "dest" - invoke("extract", archive, "-o", dest, "-q", expect=EXIT_FAILED) - invoke("extract", archive, "-o", dest, "--symlinks", "file", "-q") + invoke("extract", archive, "-C", dest, "-q", expect=EXIT_FAILED) + invoke("extract", archive, "-C", dest, "--symlinks", "file", "-q") assert (dest / "link").read_bytes() == b"target" @@ -195,7 +196,7 @@ def test_a_failed_input_does_not_stop_the_rest( good, broken, make_archive("zip"), - "-o", + "-C", dest, "-q", expect=EXIT_FAILED, @@ -207,3 +208,37 @@ def test_a_failed_input_does_not_stop_the_rest( for original in source_tree.iterdir(): restored = dest / source_tree.name / original.name assert restored.read_bytes() == original.read_bytes() + + +class TestOutputFlags: + """Test that -o names one file and -C the directory for everything else.""" + + def test_output_with_directory(self, payload: Path, tmp_path: Path) -> None: + """Test that -o contradicts -C.""" + packed = tmp_path / "a.gz" + packed.write_bytes(gzip.compress(payload.read_bytes())) + + result = invoke( + "extract", packed, "-o", tmp_path / "x", "-C", tmp_path, expect=EXIT_USAGE + ) + assert "-o contradicts -C" in result.output + + def test_output_with_several_inputs(self, payload: Path, tmp_path: Path) -> None: + """Test that -o refuses more than one input, pointing to -C.""" + inputs = [tmp_path / "a.gz", tmp_path / "b.gz"] + for packed in inputs: + packed.write_bytes(gzip.compress(payload.read_bytes())) + + result = invoke("extract", *inputs, "-o", tmp_path / "x", expect=EXIT_USAGE) + assert "use -C" in result.output + + def test_output_on_an_archive( + self, tmp_path: Path, make_archive: Callable[[str], Path] + ) -> None: + """Test that -o on an input that turns out to be an archive fails it.""" + archive = make_archive("zip") + out = tmp_path / "out" + + result = invoke("extract", archive, "-o", out, expect=EXIT_FAILED) + assert f"{archive}: -o names one file; use -C for an archive" in result.output + assert not out.exists() diff --git a/tests/python/frontend/test_dispatch.py b/tests/python/frontend/test_dispatch.py index 15a00c7..3dca039 100644 --- a/tests/python/frontend/test_dispatch.py +++ b/tests/python/frontend/test_dispatch.py @@ -19,7 +19,6 @@ looks_like_archive, plan_compress, plan_extract, - resolve_single_output, ) @@ -130,29 +129,6 @@ def test_trailing_separator( ) -class TestResolveSingleOutput: - """Test where `-o` sends a single-file job.""" - - def test_no_output_keeps_default(self, tmp_path: Path) -> None: - """Test that without `-o` the default path is used.""" - default = tmp_path / "x.gz" - - assert resolve_single_output(None, default) == default - - def test_existing_directory_receives_the_file(self, tmp_path: Path) -> None: - """Test that an existing directory gets the default filename inside it.""" - out = tmp_path / "out" - out.mkdir() - - assert resolve_single_output(out, tmp_path / "x.gz") == out / "x.gz" - - def test_anything_else_is_the_file(self, tmp_path: Path) -> None: - """Test that any other path is taken as the output file itself.""" - out = tmp_path / "out" / "y.gz" - - assert resolve_single_output(out, tmp_path / "x.gz") == out - - class TestPlanCompress: """Test `compress` picks the job from its inputs and the format asked for.""" @@ -241,18 +217,6 @@ def test_tar_inside_a_codec_is_an_archive( assert job.plan.can_run assert job.plan.options.format == name.removeprefix("out.") - def test_existing_directory_receives_the_file( - self, small_file: Path, tmp_path: Path - ) -> None: - """Test that an existing `-o` directory gets the default filename.""" - into = tmp_path / "into" - into.mkdir() - - job = plan_compress(CompressRequest([small_file], output=into, format="gz")) - - assert isinstance(job, CompressionJob) - assert job.plan.dest == into / "small.txt.gz" - def test_single_file_format_refuses_several_inputs(self, tree: Path) -> None: """Test that a single-file format on a directory errors, suggesting tar.""" with pytest.raises(ValueError, match="tar.gz"): @@ -290,14 +254,14 @@ def test_an_archive_named_like_anything_still_extracts( zf.writestr("a.txt", "a") dest = tmp_path / "out" - [job] = plan_extract(ExtractRequest([disguised], output=dest)) + [job] = plan_extract(ExtractRequest([disguised], directory=dest)) assert isinstance(job, ExtractJob) assert job.plan.can_run assert job.plan.output_dir == dest def test_an_archive_unpacks_beside_itself(self, tmp_path: Path) -> None: - """Test that without `-o` an archive extracts into its own directory.""" + """Test that without `-C` an archive extracts into its own directory.""" archive = tmp_path / "x.zip" with zipfile.ZipFile(archive, "w") as zf: zf.writestr("a.txt", "a") @@ -307,18 +271,30 @@ def test_an_archive_unpacks_beside_itself(self, tmp_path: Path) -> None: assert isinstance(job, ExtractJob) assert job.plan.output_dir == tmp_path - def test_output_directory_receives_a_single_file(self, tmp_path: Path) -> None: - """Test that an existing `-o` directory gets the suffix-stripped name.""" + def test_directory_receives_a_single_file(self, tmp_path: Path) -> None: + """Test that `-C` gets the suffix-stripped name.""" packed = tmp_path / "p.gz" packed.write_bytes(gzip.compress(b"a")) into = tmp_path / "into" into.mkdir() - [job] = plan_extract(ExtractRequest([packed], output=into)) + [job] = plan_extract(ExtractRequest([packed], directory=into)) assert isinstance(job, DecompressionJob) assert job.plan.dest == into / "p" + def test_output_refuses_an_archive(self, tmp_path: Path) -> None: + """Test that `-o` on an archive plans a job that cannot run.""" + archive = tmp_path / "x.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr("a.txt", "a") + + [job] = plan_extract(ExtractRequest([archive], output=tmp_path / "out")) + + assert isinstance(job, ExtractJob) + assert not job.plan.can_run + assert "use -C" in str(job.plan.reason_if_unavailable) + def test_each_input_is_dispatched_on_its_own(self, tmp_path: Path) -> None: """Test that a mixed list plans one job per input, in order.""" packed = tmp_path / "a.gz" From 3d7b5e62629e08f5f289fe8411b39f6a3a100c3d Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Tue, 27 Oct 2026 18:32:47 +0000 Subject: [PATCH 16/31] feat(cli): add --member flag to extract only named entries - New '--member/-m' flag is repeatable and takes an entry path, selecting that entry or a directory and everything below it - plan_extraction() filters the entries, so the plan, progress total, and listing count all describe what is actually written - A name matching nothing fails that archive after the rest is extracted - '-m' on a non-archive input fails, since a single-file container has no entries to name --- src/compresso/cli/extract.py | 13 +++++ src/compresso/frontend/archive_api.py | 11 +++- src/compresso/frontend/dispatch.py | 26 ++++++--- tests/python/cli/test_extract.py | 66 +++++++++++++++++++++++ tests/python/frontend/test_archive_api.py | 17 ++++++ 5 files changed, 126 insertions(+), 7 deletions(-) diff --git a/src/compresso/cli/extract.py b/src/compresso/cli/extract.py index 337155e..4d75536 100644 --- a/src/compresso/cli/extract.py +++ b/src/compresso/cli/extract.py @@ -126,6 +126,15 @@ def extract( help="File to write, for one compressed file that isn't an archive", ), ] = None, + member: Annotated[ + list[str] | None, + app.Option( + "--member", + "-m", + help="Extract only this entry, or the directory and all below it; " + "repeatable", + ), + ] = None, on_conflict: Annotated[ OverwriteMode | None, app.Option( @@ -214,6 +223,9 @@ def extract( inputs: The archives or compressed files to unpack. directory: The directory to write into (default: beside each input). output: The file to write, for a single input that isn't an archive. + member: Entry paths to extract from each archive (default: all); a + path matching no entry fails that archive after the rest is + extracted. on_conflict: What to do when a file exists (default: rename). force: If True, replace files that already exist. max_total_size: Cap on total extracted bytes (default: no cap). @@ -254,6 +266,7 @@ def extract( inputs=inputs, directory=directory, output=output, + files=member, options=options, restore_name=name, ) diff --git a/src/compresso/frontend/archive_api.py b/src/compresso/frontend/archive_api.py index 62a0386..24613ba 100644 --- a/src/compresso/frontend/archive_api.py +++ b/src/compresso/frontend/archive_api.py @@ -190,7 +190,7 @@ class ExtractPlan: output_dir: Directory to extract into. files: Optional subset of entry paths to extract (None = all). options: Extraction policy to apply. - entries: Entries discovered in the archive. + entries: The archive's entries that `files` selects. can_run: Whether the extract operation can proceed. reason_if_unavailable: Reason the operation cannot proceed, if any. """ @@ -416,6 +416,15 @@ def plan_extraction( reason_if_unavailable=f"Cannot read archive: {e}", ) + # As the extractor matches: a name selects that entry and anything below it + if files: + names: set[str] = {name.rstrip("/") for name in files} - {""} + entries = [ + e + for e in entries + if any(e.path == n or e.path.startswith(f"{n}/") for n in names) + ] + return ExtractPlan( archive=archive_path, output_dir=out_dir, diff --git a/src/compresso/frontend/dispatch.py b/src/compresso/frontend/dispatch.py index cf707fa..e3fdb1e 100644 --- a/src/compresso/frontend/dispatch.py +++ b/src/compresso/frontend/dispatch.py @@ -112,6 +112,8 @@ class ExtractRequest: inputs: The archives and single-file containers to unpack. directory: The `-C` value, or None to write beside each input. output: The `-o` file, for a single non-archive input. + files: The `-m` entry paths to extract from each archive, or + None for all. options: The extraction policy; single-file inputs use only its `overwrite` and `threads`. restore_name: Whether to name and date a gzip file's output as its @@ -121,6 +123,7 @@ class ExtractRequest: inputs: list[Path] directory: Path | None = None output: Path | None = None + files: list[str] | None = None options: ExtractOptions = field(default_factory=ExtractOptions) restore_name: bool = False @@ -214,7 +217,10 @@ def plan_extract(request: ExtractRequest) -> list[ExtractJob | DecompressionJob] for source in request.inputs: if looks_like_archive(source): archive_job = ExtractJob.from_archive( - archive=source, output_dir=request.directory, options=request.options + archive=source, + output_dir=request.directory, + files=request.files, + options=request.options, ) if request.output: archive_job.plan = replace( @@ -234,11 +240,19 @@ def plan_extract(request: ExtractRequest) -> list[ExtractJob | DecompressionJob] restore_name=request.restore_name, ) if request.output or not request.directory: - jobs.append(plan(dest=request.output)) - continue + job: DecompressionJob = plan(dest=request.output) + + else: + # Planned once to learn the default name, then moved into `-C` + job = plan(dest=request.directory / plan(dest=None).plan.dest.name) + + if request.files: + job.plan = replace( + job.plan, + can_run=False, + reason_if_unavailable="--member/-m applies only to archives", + ) - # Planned once to learn the default name, then moved into `-C` - job: DecompressionJob = plan(dest=None) - jobs.append(plan(dest=request.directory / job.plan.dest.name)) + jobs.append(job) return jobs diff --git a/tests/python/cli/test_extract.py b/tests/python/cli/test_extract.py index 48fab3c..94a41a3 100644 --- a/tests/python/cli/test_extract.py +++ b/tests/python/cli/test_extract.py @@ -242,3 +242,69 @@ def test_output_on_an_archive( result = invoke("extract", archive, "-o", out, expect=EXIT_FAILED) assert f"{archive}: -o names one file; use -C for an archive" in result.output assert not out.exists() + + +class TestMember: + """Test that --member/-m extracts only the entries named.""" + + @pytest.mark.parametrize("fmt", ["tar", "zip"]) + def test_only_the_chosen_entries( + self, + source_tree: Path, + tmp_path: Path, + make_archive: Callable[[str], Path], + fmt: str, + ) -> None: + """Test that the named entries are written and nothing else.""" + archive = make_archive(fmt) + dest = tmp_path / "dest" + top = source_tree.name + + invoke( + "extract", + archive, + "-C", + dest, + "--member", + f"{top}/f0.bin", + "-m", + f"{top}/f2.bin", + "-q", + ) + + assert sorted(p.name for p in (dest / top).iterdir()) == ["f0.bin", "f2.bin"] + for name in ("f0.bin", "f2.bin"): + assert (dest / top / name).read_bytes() == (source_tree / name).read_bytes() + + def test_a_missing_member_fails_after_the_rest( + self, source_tree: Path, tmp_path: Path, make_archive: Callable[[str], Path] + ) -> None: + """Test that a name matching nothing exits 1, naming it, once the rest is + written.""" + dest = tmp_path / "dest" + top = source_tree.name + + result = invoke( + "extract", + make_archive("tar"), + "-C", + dest, + "--member", + f"{top}/f1.bin", + "-m", + "nope", + expect=EXIT_FAILED, + ) + + assert "nope" in result.output + assert sorted(p.name for p in (dest / top).iterdir()) == ["f1.bin"] + + def test_a_single_file_input_fails(self, payload: Path, tmp_path: Path) -> None: + """Test that --member/-m on an input that isn't an archive fails it.""" + packed = tmp_path / "a.gz" + packed.write_bytes(gzip.compress(payload.read_bytes())) + + result = invoke("extract", packed, "-m", "x", expect=EXIT_FAILED) + + assert f"{packed}: --member/-m applies only to archives" in result.output + assert not (tmp_path / "a").exists() diff --git a/tests/python/frontend/test_archive_api.py b/tests/python/frontend/test_archive_api.py index 18aa937..3a87a41 100644 --- a/tests/python/frontend/test_archive_api.py +++ b/tests/python/frontend/test_archive_api.py @@ -262,6 +262,23 @@ def test_output_dir_default_ignores_a_relative_archive_path( assert plan.output_dir == tmp_path.resolve() + def test_files_select_the_planned_entries(self, tmp_path: Path) -> None: + """Test that the plan holds only what `files` selects, as the extractor + matches it: an entry, or a directory and everything below it.""" + archive = tmp_path / "a.tar" + make_tar( + archive, + dir_entry("d/"), + file_entry("d/x"), + file_entry("dx"), + file_entry("y"), + file_entry("z"), + ) + + plan = ExtractJob.from_archive(archive, files=["d/", "y", "nope"]).plan + + assert [e.path for e in plan.entries] == ["d/", "d/x", "y"] + class TestArchiveRoundTrip: """Test end-to-end archive -> extract round-trip.""" From dd09a47c34c3556aa9545318faf5122c7ee98118 Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Thu, 29 Oct 2026 11:17:10 +0000 Subject: [PATCH 17/31] feat(cli): expose the remaining extract options - Adds '--no-same-permissions', '--touch/-m', '--overwrite-dir', and '--max-depth' flags to the 'extract' command - '-p' with '--no-same-permissions' is a now a usage error rather than silent precedence - Changes '--member/-m' becomes to '--entry/-e' - Changes '--max-total-size' to '--max-size' and now takes a suffix, so a cap reads as '500M' or '10G' rather than a byte count - Sizes that doesn't parse fail before any work - Directories that don't exist are now created for single-file inputs as well - Sorts the vcpkg dependencies into alphabetical order --- src/compresso/cli/extract.py | 87 +++++++++++++++++++--- src/compresso/frontend/dispatch.py | 2 +- tests/python/cli/test_extract.py | 115 ++++++++++++++++++++++++++--- vcpkg.json | 8 +- 4 files changed, 187 insertions(+), 25 deletions(-) diff --git a/src/compresso/cli/extract.py b/src/compresso/cli/extract.py index 4d75536..8d2fcb9 100644 --- a/src/compresso/cli/extract.py +++ b/src/compresso/cli/extract.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re import sys from pathlib import Path from typing import Annotated @@ -27,6 +28,31 @@ ) from ._run import InputFailed, conflict_mode, require_files, run_all, run_job +_SIZE = re.compile(r"(\d+)(?:([kmgt])(?:i?b)?|b)?", re.IGNORECASE) +_UNITS = {"": 1, "k": 1024, "m": 1024**2, "g": 1024**3, "t": 1024**4} + + +def parse_size(value: str) -> int: + """Parse a byte count with an optional binary suffix, e.g. `500M` or `1KB`. + + Args: + value: The text given. + + Returns: + The count in bytes. + + Raises: + ValueError: If `value` isn't a size, which the parser reports as a + usage error. + """ + match = _SIZE.fullmatch(value.strip()) + if not match: + raise ValueError(value) + + count, unit = match.groups() + + return int(count) * _UNITS[(unit or "").lower()] + def _extract_archive(job: ExtractJob, quiet: bool, verbose: bool) -> None: """Unpack one archive. @@ -126,11 +152,11 @@ def extract( help="File to write, for one compressed file that isn't an archive", ), ] = None, - member: Annotated[ + entry: Annotated[ list[str] | None, app.Option( - "--member", - "-m", + "--entry", + "-e", help="Extract only this entry, or the directory and all below it; " "repeatable", ), @@ -148,13 +174,23 @@ def extract( bool, app.Option("--force", "-f", help="Replace files that already exist"), ] = False, - max_total_size: Annotated[ + max_size: Annotated[ int | None, app.Option( - "--max-total-size", - help="Refuse archives extracting to more than this many bytes", + "--max-size", + metavar="SIZE", + parser=parse_size, + help="Refuse archives extracting to more than this, e.g. 500M or 10G", ), ] = None, + max_depth: Annotated[ + int, + app.Option( + "--max-depth", + min=0, + help="Refuse entries nested deeper than this, or 0 for no limit", + ), + ] = 32, same_permissions: Annotated[ bool, app.Option( @@ -163,6 +199,21 @@ def extract( help="Restore modes exactly, keeping setuid/setgid/sticky and ignoring the umask", ), ] = False, + no_same_permissions: Annotated[ + bool, + app.Option("--no-same-permissions", help="Don't restore modes"), + ] = False, + touch: Annotated[ + bool, + app.Option("--touch", "-m", help="Don't restore modification times"), + ] = False, + overwrite_dir: Annotated[ + bool, + app.Option( + "--overwrite-dir", + help="Also restore the mode and time of directories that already exist", + ), + ] = False, symlinks: Annotated[ SymlinkMode | None, app.Option( @@ -223,14 +274,19 @@ def extract( inputs: The archives or compressed files to unpack. directory: The directory to write into (default: beside each input). output: The file to write, for a single input that isn't an archive. - member: Entry paths to extract from each archive (default: all); a + entry: Entry paths to extract from each archive (default: all); a path matching no entry fails that archive after the rest is extracted. on_conflict: What to do when a file exists (default: rename). force: If True, replace files that already exist. - max_total_size: Cap on total extracted bytes (default: no cap). + max_size: Cap on total extracted bytes (default: no cap). + max_depth: Cap on an entry path's depth, or 0 for none. same_permissions: If True, restore archive entries' modes exactly; single-file outputs are unaffected. + no_same_permissions: If True, don't restore archive entries' modes. + touch: If True, don't restore archive entries' modification times. + overwrite_dir: If True, directories that already exist also get the + archive's mode and modification time. symlinks: What to do with a symlink entry (default: write a zip's as files, refuse any other archive's). threads: Worker threads for xz (default: one per core). @@ -252,12 +308,19 @@ def extract( if output and len(inputs) > 1: fail("Error: -o names one file; use -C for several inputs", EXIT_USAGE) + if same_permissions and no_same_permissions: + fail("Error: -p contradicts --no-same-permissions", EXIT_USAGE) + require_files(inputs) options = ExtractOptions( overwrite=mode, - max_total_size=max_total_size or 0, + max_total_size=max_size or 0, + max_depth=max_depth, + preserve_permissions=not no_same_permissions, exact_permissions=same_permissions, + preserve_timestamps=not touch, + overwrite_dir_metadata=overwrite_dir, symlinks=symlinks, threads=threads, ) @@ -266,7 +329,7 @@ def extract( inputs=inputs, directory=directory, output=output, - files=member, + files=entry, options=options, restore_name=name, ) @@ -276,6 +339,10 @@ def run_one(job: ExtractJob | DecompressionJob) -> None: _extract_archive(job, quiet, verbose) else: + # An archive creates a missing `-C` itself; a single file needs it made + if directory: + directory.mkdir(parents=True, exist_ok=True) + _decompress_one_file(job, quiet, verbose) try: diff --git a/src/compresso/frontend/dispatch.py b/src/compresso/frontend/dispatch.py index e3fdb1e..3b9bacc 100644 --- a/src/compresso/frontend/dispatch.py +++ b/src/compresso/frontend/dispatch.py @@ -250,7 +250,7 @@ def plan_extract(request: ExtractRequest) -> list[ExtractJob | DecompressionJob] job.plan = replace( job.plan, can_run=False, - reason_if_unavailable="--member/-m applies only to archives", + reason_if_unavailable="--entry applies only to archives", ) jobs.append(job) diff --git a/tests/python/cli/test_extract.py b/tests/python/cli/test_extract.py index 94a41a3..5b9294d 100644 --- a/tests/python/cli/test_extract.py +++ b/tests/python/cli/test_extract.py @@ -10,11 +10,14 @@ import pytest +import compresso.cli.extract as extract_cmd from compresso.cli._render import ( + EXIT_CANCELLED, EXIT_FAILED, EXIT_OK, EXIT_USAGE, ) +from compresso.frontend.dispatch import ExtractRequest from ..helpers import file_entry, link_entry, make_tar, renamed, umask from .conftest import CONFLICT_MODES, invoke @@ -244,8 +247,8 @@ def test_output_on_an_archive( assert not out.exists() -class TestMember: - """Test that --member/-m extracts only the entries named.""" +class TestEntry: + """Test that --entry extracts only the entries named.""" @pytest.mark.parametrize("fmt", ["tar", "zip"]) def test_only_the_chosen_entries( @@ -265,9 +268,9 @@ def test_only_the_chosen_entries( archive, "-C", dest, - "--member", + "--entry", f"{top}/f0.bin", - "-m", + "-e", f"{top}/f2.bin", "-q", ) @@ -276,7 +279,7 @@ def test_only_the_chosen_entries( for name in ("f0.bin", "f2.bin"): assert (dest / top / name).read_bytes() == (source_tree / name).read_bytes() - def test_a_missing_member_fails_after_the_rest( + def test_a_missing_entry_fails_after_the_rest( self, source_tree: Path, tmp_path: Path, make_archive: Callable[[str], Path] ) -> None: """Test that a name matching nothing exits 1, naming it, once the rest is @@ -289,9 +292,9 @@ def test_a_missing_member_fails_after_the_rest( make_archive("tar"), "-C", dest, - "--member", + "--entry", f"{top}/f1.bin", - "-m", + "-e", "nope", expect=EXIT_FAILED, ) @@ -300,11 +303,103 @@ def test_a_missing_member_fails_after_the_rest( assert sorted(p.name for p in (dest / top).iterdir()) == ["f1.bin"] def test_a_single_file_input_fails(self, payload: Path, tmp_path: Path) -> None: - """Test that --member/-m on an input that isn't an archive fails it.""" + """Test that --entry on an input that isn't an archive fails it.""" packed = tmp_path / "a.gz" packed.write_bytes(gzip.compress(payload.read_bytes())) - result = invoke("extract", packed, "-m", "x", expect=EXIT_FAILED) + result = invoke("extract", packed, "--entry", "x", expect=EXIT_FAILED) - assert f"{packed}: --member/-m applies only to archives" in result.output + assert f"{packed}: --entry applies only to archives" in result.output assert not (tmp_path / "a").exists() + + +class TestExtractOptions: + """Test the flags that set `ExtractOptions`, named as tar names them.""" + + @pytest.mark.parametrize( + "flags,field,value", + [ + (("--no-same-permissions",), "preserve_permissions", False), + (("-m",), "preserve_timestamps", False), + (("--overwrite-dir",), "overwrite_dir_metadata", True), + (("--max-depth", "5"), "max_depth", 5), + (("--max-depth", "0"), "max_depth", 0), + (("--max-size", "500M"), "max_total_size", 500 * 1024**2), + (("--max-size", "10g"), "max_total_size", 10 * 1024**3), + (("--max-size", "1KB"), "max_total_size", 1024), + (("--max-size", "2048"), "max_total_size", 2048), + (("--max-size", "0"), "max_total_size", 0), + ], + ) + def test_flag_reaches_its_field( + self, + payload: Path, + monkeypatch: pytest.MonkeyPatch, + flags: tuple[str, ...], + field: str, + value: object, + ) -> None: + """Test that each flag sets its field, recorded before anything runs.""" + seen: list[ExtractRequest] = [] + + def record(request: ExtractRequest) -> None: + seen.append(request) + raise KeyboardInterrupt + + monkeypatch.setattr(extract_cmd, "plan_extract", record) + + invoke("extract", payload, *flags, expect=EXIT_CANCELLED) + + assert getattr(seen[0].options, field) == value + + @pytest.mark.parametrize("size", ["5X", "-1", "1.5G", "M", "10MBB"]) + def test_bad_max_size_is_usage(self, payload: Path, size: str) -> None: + """Test that a size that doesn't parse is refused before any work.""" + invoke("extract", payload, "--max-size", size, expect=EXIT_USAGE) + + def test_same_permissions_contradicts_no_same_permissions( + self, payload: Path + ) -> None: + """Test that -p with --no-same-permissions exits 2.""" + result = invoke( + "extract", payload, "-p", "--no-same-permissions", expect=EXIT_USAGE + ) + assert "-p contradicts --no-same-permissions" in result.output + + @pytest.mark.parametrize("touch", [False, True], ids=["mtime", "touch"]) + def test_touch_skips_the_stored_time(self, tmp_path: Path, touch: bool) -> None: + """Test that -m leaves an entry with the time it was extracted.""" + archive = tmp_path / "old.tar" + make_tar(archive, file_entry("a.txt", mtime=1_000_000)) + dest = tmp_path / "dest" + + invoke("extract", archive, "-C", dest, *(["-m"] if touch else []), "-q") + + assert ((dest / "a.txt").stat().st_mtime == 1_000_000) != touch + + @pytest.mark.parametrize("depth,expect", [("2", EXIT_FAILED), ("3", EXIT_OK)]) + def test_max_depth_bounds_entry_paths( + self, tmp_path: Path, depth: str, expect: int + ) -> None: + """Test that an entry nested deeper than --max-depth fails the archive.""" + archive = tmp_path / "deep.tar" + make_tar(archive, file_entry("a/b/c.txt")) + dest = tmp_path / "dest" + + invoke("extract", archive, "-C", dest, "--max-depth", depth, expect=expect) + + assert (dest / "a/b/c.txt").exists() == (expect == EXIT_OK) + + def test_missing_directory_is_created( + self, payload: Path, tmp_path: Path, make_archive: Callable[[str], Path] + ) -> None: + """Test that a `-C` that doesn't exist is made for a single file, as it + is for an archive.""" + packed = tmp_path / "a.gz" + packed.write_bytes(gzip.compress(payload.read_bytes())) + dest = tmp_path / "new" / "dir" + + invoke("extract", packed, make_archive("zip"), "-C", dest, "-q") + + assert (dest / "a").read_bytes() == payload.read_bytes() + assert (dest / "tree").is_dir() diff --git a/vcpkg.json b/vcpkg.json index 21ed388..4e269d0 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -4,13 +4,13 @@ "description": "C libraries the compresso._core extension links against on Windows", "builtin-baseline": "04a9d8e5212d01ee1dd9478eadd9caade4f8b0d4", "dependencies": [ - "zlib", "bzip2", + "libarchive", "liblzma", - "zstd", + "libzip", "lz4", "snappy", - "libzip", - "libarchive" + "zlib", + "zstd" ] } From 8616ef9b51a1c1207a9bd721a763b0d8b4fa197e Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Thu, 29 Oct 2026 18:01:50 +0000 Subject: [PATCH 18/31] refactor(benchmark): make benchmark's flags repeatable, and move printing to _render/ - print_results() becomes cli/_render/benchmark.py alongside the rest of the CLI output - New benchmark_header() takes over the settings block the command built inline - benchmark_file() drops the redundant warnings about a compressed size that changed between runs and a decompressed size that didn't match - The '--algos/--strategies/--levels' become repeatable '--algo/-a', '--strategy/-s' and '--level/-l' - Values no longer have to be split out of a comma-separated string - '--repeats/-r' is now bounded with a minimum of 1 - Adds new test_benchmark.py that covers each flag, header and table printing, and non-int/'auto' level exiting 2 --- bench/run.py | 5 +- src/compresso/__init__.py | 3 +- src/compresso/cli/_render/benchmark.py | 76 ++++++++++++++ src/compresso/cli/benchmark.py | 119 ++++++++++------------ src/compresso/frontend/dispatch.py | 2 +- src/compresso/introspect/__init__.py | 3 +- src/compresso/introspect/benchmark.py | 63 +----------- src/compresso/meson.build | 1 + tests/python/cli/test_app.py | 4 - tests/python/cli/test_benchmark.py | 99 ++++++++++++++++++ tests/python/introspect/test_benchmark.py | 43 -------- 11 files changed, 236 insertions(+), 182 deletions(-) create mode 100644 src/compresso/cli/_render/benchmark.py create mode 100644 tests/python/cli/test_benchmark.py diff --git a/bench/run.py b/bench/run.py index bead561..529b1c6 100644 --- a/bench/run.py +++ b/bench/run.py @@ -26,7 +26,8 @@ from tabulate import tabulate from compresso import _core -from compresso.introspect.benchmark import BenchmarkResult, print_results +from compresso.cli._render.benchmark import benchmark_results +from compresso.introspect.benchmark import BenchmarkResult REPO_ROOT = Path(__file__).resolve().parent.parent WORK_DIR = REPO_ROOT / "build" / "bench" @@ -326,7 +327,7 @@ def main() -> None: args = parser.parse_args() results = run(args.formats.split(","), args.size_mb, args.repeats, args.cool) - print_results(results) + benchmark_results(results) if args.compare: compare(results, args.compare) diff --git a/src/compresso/__init__.py b/src/compresso/__init__.py index f2b1bce..881c049 100644 --- a/src/compresso/__init__.py +++ b/src/compresso/__init__.py @@ -36,7 +36,7 @@ plan_archive, plan_extraction, ) -from .introspect.benchmark import benchmark_file, print_results +from .introspect.benchmark import benchmark_file from .introspect.capabilities import list_capabilities from .introspect.file_inspect import InspectResult, inspect from .introspect.speeds import get_estimated_speeds @@ -79,5 +79,4 @@ "plan_compression", "plan_decompression", "plan_extraction", - "print_results", ] diff --git a/src/compresso/cli/_render/benchmark.py b/src/compresso/cli/_render/benchmark.py new file mode 100644 index 0000000..765a6d1 --- /dev/null +++ b/src/compresso/cli/_render/benchmark.py @@ -0,0 +1,76 @@ +"""What `benchmark` prints: the run's settings, then a table of results.""" + +from __future__ import annotations + +from pathlib import Path + +from ...introspect.benchmark import BenchmarkResult + + +def _level_name(level: int | None) -> str | int: + return "auto" if level is None else level + + +def benchmark_header( + file: Path, + repeats: int, + algos: list[str] | None, + strategies: list[str] | None, + levels: list[int | None], +) -> None: + """Print what the run covers, then a blank line. + + Args: + file: The file being benchmarked. + repeats: How many times each combination runs. + algos: The algorithms asked for, or None for all. + strategies: The strategies asked for, or None for all. + levels: The levels asked for, None meaning the strategy's own. + """ + lines: list[str] = [f"Running benchmarks on: {file}", f"Repeats: {repeats}"] + if algos: + lines.append(f"Algorithms: {', '.join(algos)}") + + if strategies: + lines.append(f"Strategies: {', '.join(strategies)}") + + lines.append(f"Levels: {', '.join(str(_level_name(lv)) for lv in levels)}") + print("\n".join(lines) + "\n") + + +def benchmark_results(results: list[BenchmarkResult]) -> None: + """Print one table row per algorithm, strategy and level. + + Args: + results: The benchmark's results, in run order. + """ + headers: list[str] = [ + "Algo", + "Strategy", + "Level", + "Comp Time (s)", + "Decomp Time (s)", + "MB/s (Comp)", + "MB/s (Decomp)", + "Ratio (comp/orig)", + ] + + table_data: list[list[str | int]] = [ + [ + r.algo, + r.strategy, + _level_name(r.level), + f"{r.compress_time:.4f}", + f"{r.decompress_time:.4f}", + f"{r.comp_mb_s:.2f}", + f"{r.decomp_mb_s:.2f}", + f"{r.ratio:.3f}", + ] + for r in results + ] + + # Deferred: tabulate pulls in wcwidth, which every CLI command would + # otherwise pay for + from tabulate import tabulate + + print(tabulate(tabular_data=table_data, headers=headers, tablefmt="grid")) diff --git a/src/compresso/cli/benchmark.py b/src/compresso/cli/benchmark.py index daa6771..19d1f52 100644 --- a/src/compresso/cli/benchmark.py +++ b/src/compresso/cli/benchmark.py @@ -5,30 +5,59 @@ from pathlib import Path from typing import Annotated -from ..introspect.benchmark import benchmark_file, print_results +from ..introspect.benchmark import benchmark_file from ._app import app from ._render import EXIT_USAGE, cancelled, fail, succeed +from ._render.benchmark import benchmark_header, benchmark_results + + +def _parse_level(value: str) -> int | None: + """Parse one `-l` value, exiting 2 if it's neither a number nor `auto`. + + Args: + value: The text given. + + Returns: + The level, or None for `auto`, the strategy's own. + """ + if value.lower() == "auto": + return None + + try: + return int(value) + + except ValueError: + fail(f"Error: Invalid level {value!r}; use an integer or 'auto'", EXIT_USAGE) def benchmark( file: Annotated[Path, app.Argument(help="File to benchmark")], - algos: Annotated[ - str | None, app.Option("--algos", help="Comma-separated list of algorithms") - ] = "all", - strategies: Annotated[ - str | None, - app.Option("--strategies", help="Comma-separated list of strategies"), - ] = "all", - levels: Annotated[ - str | None, + algo: Annotated[ + list[str] | None, + app.Option("--algo", "-a", help="Algorithm to run; repeatable (default: all)"), + ] = None, + strategy: Annotated[ + list[str] | None, + app.Option( + "--strategy", "-s", help="Strategy to run; repeatable (default: all)" + ), + ] = None, + level: Annotated[ + list[str] | None, app.Option( - "--levels", - help="Comma-separated list of levels, or 'auto'; a level outside an " + "--level", + "-l", + metavar="N|auto", + help="Level to run, or 'auto'; repeatable; a level outside an " "algorithm's range is skipped for that algorithm", + show_default="auto", ), - ] = "auto", + ] = None, repeats: Annotated[ - int, app.Option("--repeats", help="Number of times to repeat each benchmark") + int, + app.Option( + "--repeats", "-r", help="Number of times to repeat each benchmark", min=1 + ), ] = 1, temp_dir: Annotated[ Path | None, @@ -47,65 +76,23 @@ def benchmark( Args: file: The path to the file to benchmark. - algos: Comma-separated list of algorithms to use (default: all available). - strategies: Comma-separated list of strategies to use (default: all). - levels: Comma-separated list of levels to use (default: all). + algo: Algorithms to run (default: all available). + strategy: Strategies to run (default: all). + level: Levels to run, or `auto` for the strategy's own (default: auto). repeats: Number of times to repeat each benchmark (default: 1). temp_dir: Temporary directory for benchmark files (default: None). update_cache: If True, update the speed estimates cache with benchmark results (default: False). """ + levels: list[int | None] = [_parse_level(v) for v in level or ["auto"]] + try: - algo_list: list[str] = [] - if algos and algos.lower() != "all": - algo_list = [a.strip() for a in algos.split(",") if a.strip()] - - strategy_list: list[str] = [] - if strategies and strategies.lower() != "all": - strategy_list = [s.strip() for s in strategies.split(",") if s.strip()] - - level_list: list[int | None] = [] - if levels: - for level in levels.split(sep=","): - level: str = level.strip() - if not level: - continue - - if level.lower() in ("auto", "default"): - level_list.append(None) - - else: - try: - level_list.append(int(level)) - - except ValueError: - fail( - f"Invalid level: {level}. Use integers or 'auto'", - EXIT_USAGE, - ) - - lines: list[str] = [ - f"Running benchmarks on: {file}", - f"Repeats: {repeats}", - ] - if algo_list: - lines.append(f"Algorithms: {', '.join(algo_list)}") - - if strategy_list: - lines.append(f"Strategies: {', '.join(strategy_list)}") - - if level_list is not None: - level_str: str = ", ".join( - str(object=level) if level is not None else "auto" - for level in level_list - ) - lines.append(f"Levels: {level_str}") - print("\n".join(lines) + "\n") + benchmark_header(file, repeats, algo, strategy, levels) results = benchmark_file( src=file, - algos=algo_list or None, - strategies=strategy_list or None, - levels=level_list or None, + algos=algo, + strategies=strategy, + levels=levels, repeats=repeats, temp_dir=temp_dir, update_cache=update_cache, @@ -114,7 +101,7 @@ def benchmark( if not results: fail("No benchmark results generated\n") - print_results(results) + benchmark_results(results) if update_cache: print() diff --git a/src/compresso/frontend/dispatch.py b/src/compresso/frontend/dispatch.py index 3b9bacc..0ba3c61 100644 --- a/src/compresso/frontend/dispatch.py +++ b/src/compresso/frontend/dispatch.py @@ -112,7 +112,7 @@ class ExtractRequest: inputs: The archives and single-file containers to unpack. directory: The `-C` value, or None to write beside each input. output: The `-o` file, for a single non-archive input. - files: The `-m` entry paths to extract from each archive, or + files: The `-e` entry paths to extract from each archive, or None for all. options: The extraction policy; single-file inputs use only its `overwrite` and `threads`. diff --git a/src/compresso/introspect/__init__.py b/src/compresso/introspect/__init__.py index c13908e..25f2d9a 100644 --- a/src/compresso/introspect/__init__.py +++ b/src/compresso/introspect/__init__.py @@ -4,7 +4,7 @@ The compression backends themselves are the C extension's, under csrc/. """ -from .benchmark import benchmark_file, print_results +from .benchmark import benchmark_file from .capabilities import list_capabilities from .file_inspect import InspectResult, inspect from .speeds import get_estimated_speeds @@ -15,5 +15,4 @@ "get_estimated_speeds", "inspect", "list_capabilities", - "print_results", ] diff --git a/src/compresso/introspect/benchmark.py b/src/compresso/introspect/benchmark.py index 69430d3..ad7bd82 100644 --- a/src/compresso/introspect/benchmark.py +++ b/src/compresso/introspect/benchmark.py @@ -221,17 +221,7 @@ def benchmark_file( level=level, ) comp_times.append(time.perf_counter() - start_time) - - if compressed_size is None: - compressed_size: int = comp_path.stat().st_size - - else: - sz: int = comp_path.stat().st_size - if sz != compressed_size: - print( - f"Warning: Compressed size changed between runs: {compressed_size} vs {sz}" - ) - compressed_size: int = sz + compressed_size = comp_path.stat().st_size # Decompression start_time: float = time.perf_counter() @@ -241,15 +231,6 @@ def benchmark_file( ) decomp_times.append(time.perf_counter() - start_time) - # Verify - if ( - not decomp_path.is_file() - or decomp_path.stat().st_size != input_size - ): - print( - f"Warning: Decompressed file size mismatch for {decomp_path}" - ) - finally: _safe_unlink(path=comp_path) _safe_unlink(path=decomp_path) @@ -273,45 +254,3 @@ def benchmark_file( update_from_benchmarks(results) return results - - -def print_results(results: list[BenchmarkResult]) -> None: - """Print benchmark results in a tabular format - - Args: - results: List of BenchmarkResult objects to print - """ - if not results: - print("No results to display.") - return - - headers: list[str] = [ - "Algo", - "Strategy", - "Level", - "Comp Time (s)", - "Decomp Time (s)", - "MB/s (Comp)", - "MB/s (Decomp)", - "Ratio (comp/orig)", - ] - - table_data: list[list[str | int]] = [] - for r in results: - row: list[str | int] = [ - r.algo, - r.strategy, - r.level if r.level is not None else "auto", - f"{r.compress_time:.4f}", - f"{r.decompress_time:.4f}", - f"{r.comp_mb_s:.2f}", - f"{r.decomp_mb_s:.2f}", - f"{r.ratio:.3f}", - ] - table_data.append(row) - - # Deferred: tabulate pulls in wcwidth, which every CLI command would - # otherwise pay for via the introspect package import - from tabulate import tabulate - - print(tabulate(tabular_data=table_data, headers=headers, tablefmt="grid")) diff --git a/src/compresso/meson.build b/src/compresso/meson.build index 66c2f7e..c3b12c3 100644 --- a/src/compresso/meson.build +++ b/src/compresso/meson.build @@ -64,6 +64,7 @@ py.install_sources( py.install_sources( 'cli/_render/__init__.py', + 'cli/_render/benchmark.py', 'cli/_render/formats.py', 'cli/_render/info.py', 'cli/_render/jobs.py', diff --git a/tests/python/cli/test_app.py b/tests/python/cli/test_app.py index 97a1eb8..b627029 100644 --- a/tests/python/cli/test_app.py +++ b/tests/python/cli/test_app.py @@ -357,10 +357,6 @@ def test_out_of_range_level_is_usage( assert message in result.output assert not t.dest.exists() - def test_bad_benchmark_level_is_usage(self, payload: Path) -> None: - """Test that a level that is not a number is a usage error.""" - invoke("benchmark", payload, "--levels", "nope", expect=EXIT_USAGE) - @pytest.mark.parametrize("flags", [(), ("-F", "7z")], ids=["suffix", "flag"]) def test_recognised_but_unsupported_archive_names_itself( self, source_tree: Path, tmp_path: Path, flags: tuple[str, ...] diff --git a/tests/python/cli/test_benchmark.py b/tests/python/cli/test_benchmark.py new file mode 100644 index 0000000..d4e6627 --- /dev/null +++ b/tests/python/cli/test_benchmark.py @@ -0,0 +1,99 @@ +"""Tests for `compresso benchmark`.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +import compresso.cli.benchmark as benchmark_cmd +from compresso.cli._render import EXIT_USAGE +from compresso.introspect.benchmark import BenchmarkResult + +from .conftest import invoke + + +@pytest.fixture +def calls(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, Any]]: + """Record each `benchmark_file` call, answering with one canned result.""" + seen: list[dict[str, Any]] = [] + + def record(**kwargs: Any) -> list[BenchmarkResult]: + seen.append(kwargs) + return [BenchmarkResult("zstd", "fast", None, 0.5, 0.25, 1000, 400)] + + monkeypatch.setattr(benchmark_cmd, "benchmark_file", record) + return seen + + +class TestBenchmark: + """Test how `benchmark` turns its flags into a run, and prints it.""" + + @pytest.mark.parametrize( + "args,algos,strategies,levels,repeats", + [ + ((), None, None, [None], 1), + ( + ( + "-a", + "zstd", + "--algo", + "lz4", + "-s", + "fast", + "--strategy", + "max_ratio", + ), + ["zstd", "lz4"], + ["fast", "max_ratio"], + [None], + 1, + ), + (("-l", "3", "--level", "AUTO", "-l", "9"), None, None, [3, None, 9], 1), + (("-r", "3"), None, None, [None], 3), + ], + ids=["defaults", "algo-strategy", "level", "repeats"], + ) + def test_flags_reach_benchmark_file( + self, + calls: list[dict[str, Any]], + payload: Path, + args: tuple[str, ...], + algos: list[str] | None, + strategies: list[str] | None, + levels: list[int | None], + repeats: int, + ) -> None: + """Test that each repeated flag arrives as a list, and an omitted -a or -s + as None, meaning all.""" + invoke("benchmark", payload, *args) + + (call,) = calls + assert call["algos"] == algos + assert call["strategies"] == strategies + assert call["levels"] == levels + assert call["repeats"] == repeats + + def test_prints_settings_then_table( + self, calls: list[dict[str, Any]], payload: Path + ) -> None: + """Test the header lines and that the table names the auto level.""" + output = invoke( + "benchmark", payload, "-a", "zstd", "-l", "auto", "-l", "3" + ).output + + assert output.startswith( + f"Running benchmarks on: {payload}\nRepeats: 1\n" + "Algorithms: zstd\nLevels: auto, 3\n\n" + ) + assert "| zstd | fast | auto |" in output + + @pytest.mark.parametrize("level", ["nope", "1.5", ""]) + def test_bad_level_is_usage( + self, calls: list[dict[str, Any]], payload: Path, level: str + ) -> None: + """Test that a level that is neither a number nor auto is a usage error.""" + invoke("benchmark", payload, "-l", level, expect=EXIT_USAGE) + + assert not calls diff --git a/tests/python/introspect/test_benchmark.py b/tests/python/introspect/test_benchmark.py index 8aeeab7..3693254 100644 --- a/tests/python/introspect/test_benchmark.py +++ b/tests/python/introspect/test_benchmark.py @@ -5,7 +5,6 @@ from compresso.introspect.benchmark import ( BenchmarkResult, benchmark_file, - print_results, ) MIB = 1024 * 1024 @@ -66,45 +65,3 @@ def test_levels_outside_a_backends_range_are_skipped( ran = {(r.algo, r.level) for r in results} assert ran == {("zlib", None), ("zstd", None), ("zstd", 15), ("snappy", None)} - - -class TestPrintResults: - """Test the print_results function.""" - - def test_print_results_with_empty_list( - self, capsys: pytest.CaptureFixture[str] - ) -> None: - """Test that no results says so rather than printing an empty table.""" - print_results([]) - - assert capsys.readouterr().out == "No results to display.\n" - - def test_print_results_with_multiple_results(self, capsys) -> None: - """Test printing results with multiple results.""" - results = [ - BenchmarkResult( - algo="zlib", - strategy="balanced", - level=6, - compress_time=1.0, - decompress_time=0.5, - input_size=1000000, - compressed_size=500000, - ), - BenchmarkResult( - algo="zstd", - strategy="fast", - level=1, - compress_time=0.5, - decompress_time=0.3, - input_size=1000000, - compressed_size=450000, - ), - ] - - print_results(results) - captured = capsys.readouterr() - - # Should contain both algo names - assert "zlib" in captured.out.lower() - assert "zstd" in captured.out.lower() From 149f1ec327cdd65e05f31d1c273b172d5beccb55 Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Thu, 29 Oct 2026 19:39:13 +0000 Subject: [PATCH 19/31] fix(cli): drop the ratio on extraction and space the summaries evenly - summary() now takes a 'ratio' flag, off for extraction and decompression - Moves the leading blank line out of summary() into the command, printed once before the jobs run - Several inputs now get one blank line before, between and after their lines - skipped() now ends with a blank line too, so it spaces like the other outputs --- src/compresso/cli/_render/jobs.py | 15 +++++++++++---- src/compresso/cli/compress.py | 3 +++ src/compresso/cli/extract.py | 16 ++++++++++++++-- tests/python/cli/test_app.py | 23 +++++++++++++++++++++-- 4 files changed, 49 insertions(+), 8 deletions(-) diff --git a/src/compresso/cli/_render/jobs.py b/src/compresso/cli/_render/jobs.py index f5e38cc..3b840e4 100644 --- a/src/compresso/cli/_render/jobs.py +++ b/src/compresso/cli/_render/jobs.py @@ -10,7 +10,13 @@ def summary( - name: Path, before: int, after: int, elapsed: float, renamed: Path | None + name: Path, + before: int, + after: int, + elapsed: float, + renamed: Path | None, + *, + ratio: bool = True, ) -> None: """Print the one line a finished input gets by default. @@ -20,11 +26,12 @@ def summary( after: Bytes written. elapsed: Seconds the job took. renamed: Where the output went instead, if its name was taken. + ratio: Whether to show the (de)compression ratio in the summary. """ - percent: float = after / before * 100 if before else 0 + percent: str = f" ({after / before * 100:.0f}%)" if ratio and before else "" note: str = f", output renamed to {renamed}" if renamed else "" print( - f"\n{name} {format_size(before)} → {format_size(after)} ({percent:.0f}%) " + f"{name} {format_size(before)} → {format_size(after)}{percent} " f"in {format_time(elapsed)}{note}\n" ) @@ -35,7 +42,7 @@ def skipped(existing: Path) -> None: Args: existing: The output that was already there. """ - print(f"{existing} skipped, already exists") + print(f"{existing} skipped, already exists\n") def compress_plan(plan: CompressionPlan) -> None: diff --git a/src/compresso/cli/compress.py b/src/compresso/cli/compress.py index 98e663e..311d6e6 100644 --- a/src/compresso/cli/compress.py +++ b/src/compresso/cli/compress.py @@ -261,6 +261,9 @@ def run_one(job: CompressionJob | ArchiveJob) -> None: else: _compress_one_file(job, quiet, verbose) + if not (quiet or verbose): + print() + sys.exit(run_all([job], run_one, "Compression")) except KeyboardInterrupt: diff --git a/src/compresso/cli/extract.py b/src/compresso/cli/extract.py index 8d2fcb9..6798f5f 100644 --- a/src/compresso/cli/extract.py +++ b/src/compresso/cli/extract.py @@ -81,7 +81,9 @@ def _extract_archive(job: ExtractJob, quiet: bool, verbose: bool) -> None: extract_result(plan, elapsed) elif not quiet: - summary(plan.archive, plan.archive.stat().st_size, total, elapsed, None) + summary( + plan.archive, plan.archive.stat().st_size, total, elapsed, None, ratio=False + ) def _decompress_one_file(job: DecompressionJob, quiet: bool, verbose: bool) -> None: @@ -128,7 +130,14 @@ def _decompress_one_file(job: DecompressionJob, quiet: bool, verbose: bool) -> N else: renamed: Path | None = written if written != plan.dest else None - summary(plan.src, compressed_size, written.stat().st_size, elapsed, renamed) + summary( + plan.src, + compressed_size, + written.stat().st_size, + elapsed, + renamed, + ratio=False, + ) def extract( @@ -345,6 +354,9 @@ def run_one(job: ExtractJob | DecompressionJob) -> None: _decompress_one_file(job, quiet, verbose) + if not (quiet or verbose): + print() + try: sys.exit(run_all(plan_extract(request), run_one, "Extraction")) diff --git a/tests/python/cli/test_app.py b/tests/python/cli/test_app.py index b627029..bacaa9f 100644 --- a/tests/python/cli/test_app.py +++ b/tests/python/cli/test_app.py @@ -217,12 +217,31 @@ class TestOutput: def test_default_is_one_summary_line( self, job_args: tuple[list[str | Path], Path] ) -> None: - """Test that a job prints `name before → after (n%) in time` and no more.""" + """Test that a job prints `name before → after (n%) in time` and no + more, without the ratio on extraction.""" args, name = job_args output = invoke(*args).output + ratio = r"\(\d+%\) " if args[0] == "compress" else "" + + assert re.fullmatch( + rf"\n{re.escape(str(name))} \S+ \S+ → \S+ \S+ {ratio}in \S+\n\n", output + ), output + + def test_inputs_are_one_blank_line_apart( + self, payload: Path, tmp_path: Path + ) -> None: + """Test that several inputs' lines have one blank line before, between + and after them.""" + inputs = [tmp_path / "a.gz", tmp_path / "b.gz"] + for path in inputs: + path.write_bytes(gzip.compress(payload.read_bytes())) + + output = invoke("extract", *inputs, "-C", tmp_path / "dest").output + assert re.fullmatch( - rf"\n{re.escape(str(name))} \S+ \S+ → \S+ \S+ \(\d+%\) in \S+\n\n", output + "".join(rf"\n{re.escape(str(path))} [^\n]+\n" for path in inputs) + "\n", + output, ), output def test_verbose_prints_the_blocks( From 19bda284107dd975816169f39874e26ec832aca1 Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Thu, 29 Oct 2026 23:05:12 +0000 Subject: [PATCH 20/31] feat(core): match entry paths against globs - New path_matches() tests a pattern against any run of whole components in an entry path - The matcher uses fnmatch(3) syntax with no flags, and backtracks only to the last '*', keeping it linear time based on path length - An unterminated '[' is a literal bracket, and a zip directory's trailing slash is trimmed before matching - Adds 16 test case rows, with the usual '_Static_assert' guarding the 'TEST_RANGE' --- src/compresso/csrc/archives/select.c | 72 ++++++++++++++++++++++++++++ src/compresso/csrc/archives/select.h | 5 ++ tests/c/test_select.c | 36 ++++++++++++++ 3 files changed, 113 insertions(+) diff --git a/src/compresso/csrc/archives/select.c b/src/compresso/csrc/archives/select.c index 902dc97..322899f 100644 --- a/src/compresso/csrc/archives/select.c +++ b/src/compresso/csrc/archives/select.c @@ -58,6 +58,78 @@ int entry_is_selected(const ArchiveEntry *entry, Selection *sel) { } } +// The pattern after its first token if that token matches `c`, else NULL +static const char *match_token(const char *pat, unsigned char c) { + if (*pat == '?') + return pat + 1; + if (*pat == '\\' && pat[1]) + return (unsigned char)pat[1] == c ? pat + 2 : NULL; + + if (*pat == '[') { + const char *p = pat + 1; + int negate = *p == '!' || *p == '^'; + if (negate) + p++; + // A `]` straight after the opening is literal + int hit = 0; + do { + if (*p == '\0') + return c == '[' ? pat + 1 : NULL; // Unterminated, so `[` is literal + unsigned char lo = (unsigned char)*p, hi = lo; + if (p[1] == '-' && p[2] && p[2] != ']') { + hi = (unsigned char)p[2]; + p += 2; + } + hit |= lo <= c && c <= hi; + p++; + } while (*p != ']'); + return hit != negate ? p + 1 : NULL; + } + + return (unsigned char)*pat == c ? pat + 1 : NULL; +} + +// Whether `pat` matches `s` up to its end or a '/'; backtracks only to the +// last `*`, so the time stays linear: O(len * strlen(pat)) +static int glob_match(const char *pat, const char *s, size_t len) { + const char *star = NULL; + size_t retry = 0, i = 0; + while (i < len) { + if (*pat == '\0' && i > 0 && s[i] == '/') + return 1; + if (*pat == '*') { + star = ++pat; + retry = i; + continue; + } + const char *next = match_token(pat, (unsigned char)s[i]); + if (next) { + pat = next; + i++; + } else if (star) { + pat = star; + i = ++retry; + } else { + return 0; + } + } + while (*pat == '*') + pat++; + return *pat == '\0'; +} + +int path_matches(const char *path, const char *pat) { + size_t len = strlen(path); + while (len > 0 && path[len - 1] == '/') + len--; + + for (size_t start = 0; start < len; start++) + if ((start == 0 || path[start - 1] == '/') && + glob_match(pat, path + start, len - start)) + return 1; + return 0; +} + static PyObject *join_str(const char *sep, PyObject *items) { PyObject *s = PyUnicode_FromString(sep); PyObject *joined = s ? PyUnicode_Join(s, items) : NULL; diff --git a/src/compresso/csrc/archives/select.h b/src/compresso/csrc/archives/select.h index c6f34a1..8e543de 100644 --- a/src/compresso/csrc/archives/select.h +++ b/src/compresso/csrc/archives/select.h @@ -23,6 +23,11 @@ int selection_init(Selection *sel, const char **files, size_t num_files); // marks every name that matches, so "d" and "d/x" both count as found int entry_is_selected(const ArchiveEntry *entry, Selection *sel); +// Whether the glob `pat` matches any run of whole components in `path`: e.g. +// "*.log" matches "a/b.log", and "a" matches everything below "a/"; the syntax +// is fnmatch(3)'s with no flags, so `*` and `?` also match '/' +int path_matches(const char *path, const char *pat); + // Raises an error listing unmatched requested names and skipped hardlinks int report_incomplete(const Selection *sel, PyObject *skipped_links, CoreContext *ctx); diff --git a/tests/c/test_select.c b/tests/c/test_select.c index b9e90ee..e28dddf 100644 --- a/tests/c/test_select.c +++ b/tests/c/test_select.c @@ -66,3 +66,39 @@ void test_names_select_whole_components(int index) { TEST_ASSERT_EQUAL_STRING(c->selected, selected); TEST_ASSERT_EQUAL_STRING(c->unmatched, unmatched); } + +typedef struct { + const char *pat; + const char *path; + int matches; +} MatchCase; + +static const MatchCase MATCH_CASES[] = { + {"*.log", "a/b.log", 1}, + {"*.log", "b.log.gz", 0}, + {"node_modules", "x/node_modules_old/y.js", 0}, + {"a/b", "x/a/b/c", 1}, + {"b", "ab", 0}, + {"a*c", "a/b/c", 1}, // `*` crosses '/' + {"a*b*c", "aXbYbZc", 1}, + {"?.txt", "a.txt", 1}, + {"[!ab].txt", "a.txt", 0}, + {"[a-c]x", "bx", 1}, + {"[]]", "]", 1}, + {"[a", "[a", 1}, // Unterminated, so literal + {"\\*", "*", 1}, + {"\\*", "a", 0}, + {"d", "d/", 1}, // A zip directory's trailing slash + {"", "/a", 0}, // A tar's absolute path has no empty component to match +}; + +// The TEST_RANGE below must span exactly these rows +_Static_assert(sizeof(MATCH_CASES) / sizeof(MATCH_CASES[0]) == 16, + "update TEST_RANGE"); + +TEST_RANGE([ 0, 15, 1 ]) +void test_path_matches(int index) { + const MatchCase *c = &MATCH_CASES[index]; + TEST_ASSERT_EQUAL_INT_MESSAGE(c->matches, path_matches(c->path, c->pat), + c->pat); +} From 3dfdd12eae3cee78ebb6e48875a2b0bc3869504e Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Fri, 30 Oct 2026 17:42:18 +0000 Subject: [PATCH 21/31] feat(core): filter archive entries with include and exclude globs - create_archive()/extract_archive() now take 'include' and 'exclude' patterns, held in a 'PathFilter' - An exclude wins, and with any includes an entry must match one - Creating matches before the stat, so an excluded tree is never descended into - A directory that doesn't match an include is kept only provisionally, pruned afterwards if nothing below it survived - An 'include' adds to 'files', an exclude overrides both, and a directory entry is kept only if it matches itself - Includes that match no entry join the unmatched names in 'IncompleteExtractionError', once the rest is extracted, so a typo isn't silent - Excludes are applied after that check, so a name or include the filter drops still counts as found - entry_name() becomes walk_entry_name(), so the walk and the filter agree on the name a pattern is matched against - The converter refuses a bare str or bytes, which would otherwise be taken as a sequence of one-character patterns - Adds new tests covering includes, excludes, and combinations --- src/compresso/_core.pyi | 18 +++- src/compresso/csrc/_core.c | 96 +++++++++++++++++---- src/compresso/csrc/archives.h | 18 +++- src/compresso/csrc/archives/archive_ops.c | 37 ++------ src/compresso/csrc/archives/extract.c | 6 +- src/compresso/csrc/archives/select.c | 53 ++++++++++-- src/compresso/csrc/archives/select.h | 21 +++-- src/compresso/csrc/archives/walk.c | 100 ++++++++++++++++++++-- src/compresso/csrc/archives/walk.h | 22 +++-- tests/c/test_select.c | 4 +- tests/python/test_archive_create.py | 61 +++++++++++++ tests/python/test_archive_extract.py | 68 +++++++++++++++ 12 files changed, 420 insertions(+), 84 deletions(-) diff --git a/src/compresso/_core.pyi b/src/compresso/_core.pyi index 7400e69..c105652 100644 --- a/src/compresso/_core.pyi +++ b/src/compresso/_core.pyi @@ -153,6 +153,8 @@ def create_archive( threads: int = ..., progress: ProgressFn | None = ..., cancel: CancelToken | None = ..., + include: Sequence[str] = ..., + exclude: Sequence[str] = ..., ) -> str: """Create an archive from the given input paths in the given format. @@ -161,6 +163,10 @@ def create_archive( Returns the path actually written, which RENAME may have changed from `output_path`; SKIP returns `output_path` unchanged and writes nothing. + + `include` and `exclude` are globs matched against entry names; an exclude + wins and isn't descended into, and with any includes, only matching entries + and the directories holding them are stored. """ def extract_archive( @@ -179,6 +185,8 @@ def extract_archive( threads: int = ..., progress: ProgressFn | None = ..., cancel: CancelToken | None = ..., + include: Sequence[str] = ..., + exclude: Sequence[str] = ..., ) -> None: """Extract an archive to output_dir; an empty `files` extracts every entry. @@ -187,12 +195,18 @@ def extract_archive( `overwrite` is 0 = error, 1 = skip, 2 = overwrite, 3 = rename; `allow_symlinks` is 0 = deny, 1 = allow if the target stays inside - output_dir, 2 = rewrite to regular files, -1 = the format's default (2 for zip, 0 otherwise); - `max_total_size` and `max_depth` treat 0 as unlimited; + output_dir, 2 = rewrite to regular files, -1 = the format's default (2 for zip, + 0 otherwise); `max_total_size` and `max_depth` treat 0 as unlimited; `overwrite_dir_metadata` also restores the mode and mtime of a directory that already exists, as GNU tar does, except when `overwrite` is 1 (skip); `exact_permissions` restores modes verbatim, where by default setuid/setgid/sticky are dropped and the umask applies. + + `include` and `exclude` are matched as for `create_archive`: as in bsdtar, + an entry is extracted if `files` names it or an include matches it, and an + exclude overrides both; a directory entry is kept only if it matches + itself; an include that no entry matches raises `IncompleteExtractionError`, + as a name in `files` does, after the rest is extracted. """ def detect_format(file_path: StrOrBytesPath) -> str: diff --git a/src/compresso/csrc/_core.c b/src/compresso/csrc/_core.c index 26cc5a4..ae2cca0 100644 --- a/src/compresso/csrc/_core.c +++ b/src/compresso/csrc/_core.c @@ -240,6 +240,54 @@ static int path_list_converter(PyObject *obj, void *out) { return 0; } +// Glob patterns for `include` and `exclude`, decoded as UTF-8 as archives store +// entry names; `keepalive` holds the str objects that own that UTF-8 +static int pattern_list_converter(PyObject *obj, void *out) { + PathList *list = out; + if (!obj) { + Py_CLEAR(list->keepalive); + free(list->paths); + list->paths = NULL; + return 1; + } + + if (PyUnicode_Check(obj) || PyBytes_Check(obj)) { + PyErr_Format(PyExc_TypeError, "patterns must be a sequence of str, not %s", + Py_TYPE(obj)->tp_name); + return 0; + } + + list->keepalive = PySequence_Tuple(obj); + if (!list->keepalive) { + return 0; + } + + Py_ssize_t n = PyTuple_GET_SIZE(list->keepalive); + list->paths = n > 0 ? safe_malloc((size_t)n * sizeof(char *)) : NULL; + list->count = (size_t)n; + if (n > 0 && !list->paths) { + goto fail; + } + + for (Py_ssize_t i = 0; i < n; i++) { + PyObject *item = PyTuple_GET_ITEM(list->keepalive, i); + if (!PyUnicode_Check(item)) { + PyErr_Format(PyExc_TypeError, "patterns must contain only str, not %s", + Py_TYPE(item)->tp_name); + goto fail; + } + list->paths[i] = PyUnicode_AsUTF8(item); + if (!list->paths[i]) { + goto fail; + } + } + return Py_CLEANUP_SUPPORTED; + +fail: + pattern_list_converter(NULL, list); + return 0; +} + // ---- Argument Parsing ---- // Raises ValueError for an unknown strategy name @@ -366,7 +414,8 @@ static PyObject *py_create_archive(PyObject *self, PyObject *args, PyObject *kwargs) { static char *kwlist[] = {"output_path", "format", "input_paths", "compression_level", "overwrite", "threads", - "progress", "cancel", NULL}; + "progress", "cancel", "include", + "exclude", NULL}; PyObject *output_path_bytes = NULL; const char *format_name = NULL; @@ -376,14 +425,18 @@ static PyObject *py_create_archive(PyObject *self, PyObject *args, int threads = 0; // One per core PyObject *progress = NULL; PyObject *cancel = NULL; + PathList include = {0}; + PathList exclude = {0}; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&sO&|i$iiOO", kwlist, - PyUnicode_FSConverter, &output_path_bytes, - &format_name, path_list_converter, &inputs, - &compression_level, &overwrite_existing, - &threads, &progress, &cancel)) { + if (!PyArg_ParseTupleAndKeywords( + args, kwargs, "O&sO&|i$iiOOO&O&", kwlist, PyUnicode_FSConverter, + &output_path_bytes, &format_name, path_list_converter, &inputs, + &compression_level, &overwrite_existing, &threads, &progress, &cancel, + pattern_list_converter, &include, pattern_list_converter, &exclude)) { return NULL; // Error already set } + PathFilter filter = {include.paths, include.count, exclude.paths, + exclude.count}; CompressionPipeline pipe = pipeline_from_name(format_name, compression_level); CoreContext ctx; @@ -405,12 +458,15 @@ static PyObject *py_create_archive(PyObject *self, PyObject *args, char actual_path[FS_PATH_MAX]; result = create_archive(PyBytes_AS_STRING(output_path_bytes), &pipe, - inputs.paths, inputs.count, overwrite_existing, - actual_path, sizeof(actual_path), &ctx); + inputs.paths, inputs.count, &filter, + overwrite_existing, actual_path, sizeof(actual_path), + &ctx); done: Py_DECREF(output_path_bytes); path_list_converter(NULL, &inputs); + pattern_list_converter(NULL, &include); + pattern_list_converter(NULL, &exclude); return finish_call(self, result, &ctx, actual_path); } @@ -430,6 +486,8 @@ static PyObject *py_extract_archive(PyObject *self, PyObject *args, "threads", "progress", "cancel", + "include", + "exclude", NULL}; PyObject *archive_path_bytes = NULL; @@ -438,6 +496,8 @@ static PyObject *py_extract_archive(PyObject *self, PyObject *args, int threads = 0; // One per core PyObject *progress = NULL; PyObject *cancel = NULL; + PathList include = {0}; + PathList exclude = {0}; // Anything the caller does not pass keeps its default ExtractionPolicy policy = extraction_policy_default(); @@ -448,14 +508,18 @@ static PyObject *py_extract_archive(PyObject *self, PyObject *args, unsigned int max_depth = (unsigned int)policy.max_depth; if (!PyArg_ParseTupleAndKeywords( - args, kwargs, "O&O&O|$iKIpppipiOO", kwlist, PyUnicode_FSConverter, - &archive_path_bytes, PyUnicode_FSConverter, &output_dir_bytes, - &files_obj, &policy.overwrite_existing, &max_total_size, &max_depth, - &policy.preserve_permissions, &policy.exact_permissions, - &policy.preserve_timestamps, &policy.allow_symlinks, - &policy.overwrite_dir_metadata, &threads, &progress, &cancel)) { + args, kwargs, "O&O&O|$iKIpppipiOOO&O&", kwlist, + PyUnicode_FSConverter, &archive_path_bytes, PyUnicode_FSConverter, + &output_dir_bytes, &files_obj, &policy.overwrite_existing, + &max_total_size, &max_depth, &policy.preserve_permissions, + &policy.exact_permissions, &policy.preserve_timestamps, + &policy.allow_symlinks, &policy.overwrite_dir_metadata, &threads, + &progress, &cancel, pattern_list_converter, &include, + pattern_list_converter, &exclude)) { return NULL; // Error already set } + PathFilter filter = {include.paths, include.count, exclude.paths, + exclude.count}; policy.max_total_size = (uint64_t)max_total_size; policy.max_depth = (uint32_t)max_depth; @@ -521,11 +585,13 @@ static PyObject *py_extract_archive(PyObject *self, PyObject *args, result = extract_archive(PyBytes_AS_STRING(archive_path_bytes), PyBytes_AS_STRING(output_dir_bytes), files, - num_files, &policy, &ctx); + num_files, &filter, &policy, &ctx); done: Py_DECREF(archive_path_bytes); Py_DECREF(output_dir_bytes); + pattern_list_converter(NULL, &include); + pattern_list_converter(NULL, &exclude); free(files); Py_XDECREF(files_tuple); return finish_call(self, result, &ctx, NULL); diff --git a/src/compresso/csrc/archives.h b/src/compresso/csrc/archives.h index bacaa4d..fd4aba2 100644 --- a/src/compresso/csrc/archives.h +++ b/src/compresso/csrc/archives.h @@ -157,19 +157,31 @@ int pipeline_is_valid(const CompressionPipeline *p); // ---- High-Level Operations ---- +// Globs on entry names: an exclude wins, and with any includes an entry must +// match one, or be a directory holding one that does (when archiving) or a +// requested name (when extracting) +typedef struct { + const char **include; + size_t num_include; + const char **exclude; + size_t num_exclude; +} PathFilter; + // `overwrite_existing` follows the same 0-3 scheme as // `ExtractionPolicy.overwrite_existing`; on success, the path actually written // (which RENAME may have changed) is copied into `out_actual_path` int create_archive(const char *output_path, const CompressionPipeline *pipeline, const char **input_paths, size_t num_paths, - int overwrite_existing, char *out_actual_path, - size_t out_actual_path_size, CoreContext *ctx); + const PathFilter *filter, int overwrite_existing, + char *out_actual_path, size_t out_actual_path_size, + CoreContext *ctx); // Entries are checked against `policy`; any failure but a cancel undoes what // the extraction wrote, restoring what OVERWRITE replaced int extract_archive(const char *archive_path, const char *output_dir, const char **files, size_t num_files, - const ExtractionPolicy *policy, CoreContext *ctx); + const PathFilter *filter, const ExtractionPolicy *policy, + CoreContext *ctx); // `ctx` carries signal checks and warnings; there is no progress to report PyObject *list_archive_contents(const char *archive_path, CoreContext *ctx); diff --git a/src/compresso/csrc/archives/archive_ops.c b/src/compresso/csrc/archives/archive_ops.c index 9326dbe..351542c 100644 --- a/src/compresso/csrc/archives/archive_ops.c +++ b/src/compresso/csrc/archives/archive_ops.c @@ -81,32 +81,9 @@ static int check_storable_entry_name(const char *name) { // A walked path's entry name, or NULL with an exception set static char *entry_name(const WalkItem *item) { - const char *rel_path = item->path + item->prefix_len; - while (FS_IS_SEP(*rel_path)) - rel_path++; - - char *name = strdup(rel_path); - if (!name) { - PyErr_NoMemory(); + char *name = walk_entry_name(item->path, item->prefix_len); + if (!name) return NULL; - } - - // Normalise to '/', collapsing runs and dropping a trailing one; writers add - // the trailing separator a directory needs - char *w = name; - for (const char *r = name; *r; r++) { - if (FS_IS_SEP(*r)) { - // Keep a leading separator so the absolute name is still rejected below - if (w > name && w[-1] == '/') - continue; - *w++ = '/'; - } else { - *w++ = *r; - } - } - while (w > name + 1 && w[-1] == '/') - w--; - *w = '\0'; if (check_storable_entry_name(name) != 0) { free(name); @@ -210,8 +187,9 @@ static int add_walked_entries(const CArchive *archive, void *writer, int create_archive(const char *output_path, const CompressionPipeline *pipeline, const char **input_paths, size_t num_paths, - int overwrite_existing, char *out_actual_path, - size_t out_actual_path_size, CoreContext *ctx) { + const PathFilter *filter, int overwrite_existing, + char *out_actual_path, size_t out_actual_path_size, + CoreContext *ctx) { if (!pipeline_is_valid(pipeline) || pipeline->archive == ARCHIVE_NONE) { char name[32]; pipeline_display_name(pipeline, name, sizeof(name)); @@ -276,8 +254,9 @@ int create_archive(const char *output_path, const CompressionPipeline *pipeline, own_files_add(&own, output_path); WalkList walked = {0}; - int ret = walk_sources(input_paths, num_paths, &own, archive->stores_specials, - archive->stores_hardlinks, ctx, &walked); + int ret = walk_sources(input_paths, num_paths, filter, &own, + archive->stores_specials, archive->stores_hardlinks, + ctx, &walked); if (ret == 0) { ctx_begin_stage(ctx, walked.file_bytes); ret = add_walked_entries(archive, writer, &walked, ctx); diff --git a/src/compresso/csrc/archives/extract.c b/src/compresso/csrc/archives/extract.c index 1975540..d50d3f3 100644 --- a/src/compresso/csrc/archives/extract.c +++ b/src/compresso/csrc/archives/extract.c @@ -727,7 +727,8 @@ static int extract_entries(const CArchive *archive, void *reader, int extract_archive(const char *archive_path, const char *output_dir, const char **files, size_t num_files, - const ExtractionPolicy *policy, CoreContext *ctx) { + const PathFilter *filter, const ExtractionPolicy *policy, + CoreContext *ctx) { const CArchive *archive; CompressionPipeline pipe; FILE *src = open_archive_source(archive_path, &archive, &pipe, ctx); @@ -763,7 +764,7 @@ int extract_archive(const char *archive_path, const char *output_dir, Selection sel; PyObject *skipped_links = PyList_New(0); - if (!skipped_links || selection_init(&sel, files, num_files) != 0) { + if (!skipped_links || selection_init(&sel, files, num_files, filter) != 0) { Py_XDECREF(skipped_links); if (src) fclose(src); @@ -828,5 +829,6 @@ int extract_archive(const char *archive_path, const char *output_dir, ret = report_incomplete(&sel, skipped_links, ctx); Py_DECREF(skipped_links); free(sel.items); + free(sel.include_found); return ret; } diff --git a/src/compresso/csrc/archives/select.c b/src/compresso/csrc/archives/select.c index 322899f..e44e79f 100644 --- a/src/compresso/csrc/archives/select.c +++ b/src/compresso/csrc/archives/select.c @@ -11,14 +11,23 @@ static int selected_name_cmp(const void *a, const void *b) { return c ? c : (x->len > y->len) - (x->len < y->len); } -int selection_init(Selection *sel, const char **files, size_t num_files) { - *sel = (Selection){0}; +int selection_init(Selection *sel, const char **files, size_t num_files, + const PathFilter *filter) { + *sel = (Selection){.filter = *filter}; + if (filter->num_include > 0) { + sel->include_found = safe_malloc(filter->num_include); + if (!sel->include_found) + return -1; + memset(sel->include_found, 0, filter->num_include); + } if (num_files == 0) return 0; sel->items = safe_malloc(num_files * sizeof(*sel->items)); - if (!sel->items) + if (!sel->items) { + free(sel->include_found); return -1; + } for (size_t i = 0; i < num_files; i++) { size_t len = strlen(files[i]); @@ -39,11 +48,9 @@ int selection_init(Selection *sel, const char **files, size_t num_files) { } int entry_is_selected(const ArchiveEntry *entry, Selection *sel) { - if (sel->count == 0) - return 1; - - int selected = 0; - for (const char *p = entry->path;; p++) { + const PathFilter *filter = &sel->filter; + int selected = sel->count == 0 && filter->num_include == 0; + for (const char *p = entry->path; sel->count > 0; p++) { if ((*p == '/' || *p == '\0') && p > entry->path) { SelectedName key = {entry->path, (size_t)(p - entry->path), 0}; SelectedName *hit = bsearch(&key, sel->items, sel->count, @@ -54,8 +61,20 @@ int entry_is_selected(const ArchiveEntry *entry, Selection *sel) { } } if (*p == '\0') - return selected; + break; } + + for (size_t i = 0; i < filter->num_include; i++) + if ((!selected || !sel->include_found[i]) && + path_matches(entry->path, filter->include[i])) { + sel->include_found[i] = 1; + selected = 1; + } + + // Excludes come last, so dropped entries still counts as found and aren't + // reported as missing from the archive + return selected && + !path_matches_any(entry->path, filter->exclude, filter->num_exclude); } // The pattern after its first token if that token matches `c`, else NULL @@ -130,6 +149,13 @@ int path_matches(const char *path, const char *pat) { return 0; } +int path_matches_any(const char *path, const char **pats, size_t count) { + for (size_t i = 0; i < count; i++) + if (path_matches(path, pats[i])) + return 1; + return 0; +} + static PyObject *join_str(const char *sep, PyObject *items) { PyObject *s = PyUnicode_FromString(sep); PyObject *joined = s ? PyUnicode_Join(s, items) : NULL; @@ -167,6 +193,15 @@ int report_incomplete(const Selection *sel, PyObject *skipped_links, if (rc != 0) goto done; } + for (size_t i = 0; i < sel->filter.num_include; i++) { + if (sel->include_found[i]) + continue; + PyObject *pattern = PyUnicode_FromString(sel->filter.include[i]); + int rc = pattern ? PyList_Append(names, pattern) : -1; + Py_XDECREF(pattern); + if (rc != 0) + goto done; + } if (append_part(parts, "Not found in archive", names) != 0 || append_part(parts, "Hardlink target missing", skipped_links) != 0) goto done; diff --git a/src/compresso/csrc/archives/select.h b/src/compresso/csrc/archives/select.h index 8e543de..ff5cabc 100644 --- a/src/compresso/csrc/archives/select.h +++ b/src/compresso/csrc/archives/select.h @@ -4,7 +4,8 @@ #include "../archives.h" #include -// The requested names, sorted for bsearch; none means everything +// The requested names, sorted for bsearch; with no includes, none means +// everything typedef struct { const char *name; size_t len; // Without trailing slashes @@ -14,13 +15,18 @@ typedef struct { typedef struct { SelectedName *items; size_t count; + PathFilter filter; + char *include_found; // One flag per include; NULL with none } Selection; -// Borrows `files`, which must outlive `sel`; free `sel->items` when done -int selection_init(Selection *sel, const char **files, size_t num_files); +// Borrows `files` and the filter's patterns, which must outlive `sel`; free +// `sel->items` and `sel->include_found` when done +int selection_init(Selection *sel, const char **files, size_t num_files, + const PathFilter *filter); -// Whether `entry` or a directory above it was requested, as tar matches names; -// marks every name that matches, so "d" and "d/x" both count as found +// Whether `entry` or a directory above it was requested, as tar matches names, +// or an include matches it, and no exclude does; marks every name and include +// that matches, so "d" and "d/x" both count as found int entry_is_selected(const ArchiveEntry *entry, Selection *sel); // Whether the glob `pat` matches any run of whole components in `path`: e.g. @@ -28,7 +34,10 @@ int entry_is_selected(const ArchiveEntry *entry, Selection *sel); // is fnmatch(3)'s with no flags, so `*` and `?` also match '/' int path_matches(const char *path, const char *pat); -// Raises an error listing unmatched requested names and skipped hardlinks +// Whether any of `count` patterns matches `path` +int path_matches_any(const char *path, const char **pats, size_t count); + +// Raises an error listing unmatched names and includes, and skipped hardlinks int report_incomplete(const Selection *sel, PyObject *skipped_links, CoreContext *ctx); diff --git a/src/compresso/csrc/archives/walk.c b/src/compresso/csrc/archives/walk.c index 1409920..cbaa4e1 100644 --- a/src/compresso/csrc/archives/walk.c +++ b/src/compresso/csrc/archives/walk.c @@ -2,6 +2,7 @@ #include #include "walk.h" #include "../common.h" +#include "select.h" #include #include @@ -50,10 +51,8 @@ static int push(WalkList *list, char *path, size_t prefix_len) { static int is_skipped(const WalkItem *item, const OwnFiles *own, int keep_specials, CoreContext *ctx) { - if (item->st.type == FS_TYPE_OTHER && - !(keep_specials && item->st.special)) { - ctx_log(ctx, CTX_LOG_WARNING, "Skipped %s: %s are not archived", - item->path, + if (item->st.type == FS_TYPE_OTHER && !(keep_specials && item->st.special)) { + ctx_log(ctx, CTX_LOG_WARNING, "Skipped %s: %s are not archived", item->path, keep_specials ? "sockets" : "FIFOs, sockets and devices"); return 1; } @@ -141,8 +140,7 @@ static int find_hardlinks(WalkList *list) { for (size_t i = 1; i < n; i++) { if (!fs_same_file(&linked[i]->st, &linked[i - 1]->st)) continue; - linked[i]->link = - linked[i - 1]->link ? linked[i - 1]->link : linked[i - 1]; + linked[i]->link = linked[i - 1]->link ? linked[i - 1]->link : linked[i - 1]; if (linked[i]->st.type == FS_TYPE_FILE) list->file_bytes -= linked[i]->st.size; } @@ -150,9 +148,68 @@ static int find_hardlinks(WalkList *list) { return 0; } +char *walk_entry_name(const char *path, size_t prefix_len) { + const char *rel_path = path + prefix_len; + while (FS_IS_SEP(*rel_path)) + rel_path++; + + char *name = strdup(rel_path); + if (!name) { + PyErr_NoMemory(); + return NULL; + } + + // Normalise to '/', collapsing runs and dropping a trailing one; writers add + // the trailing separator a directory needs + char *w = name; + for (const char *r = name; *r; r++) { + if (FS_IS_SEP(*r)) { + // Keep a leading separator so the absolute name is still rejected + if (w > name && w[-1] == '/') + continue; + *w++ = '/'; + } else { + *w++ = *r; + } + } + while (w > name + 1 && w[-1] == '/') + w--; + *w = '\0'; + return name; +} + +// Whether `path` is below the directory `dir`, as fs_join builds child paths +static int is_below(const char *path, const char *dir) { + size_t n = strlen(dir); + return strncmp(path, dir, n) == 0 && + (FS_IS_SEP(path[n]) || (n > 0 && FS_IS_SEP(dir[n - 1]))); +} + +// Drops each provisional directory with nothing kept below it; depth-first, so +// the nearest kept item after a directory is below it if anything is +static void prune_provisional(WalkList *list) { + const char *next_kept = NULL; + for (size_t i = list->count; i-- > 0;) { + WalkItem *item = &list->items[i]; + if (item->provisional && !(next_kept && is_below(next_kept, item->path))) { + free(item->path); + item->path = NULL; + } else { + next_kept = item->path; + } + } + + size_t kept = 0; + for (size_t i = 0; i < list->count; i++) + if (list->items[i].path) + list->items[kept++] = list->items[i]; + list->count = kept; +} + int walk_sources(const char **input_paths, size_t num_paths, - const OwnFiles *own, int keep_specials, int link_hardlinks, - CoreContext *ctx, WalkList *out) { + const PathFilter *filter, const OwnFiles *own, + int keep_specials, int link_hardlinks, CoreContext *ctx, + WalkList *out) { // Depth-first like the archive's order, on a heap stack rather than the C one WalkList pending = {0}; int ret = 0; @@ -172,13 +229,35 @@ int walk_sources(const char **input_paths, size_t num_paths, } WalkItem item = pending.items[--pending.count]; + + // Matched before the stat, so an excluded tree costs nothing to skip + int included = 1; + if (filter->num_include > 0 || filter->num_exclude > 0) { + char *name = walk_entry_name(item.path, item.prefix_len); + if (!name) { + free(item.path); + ret = -1; + break; + } + int excluded = + path_matches_any(name, filter->exclude, filter->num_exclude); + included = filter->num_include == 0 || + path_matches_any(name, filter->include, filter->num_include); + free(name); + if (excluded) { + free(item.path); + continue; + } + } + if (fs_stat_path(item.path, &item.st) != 0) { PyErr_SetFromErrnoWithFilename(PyExc_OSError, item.path); free(item.path); ret = -1; break; } - if (is_skipped(&item, own, keep_specials, ctx)) { + if ((!included && item.st.type != FS_TYPE_DIR) || + is_skipped(&item, own, keep_specials, ctx)) { free(item.path); continue; } @@ -187,6 +266,7 @@ int walk_sources(const char **input_paths, size_t num_paths, if (ret != 0) break; out->items[out->count - 1].st = item.st; + out->items[out->count - 1].provisional = !included; if (item.st.type == FS_TYPE_DIR) ret = push_children(&pending, &item); @@ -195,6 +275,8 @@ int walk_sources(const char **input_paths, size_t num_paths, } walk_list_free(&pending); + if (ret == 0) + prune_provisional(out); if (ret == 0 && link_hardlinks) ret = find_hardlinks(out); return ret; diff --git a/src/compresso/csrc/archives/walk.h b/src/compresso/csrc/archives/walk.h index 08a3e5b..4e394ef 100644 --- a/src/compresso/csrc/archives/walk.h +++ b/src/compresso/csrc/archives/walk.h @@ -1,6 +1,7 @@ #ifndef WALK_H #define WALK_H +#include "../archives.h" #include "../context.h" #include "../fsutil.h" #include @@ -20,6 +21,7 @@ typedef struct WalkItem { size_t prefix_len; // Its input's parent prefix, which entry names drop fs_stat st; const struct WalkItem *link; // The file's first name, if this is a later one + int provisional; // A directory kept only if something below it is } WalkItem; typedef struct { @@ -28,14 +30,20 @@ typedef struct { uint64_t file_bytes; // The regular files' total, for progress } WalkList; -// Every input and everything below it, depth-first, symlinks not followed. -// Warns and skips `own` and sockets, and FIFOs and devices unless -// `keep_specials`; with `link_hardlinks`, a file's later names get `link` and -// aren't counted in `file_bytes`; returns COMP_CANCELLED, or -1 with an -// exception set +// The entry name of `path`, found below an input whose parent prefix is +// `prefix_len` long: '/'-separated, with runs collapsed and no trailing one; +// NULL with an exception set +char *walk_entry_name(const char *path, size_t prefix_len); + +// Every input and everything below it that `filter` keeps, depth-first, +// symlinks not followed; an excluded directory isn't descended into; warns and +// skips `own` and sockets, and FIFOs and devices unless `keep_specials`; with +// `link_hardlinks`, a file's later names get `link` and aren't counted in +// `file_bytes`; returns COMP_CANCELLED, or -1 with an exception set int walk_sources(const char **input_paths, size_t num_paths, - const OwnFiles *own, int keep_specials, int link_hardlinks, - CoreContext *ctx, WalkList *out); + const PathFilter *filter, const OwnFiles *own, + int keep_specials, int link_hardlinks, CoreContext *ctx, + WalkList *out); void walk_list_free(WalkList *list); diff --git a/tests/c/test_select.c b/tests/c/test_select.c index e28dddf..828bbe5 100644 --- a/tests/c/test_select.c +++ b/tests/c/test_select.c @@ -40,8 +40,8 @@ TEST_RANGE([ 0, 6, 1 ]) void test_names_select_whole_components(int index) { const SelectCase *c = &SELECT_CASES[index]; Selection sel; - TEST_ASSERT_EQUAL_INT( - 0, selection_init(&sel, (const char **)c->files, c->num_files)); + TEST_ASSERT_EQUAL_INT(0, selection_init(&sel, (const char **)c->files, + c->num_files, &(PathFilter){0})); char selected[128] = ""; for (size_t i = 0; i < sizeof(NAMES) / sizeof(NAMES[0]); i++) { diff --git a/tests/python/test_archive_create.py b/tests/python/test_archive_create.py index c5a3c3e..d4814f6 100644 --- a/tests/python/test_archive_create.py +++ b/tests/python/test_archive_create.py @@ -264,6 +264,67 @@ def test_entries_follow_their_directory(self, tmp_path: Path) -> None: assert listed[i + 1 : i + 1 + len(inside)] == inside, listed +class TestArchiveFilter: + """Test that `include` and `exclude` narrow the walk.""" + + @pytest.fixture + def tree(self, tmp_path: Path) -> Path: + """A tree with text, logs, a node_modules and an empty directory.""" + for rel in ("t/a/keep.txt", "t/a/drop.log", "t/a/node_modules/m.txt", "t/b/n"): + (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / rel).write_text(rel) + (tmp_path / "t/empty").mkdir() + return tmp_path / "t" + + @pytest.mark.parametrize( + ("include", "exclude", "expected"), + [ + ( + [], + ["node_modules", "*.log"], + ["t/", "t/a/", "t/a/keep.txt", "t/b/", "t/b/n", "t/empty/"], + ), + ( + ["*.txt"], + [], + [ + "t/", + "t/a/", + "t/a/keep.txt", + "t/a/node_modules/", + "t/a/node_modules/m.txt", + ], + ), + (["*.txt"], ["node_modules"], ["t/", "t/a/", "t/a/keep.txt"]), + ], + ids=["exclude", "include-keeps-holding-dirs", "exclude-wins"], + ) + def test_entries_kept( + self, + tree: Path, + tmp_path: Path, + include: list[str], + exclude: list[str], + expected: list[str], + ) -> None: + """Test which entries a filter stores.""" + output = tmp_path / "out.tar" + + _core.create_archive( + str(output), "tar", [str(tree)], include=include, exclude=exclude + ) + + listed = [e["path"] for e in _core.list_archive_contents(str(output))] + assert sorted(listed) == expected + + def test_bare_string_is_refused(self, tree: Path, tmp_path: Path) -> None: + """Test that a str isn't taken as a sequence of one-character patterns.""" + with pytest.raises(TypeError, match="sequence of str, not str"): + _core.create_archive( + str(tmp_path / "out.tar"), "tar", [str(tree)], include="*.txt" + ) + + class TestSizeChange: """Test a file whose size changes between the walk and its copy.""" diff --git a/tests/python/test_archive_extract.py b/tests/python/test_archive_extract.py index 6d04703..688df7c 100644 --- a/tests/python/test_archive_extract.py +++ b/tests/python/test_archive_extract.py @@ -132,6 +132,34 @@ def test_selects_at_component_boundaries( extracted = [p.relative_to(out).as_posix() for p in out.rglob("*")] assert sorted(p for p in extracted if (out / p).is_file()) == expected + @pytest.mark.parametrize( + ("files", "include", "exclude", "expected"), + [ + ([], ["*.txt"], ["sub"], ["d/x.txt", "dx.txt", "e.txt"]), + (["d"], [], ["y.txt"], ["d/x.txt"]), + (["d"], ["e.txt"], ["y.txt"], ["d/x.txt", "e.txt"]), + ], + ids=["include-and-exclude", "with-files", "files-or-include"], + ) + def test_filter_shapes_the_selection( + self, + archive: Path, + tmp_path: Path, + files: list[str], + include: list[str], + exclude: list[str], + expected: list[str], + ) -> None: + """Test that includes add to `files`, as in bsdtar, and excludes override both.""" + out = tmp_path / "out" + + _core.extract_archive( + str(archive), str(out), files, include=include, exclude=exclude + ) + + extracted = [p.relative_to(out).as_posix() for p in out.rglob("*")] + assert sorted(p for p in extracted if (out / p).is_file()) == expected + def test_unmatched_names_are_reported_after_the_rest( self, archive: Path, tmp_path: Path ) -> None: @@ -145,6 +173,46 @@ def test_unmatched_names_are_reported_after_the_rest( assert [p.name for p in out.iterdir()] == ["e.txt"] + def test_unmatched_includes_are_reported_after_the_rest( + self, archive: Path, tmp_path: Path + ) -> None: + """Test that an include matching nothing raises, as bsdtar's does.""" + out = tmp_path / "out" + + with pytest.raises( + IncompleteExtractionError, match=r"Not found in archive: nope, zzz\*$" + ): + _core.extract_archive( + str(archive), str(out), [], include=["e.txt", "nope", "zzz*"] + ) + + assert [p.name for p in out.iterdir()] == ["e.txt"] + + @pytest.mark.parametrize( + ("files", "include", "exclude"), + [ + (["e.txt"], [], ["*"]), + ([], ["e.txt"], ["*"]), + ], + ids=["excluded-name", "excluded-include"], + ) + def test_filtered_out_matches_are_not_reported( + self, + archive: Path, + tmp_path: Path, + files: list[str], + include: list[str], + exclude: list[str], + ) -> None: + """Test that what the filter drops still counts as found, as in tar.""" + out = tmp_path / "out" + + _core.extract_archive( + str(archive), str(out), files, include=include, exclude=exclude + ) + + assert not [p for p in out.rglob("*") if p.is_file()] + class TestRecognisedButUnsupportedArchive: """Test that a detected archive format without a backend names itself.""" From 68acf7ea575dcb3478620eac0c13145f172a016a Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Sat, 31 Oct 2026 14:09:26 +0000 Subject: [PATCH 22/31] feat(cli): filter archive entries with --include and --exclude flags - Adds both flags to 'compress' and 'extract' commands, wiring them to 'ArchiveOptions'/'ExtractOptions' and the C filter - A single-file input or output refuses them, since a glob has nothing to select in a one-entry container - The plan now applies the same globs, prunes excluded directories, and then lstats each file - Now counts hardlinked files once where the format stores links, so the progress bar's total matches the archive - path_matches() and format_stores_hardlinks() are exported from '_core' so the plan matches entry names exactly as the walk and the extractor do - plan_extraction() mirrors the extractor,with name or include selecting, exclude overrides both, so planned entries are the ones written - A run whose excludes leave nothing still exits 0, but informs the user through new report_warning() (unless '-q') - Moves the parenthesised defaults in '-o' and '--symlinks' help into 'show_default' --- src/compresso/_core.pyi | 6 ++ src/compresso/cli/_render/__init__.py | 10 ++ src/compresso/cli/compress.py | 23 ++++- src/compresso/cli/extract.py | 33 ++++++- src/compresso/csrc/_core.c | 66 ++++++++++--- src/compresso/frontend/archive_api.py | 132 +++++++++++++++++++++----- src/compresso/frontend/dispatch.py | 18 +++- tests/python/cli/test_compress.py | 31 ++++++ tests/python/cli/test_extract.py | 80 +++++++++++++++- tests/python/test_archive_create.py | 100 ++++++++++++++++++- tests/python/test_archive_extract.py | 34 +++++++ 11 files changed, 489 insertions(+), 44 deletions(-) diff --git a/src/compresso/_core.pyi b/src/compresso/_core.pyi index c105652..a7aa8f9 100644 --- a/src/compresso/_core.pyi +++ b/src/compresso/_core.pyi @@ -221,6 +221,12 @@ def gzip_stored_name(file_path: StrOrBytesPath) -> bytes | None: def format_is_archive(format: str) -> bool: """Return whether a format's container can hold more than one entry.""" +def format_stores_hardlinks(format: str) -> bool: + """Return whether an archive format stores a file's later names as links.""" + +def path_matches(path: str, patterns: Sequence[str]) -> bool: + """Return whether any glob matches an entry path, as `include` and `exclude` do.""" + class _StandaloneFormatDict(TypedDict): """One single file container format, as defined by `standalone_formats`.""" diff --git a/src/compresso/cli/_render/__init__.py b/src/compresso/cli/_render/__init__.py index a61925f..1e6e5c8 100644 --- a/src/compresso/cli/_render/__init__.py +++ b/src/compresso/cli/_render/__init__.py @@ -77,6 +77,16 @@ def report_error(message: str) -> None: app.echo(message=app.style(text=f"✗ {message}", fg="red"), err=True) +def report_warning(message: str) -> None: + """Print a warning to stderr. + + Args: + message: Text to show, without the leading marker. + """ + sys.stdout.flush() + app.echo(message=app.style(text=f"! {message}", fg="yellow"), err=True) + + def fail(message: str, code: int = EXIT_FAILED) -> NoReturn: """Print an error to stderr and exit. diff --git a/src/compresso/cli/compress.py b/src/compresso/cli/compress.py index 311d6e6..2cff952 100644 --- a/src/compresso/cli/compress.py +++ b/src/compresso/cli/compress.py @@ -121,7 +121,7 @@ def compress( output: Annotated[ Path | None, app.Option( - "--output", "-o", help="Output path (default: named after the input)" + "--output", "-o", help="Output path", show_default="named after the input" ), ] = None, format: Annotated[ @@ -161,6 +161,21 @@ def compress( help="Compression level; each algorithm's range is shown by `list`", ), ] = None, + include: Annotated[ + list[str] | None, + app.Option( + "--include", + help="Archive only what matches this, and the dirs holding it; repeatable", + ), + ] = None, + exclude: Annotated[ + list[str] | None, + app.Option( + "--exclude", + help="Leave out what matches this glob and all below it, overriding " + "--include; repeatable", + ), + ] = None, on_conflict: Annotated[ OverwriteMode | None, app.Option( @@ -216,6 +231,10 @@ def compress( strategy: Strategy for the Compresso container (default: balanced); any other format refuses it, as does `-a`. level: Compression level (default: the format's own). + include: Globs an archive's entries must match (default: all); + a single-file output refuses it. + exclude: Globs an archive's entries must not match; a single-file + output refuses it. on_conflict: What to do when the output exists (default: rename). force: If True, replace an existing output. threads: Worker threads for zstd and xz (default: one per core). @@ -247,6 +266,8 @@ def compress( level=level, overwrite=overwrite_mode, threads=threads, + include=include or (), + exclude=exclude or (), ) try: diff --git a/src/compresso/cli/extract.py b/src/compresso/cli/extract.py index 6798f5f..88a92c5 100644 --- a/src/compresso/cli/extract.py +++ b/src/compresso/cli/extract.py @@ -17,7 +17,7 @@ ) from ..frontend.dispatch import ExtractRequest, plan_extract from ._app import app -from ._render import EXIT_USAGE, cancelled, fail +from ._render import EXIT_USAGE, cancelled, fail, report_warning from ._render.jobs import ( decompress_plan, decompress_result, @@ -77,6 +77,10 @@ def _extract_archive(job: ExtractJob, quiet: bool, verbose: bool) -> None: job, plan.archive, "Extracting", total, quiet=quiet, what="Extraction" ) + # Names and includes the filter dropped still count as found + if plan.options.exclude and not plan.entries and not quiet: + report_warning(f"{plan.archive}: nothing left to extract") + if verbose: extract_result(plan, elapsed) @@ -170,6 +174,22 @@ def extract( "repeatable", ), ] = None, + include: Annotated[ + list[str] | None, + app.Option( + "--include", + help="Extract entries matching this glob, as well as any --entry; " + "repeatable", + ), + ] = None, + exclude: Annotated[ + list[str] | None, + app.Option( + "--exclude", + help="Skip entries matching this glob, overriding --entry and " + "--include; repeatable", + ), + ] = None, on_conflict: Annotated[ OverwriteMode | None, app.Option( @@ -228,9 +248,8 @@ def extract( app.Option( "--symlinks", case_sensitive=False, - help="Refuse symlinks, create those staying inside the output, " - "or write each as a file holding its target " - "(default: file for zip, deny otherwise)", + help="Refuse, create (non-escaping), or write as a file", + show_default="file for zip, deny otherwise", ), ] = None, threads: Annotated[ @@ -286,6 +305,10 @@ def extract( entry: Entry paths to extract from each archive (default: all); a path matching no entry fails that archive after the rest is extracted. + include: Globs selecting entries as well as any `entry` names; one + matching no entry fails that archive likewise. + exclude: Globs dropping entries, overriding `entry` and `include`; a + run left with nothing to extract warns, but still succeeds. on_conflict: What to do when a file exists (default: rename). force: If True, replace files that already exist. max_size: Cap on total extracted bytes (default: no cap). @@ -332,6 +355,8 @@ def extract( overwrite_dir_metadata=overwrite_dir, symlinks=symlinks, threads=threads, + include=include or (), + exclude=exclude or (), ) request = ExtractRequest( diff --git a/src/compresso/csrc/_core.c b/src/compresso/csrc/_core.c index ae2cca0..ac867a7 100644 --- a/src/compresso/csrc/_core.c +++ b/src/compresso/csrc/_core.c @@ -1,5 +1,6 @@ #define PY_SSIZE_T_CLEAN #include +#include "archives/select.h" #include "codec/crc32c.h" #include "common.h" #include "fsutil.h" @@ -457,10 +458,9 @@ static PyObject *py_create_archive(PyObject *self, PyObject *args, } char actual_path[FS_PATH_MAX]; - result = create_archive(PyBytes_AS_STRING(output_path_bytes), &pipe, - inputs.paths, inputs.count, &filter, - overwrite_existing, actual_path, sizeof(actual_path), - &ctx); + result = create_archive( + PyBytes_AS_STRING(output_path_bytes), &pipe, inputs.paths, inputs.count, + &filter, overwrite_existing, actual_path, sizeof(actual_path), &ctx); done: Py_DECREF(output_path_bytes); @@ -508,14 +508,13 @@ static PyObject *py_extract_archive(PyObject *self, PyObject *args, unsigned int max_depth = (unsigned int)policy.max_depth; if (!PyArg_ParseTupleAndKeywords( - args, kwargs, "O&O&O|$iKIpppipiOOO&O&", kwlist, - PyUnicode_FSConverter, &archive_path_bytes, PyUnicode_FSConverter, - &output_dir_bytes, &files_obj, &policy.overwrite_existing, - &max_total_size, &max_depth, &policy.preserve_permissions, - &policy.exact_permissions, &policy.preserve_timestamps, - &policy.allow_symlinks, &policy.overwrite_dir_metadata, &threads, - &progress, &cancel, pattern_list_converter, &include, - pattern_list_converter, &exclude)) { + args, kwargs, "O&O&O|$iKIpppipiOOO&O&", kwlist, PyUnicode_FSConverter, + &archive_path_bytes, PyUnicode_FSConverter, &output_dir_bytes, + &files_obj, &policy.overwrite_existing, &max_total_size, &max_depth, + &policy.preserve_permissions, &policy.exact_permissions, + &policy.preserve_timestamps, &policy.allow_symlinks, + &policy.overwrite_dir_metadata, &threads, &progress, &cancel, + pattern_list_converter, &include, pattern_list_converter, &exclude)) { return NULL; // Error already set } PathFilter filter = {include.paths, include.count, exclude.paths, @@ -820,6 +819,43 @@ static PyObject *py_format_is_archive(PyObject *Py_UNUSED(self), pipe.archive != ARCHIVE_NONE); } +static PyObject *py_format_stores_hardlinks(PyObject *Py_UNUSED(self), + PyObject *args) { + const char *format_name = NULL; + + if (!PyArg_ParseTuple(args, "s", &format_name)) { + return NULL; // Error already set + } + + CompressionPipeline pipe = pipeline_from_name(format_name, -1); + const CArchive *archive = + pipeline_is_valid(&pipe) ? find_archive_by_id(pipe.archive) : NULL; + + return PyBool_FromLong(archive && archive->stores_hardlinks); +} + +static PyObject *py_path_matches(PyObject *Py_UNUSED(self), PyObject *args) { + PyObject *path = NULL; + PathList patterns = {0}; + + if (!PyArg_ParseTuple(args, "UO&", &path, pattern_list_converter, + &patterns)) { + return NULL; // Error already set + } + + // The inverse of decode_entry_name, so a non-UTF-8 name keeps its bytes + PyObject *encoded = + PyUnicode_AsEncodedString(path, "utf-8", "surrogateescape"); + PyObject *result = + encoded ? PyBool_FromLong(path_matches_any( + PyBytes_AS_STRING(encoded), patterns.paths, patterns.count)) + : NULL; + + Py_XDECREF(encoded); + pattern_list_converter(NULL, &patterns); + return result; +} + // The set entries of a FormatDesc name slot, which fill from the front static PyObject *names_to_py(const char *const names[2]) { if (!names[0]) @@ -959,6 +995,12 @@ static PyMethodDef CoreMethods[] = { "Detect the format of a file."}, {"gzip_stored_name", (PyCFunction)py_gzip_stored_name, METH_VARARGS, "Return the name a gzip header stores (FNAME), or None."}, + {"format_stores_hardlinks", (PyCFunction)py_format_stores_hardlinks, + METH_VARARGS, + "Return whether an archive format stores a file's later names as links."}, + {"path_matches", (PyCFunction)py_path_matches, METH_VARARGS, + "Return whether any glob matches an entry path, as the archive filters " + "match it."}, {"format_is_archive", (PyCFunction)py_format_is_archive, METH_VARARGS, "Check if a format is an archive format."}, {"standalone_formats", (PyCFunction)py_standalone_formats, METH_NOARGS, diff --git a/src/compresso/frontend/archive_api.py b/src/compresso/frontend/archive_api.py index 24613ba..643c9a5 100644 --- a/src/compresso/frontend/archive_api.py +++ b/src/compresso/frontend/archive_api.py @@ -2,7 +2,9 @@ from __future__ import annotations -from collections.abc import Sequence +import os +import stat +from collections.abc import Iterator, Sequence from dataclasses import dataclass, replace from enum import StrEnum from pathlib import Path @@ -15,7 +17,9 @@ create_archive, extract_archive, format_is_archive, + format_stores_hardlinks, list_archive_contents, + path_matches, ) from .._levels import to_core_level from ._formats import DEFAULT_ARCHIVE_FORMAT, infer_format @@ -117,12 +121,18 @@ class ArchiveOptions: overwrite: What to do when the destination archive already exists. threads: Worker threads for tar.zst and tar.xz, or 0 for one per core; the output is the same for any count. + include: Globs on entry names; with any, only what matches is stored, + plus the directories holding it. + exclude: Globs on entry names that aren't stored or descended into, + overriding `include`. """ format: str | None = None compression_level: int | None = None overwrite: OverwriteMode = OverwriteMode.RENAME threads: int = 0 + include: Sequence[str] = () + exclude: Sequence[str] = () @dataclass(frozen=True) @@ -168,6 +178,9 @@ class ExtractOptions: symlinks: What to do with a symlink entry; by default a zip's are written as files and any other archive's are refused. threads: Worker threads for tar.xz, or 0 for one per core. + include: Globs selecting entries as well as any named by `files`; one + that matches nothing fails the run as a missing name does. + exclude: Globs dropping entries, overriding `files` and `include`. """ overwrite: OverwriteMode = OverwriteMode.RENAME @@ -179,6 +192,8 @@ class ExtractOptions: overwrite_dir_metadata: bool = False symlinks: SymlinkMode | None = None threads: int = 0 + include: Sequence[str] = () + exclude: Sequence[str] = () @dataclass(frozen=True) @@ -190,7 +205,8 @@ class ExtractPlan: output_dir: Directory to extract into. files: Optional subset of entry paths to extract (None = all). options: Extraction policy to apply. - entries: The archive's entries that `files` selects. + entries: The archive's entries that `files` and the options' globs + select. can_run: Whether the extract operation can proceed. reason_if_unavailable: Reason the operation cannot proceed, if any. """ @@ -297,9 +313,7 @@ def plan_archive( reason_if_unavailable=str(e), ) - total_input_size: int = sum( - f.stat().st_size for f in _iter_files(source_paths) if f.is_file() - ) + total_input_size: int = _stored_size(source_paths, options) return ArchivePlan( sources=source_paths, @@ -312,21 +326,84 @@ def plan_archive( ) -def _iter_files(paths: list[Path]): - """Yield every file under the given paths, recursing into directories. +def _stored_size(sources: list[Path], options: ArchiveOptions) -> int: + """Return the bytes the C walk would store from the sources. Args: - paths: List of paths to iterate over. + sources: The source paths; hardlinks can span them. + options: Archive options, with the format set. + + Returns: + The total size of the regular files kept, a hardlinked file counting + once if the format stores its later names as links. + """ + links_once = format_stores_hardlinks(options.format or DEFAULT_ARCHIVE_FORMAT) + seen: set[tuple[int, int]] = set() + total = 0 + + for info in (i for s in sources for i in _stored_files(s, options)): + if links_once and info.st_nlink > 1 and info.st_ino: + identity = (info.st_dev, info.st_ino) + if identity in seen: + continue + seen.add(identity) + + total += info.st_size + + return total + + +def _stored_files(source: Path, options: ArchiveOptions) -> Iterator[os.stat_result]: + """Yield the lstat of each regular file the C walk keeps from a source. + + Only `exclude` prunes: a directory no include matches can still hold files + that one does. + + Args: + source: The source path; a symlink stays a link, not its target. + options: Archive options. Yields: - Each file under the given paths. + Each kept regular file's lstat. """ - for p in paths: - if p.is_dir(): - yield from (f for f in p.rglob("*")) + name = source.relative_to(source.parent).as_posix() - else: - yield p + if source.is_symlink() or not source.is_dir(): + candidates = [(source, name)] + + elif path_matches(name, options.exclude): + return + + else: + candidates = _walk_files(source, options) + + for path, entry in candidates: + if path_matches(entry, options.exclude): + continue + + if options.include and not path_matches(entry, options.include): + continue + + info = os.lstat(path) + if stat.S_ISREG(info.st_mode): + yield info + + +def _walk_files(source: Path, options: ArchiveOptions) -> Iterator[tuple[str, str]]: + """Yield each non-directory under a source, pruning excluded directories. + + Args: + source: The directory to walk. + options: Archive options. + + Yields: + Each path on disk with its entry name. + """ + for top, dirs, files in os.walk(source): + rel = Path(top).relative_to(source.parent).as_posix() + dirs[:] = [d for d in dirs if not path_matches(f"{rel}/{d}", options.exclude)] + for f in files: + yield os.path.join(top, f), f"{rel}/{f}" def plan_extraction( @@ -416,14 +493,21 @@ def plan_extraction( reason_if_unavailable=f"Cannot read archive: {e}", ) - # As the extractor matches: a name selects that entry and anything below it - if files: - names: set[str] = {name.rstrip("/") for name in files} - {""} - entries = [ - e - for e in entries - if any(e.path == n or e.path.startswith(f"{n}/") for n in names) - ] + # As the extractor matches: a name selects that entry and anything below it, + # an include adds to those, and an exclude overrides both + names: set[str] = {name.rstrip("/") for name in files or ()} - {""} + + def selected(path: str) -> bool: + if path_matches(path, opts.exclude): + return False + + return ( + not (files or opts.include) + or any(path == n or path.startswith(f"{n}/") for n in names) + or path_matches(path, opts.include) + ) + + entries = [e for e in entries if selected(e.path)] return ExtractPlan( archive=archive_path, @@ -507,6 +591,8 @@ def run( to_core_level(self.plan.options.compression_level), overwrite=_OVERWRITE_CODES[self.plan.options.overwrite], threads=self.plan.options.threads, + include=self.plan.options.include, + exclude=self.plan.options.exclude, progress=to_core_progress(progress, total), cancel=cancel, ) @@ -604,6 +690,8 @@ def run( -1 if options.symlinks is None else _SYMLINK_CODES[options.symlinks] ), threads=options.threads, + include=options.include, + exclude=options.exclude, progress=to_core_progress(progress, total), cancel=cancel, ) diff --git a/src/compresso/frontend/dispatch.py b/src/compresso/frontend/dispatch.py index 0ba3c61..7b757d1 100644 --- a/src/compresso/frontend/dispatch.py +++ b/src/compresso/frontend/dispatch.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Sequence from dataclasses import dataclass, field, replace from functools import partial from pathlib import Path @@ -92,6 +93,8 @@ class CompressRequest: level: The compression level, or None for the format's own. overwrite: What to do when the output already exists. threads: Worker threads, or 0 for one per core. + include: Globs an archive's entries must match; see `ArchiveOptions`. + exclude: Globs an archive's entries must not match. """ inputs: list[Path] @@ -102,6 +105,8 @@ class CompressRequest: level: int | None = None overwrite: OverwriteMode = OverwriteMode.RENAME threads: int = 0 + include: Sequence[str] = () + exclude: Sequence[str] = () @dataclass(frozen=True) @@ -143,7 +148,7 @@ def plan_compress(request: CompressRequest) -> CompressionJob | ArchiveJob: Raises: ValueError: If a single-file format was asked for with several inputs - or a directory. + or a directory, or globs were given for a single-file output. """ fmt: str | None = request.format or infer_format(request.output) many: bool = len(request.inputs) > 1 or any(p.is_dir() for p in request.inputs) @@ -165,6 +170,8 @@ def plan_compress(request: CompressRequest) -> CompressionJob | ArchiveJob: compression_level=request.level, overwrite=request.overwrite, threads=request.threads, + include=request.include, + exclude=request.exclude, ) archive_job = ArchiveJob.from_paths( @@ -179,6 +186,9 @@ def plan_compress(request: CompressRequest) -> CompressionJob | ArchiveJob: return archive_job + if request.include or request.exclude: + raise ValueError("--include and --exclude apply only to archives") + source: Path = request.inputs[0] options = CompressionOptions( algo=request.algo, @@ -246,11 +256,13 @@ def plan_extract(request: ExtractRequest) -> list[ExtractJob | DecompressionJob] # Planned once to learn the default name, then moved into `-C` job = plan(dest=request.directory / plan(dest=None).plan.dest.name) - if request.files: + if request.files or request.options.include or request.options.exclude: job.plan = replace( job.plan, can_run=False, - reason_if_unavailable="--entry applies only to archives", + reason_if_unavailable=( + "--entry, --include and --exclude apply only to archives" + ), ) jobs.append(job) diff --git a/tests/python/cli/test_compress.py b/tests/python/cli/test_compress.py index 256c7c5..b2cde1d 100644 --- a/tests/python/cli/test_compress.py +++ b/tests/python/cli/test_compress.py @@ -2,6 +2,7 @@ from __future__ import annotations +import tarfile from pathlib import Path import pytest @@ -149,6 +150,36 @@ def test_single_file_format_refuses_several_inputs( assert "single file" in result.output + def test_filters_narrow_an_archive(self, source_tree: Path, tmp_path: Path) -> None: + """Test that repeated --include and --exclude reach the archive.""" + output = tmp_path / "o.tar" + + invoke( + "compress", + source_tree, + "-o", + output, + "--include", + "f0.bin", + "--include", + "f1.bin", + "--exclude", + "f1.bin", + "-q", + ) + + with tarfile.open(output) as tar: + assert sorted(tar.getnames()) == ["tree", "tree/f0.bin"] + + @pytest.mark.parametrize("flag", ["--include", "--exclude"]) + def test_filters_are_refused_for_a_single_file( + self, payload: Path, flag: str + ) -> None: + """Test that a glob has nothing to select in a one-file container.""" + result = invoke("compress", payload, flag, "x", expect=EXIT_USAGE) + + assert "apply only to archives" in result.output + def test_unknown_format_is_refused(self, payload: Path) -> None: """Test that an unrecognised -F names neither container.""" invoke("compress", payload, "-F", "bogus", expect=EXIT_USAGE) diff --git a/tests/python/cli/test_extract.py b/tests/python/cli/test_extract.py index 5b9294d..b01e633 100644 --- a/tests/python/cli/test_extract.py +++ b/tests/python/cli/test_extract.py @@ -309,7 +309,85 @@ def test_a_single_file_input_fails(self, payload: Path, tmp_path: Path) -> None: result = invoke("extract", packed, "--entry", "x", expect=EXIT_FAILED) - assert f"{packed}: --entry applies only to archives" in result.output + assert ( + f"{packed}: --entry, --include and --exclude apply only to archives" + in result.output + ) + assert not (tmp_path / "a").exists() + + +class TestFilter: + """Test that --include and --exclude select entries.""" + + def test_include_adds_to_entry_and_exclude_overrides( + self, source_tree: Path, tmp_path: Path, make_archive: Callable[[str], Path] + ) -> None: + """Test bsdtar's union, with the exclude winning over both.""" + dest = tmp_path / "dest" + top = source_tree.name + + invoke( + "extract", + make_archive("tar"), + "-C", + dest, + "-e", + f"{top}/f0.bin", + "--include", + "f[12].bin", + "--exclude", + "f2.bin", + "-q", + ) + + assert sorted(p.name for p in (dest / top).iterdir()) == ["f0.bin", "f1.bin"] + + @pytest.mark.parametrize("flags", [(), ("-e", "tree/f0.bin")]) + def test_excluding_everything_warns_and_succeeds( + self, + tmp_path: Path, + make_archive: Callable[[str], Path], + flags: tuple[str, ...], + ) -> None: + """Test that a filter leaving nothing exits 0 as tar does, but says so.""" + archive = make_archive("tar") + + result = invoke( + "extract", archive, "-C", tmp_path / "d", *flags, "--exclude", "*" + ) + + assert f"{archive}: nothing left to extract" in result.output + assert not list((tmp_path / "d").iterdir()) + + def test_quiet_silences_the_warning( + self, make_archive: Callable[[str], Path] + ) -> None: + """Test that -q suppresses even the warning.""" + result = invoke("extract", make_archive("tar"), "--exclude", "*", "-q") + + assert result.output == "" + + def test_a_partly_excluded_run_does_not_warn( + self, tmp_path: Path, make_archive: Callable[[str], Path] + ) -> None: + """Test that the warning needs every entry gone.""" + result = invoke( + "extract", make_archive("tar"), "-C", tmp_path / "d", "--exclude", "f0.bin" + ) + + assert "nothing left" not in result.output + + @pytest.mark.parametrize("flag", ["--include", "--exclude"]) + def test_a_single_file_input_fails( + self, payload: Path, tmp_path: Path, flag: str + ) -> None: + """Test that a glob on an input that isn't an archive fails it.""" + packed = tmp_path / "a.gz" + packed.write_bytes(gzip.compress(payload.read_bytes())) + + result = invoke("extract", packed, flag, "x", expect=EXIT_FAILED) + + assert "apply only to archives" in result.output assert not (tmp_path / "a").exists() diff --git a/tests/python/test_archive_create.py b/tests/python/test_archive_create.py index d4814f6..c68ac73 100644 --- a/tests/python/test_archive_create.py +++ b/tests/python/test_archive_create.py @@ -14,7 +14,7 @@ import orjson import pytest -from compresso import BackendError, _core +from compresso import ArchiveJob, ArchiveOptions, BackendError, _core from .helpers import ARCHIVE_AND_LIST, run_child @@ -324,6 +324,72 @@ def test_bare_string_is_refused(self, tree: Path, tmp_path: Path) -> None: str(tmp_path / "out.tar"), "tar", [str(tree)], include="*.txt" ) + @pytest.mark.parametrize( + ("include", "exclude"), + [([], ["node_modules", "*.log"]), (["*.txt"], []), (["t/a"], ["*.log"])], + ) + def test_plan_totals_what_the_filter_keeps( + self, tree: Path, tmp_path: Path, include: list[str], exclude: list[str] + ) -> None: + """Test that the plan's total matches the bytes the walk stores.""" + options = ArchiveOptions(format="tar", include=include, exclude=exclude) + job = ArchiveJob.from_paths([tree], tmp_path / "out.tar", options) + + result = job.run() + + assert result.ok, result.error + stored = _core.list_archive_contents(str(job.plan.output)) + assert job.plan.total_input_size == sum(e["size"] for e in stored) + + def test_plan_does_not_walk_an_excluded_tree( + self, tree: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Test that the plan prunes excluded directories, as the C walk does.""" + scanned: list[str] = [] + real_scandir = os.scandir + + def spy(path): + scanned.append(os.fspath(path)) + return real_scandir(path) + + monkeypatch.setattr(os, "scandir", spy) + + ArchiveJob.from_paths( + [tree], + tmp_path / "out.tar", + ArchiveOptions(format="tar", exclude=["node_modules"]), + ) + + assert str(tree / "a") in scanned + assert not any("node_modules" in path for path in scanned) + + def test_plan_counts_only_a_kept_source(self, tree: Path, tmp_path: Path) -> None: + """Test that an excluded source adds nothing to the plan's total.""" + keep = tree / "a" / "keep.txt" + options = ArchiveOptions(format="tar", exclude=["drop.log", "b"]) + + job = ArchiveJob.from_paths( + [keep, tree / "a" / "drop.log", tree / "b"], tmp_path / "out.tar", options + ) + + assert job.plan.total_input_size == keep.stat().st_size + + @pytest.mark.usefixtures("needs_symlinks") + def test_plan_counts_a_symlink_as_a_link(self, tree: Path, tmp_path: Path) -> None: + """Test that a link adds no bytes, wherever it is, like the walk.""" + target = tree / "a" / "keep.txt" + (tree / "b" / "link").symlink_to(target) + source_link = tmp_path / "source-link" + source_link.symlink_to(tree) + + def total(*sources: Path) -> int: + options = ArchiveOptions(format="tar") + job = ArchiveJob.from_paths(list(sources), tmp_path / "o.tar", options) + return job.plan.total_input_size + + assert total(tree / "b") == (tree / "b" / "n").stat().st_size + assert total(source_link) == 0 + class TestSizeChange: """Test a file whose size changes between the walk and its copy.""" @@ -497,6 +563,38 @@ def test_zip_keeps_copies(self, tmp_path: Path) -> None: with zipfile.ZipFile(archive) as zf: assert zf.read("src/b") == b"x" + @pytest.mark.parametrize( + ("name", "expected"), [("a.tar.gz", 1), ("a.tar", 1), ("a.zip", 2)] + ) + def test_plan_counts_a_file_as_the_format_stores_it( + self, tmp_path: Path, name: str, expected: int + ) -> None: + """Test that a link adds no bytes where it's stored as one, across sources.""" + first = tmp_path / "x" + second = tmp_path / "y" + first.mkdir() + second.mkdir() + (first / "a").write_text("data") + os.link(first / "a", second / "b") + + job = ArchiveJob.from_paths([first, second], tmp_path / name) + + assert job.plan.total_input_size == 4 * expected + + def test_plan_counts_a_link_whose_first_name_is_excluded( + self, tmp_path: Path + ) -> None: + """Test that a name the filter drops doesn't hide the one it keeps.""" + src = tmp_path / "src" + src.mkdir() + (src / "a.log").write_text("data") + os.link(src / "a.log", src / "b.txt") + options = ArchiveOptions(format="tar", exclude=["*.log"]) + + job = ArchiveJob.from_paths([src], tmp_path / "out.tar", options) + + assert job.plan.total_input_size == 4 + class TestZipModesAndSymlinks: """Test that zip keeps Unix modes and symlinks in its external attributes.""" diff --git a/tests/python/test_archive_extract.py b/tests/python/test_archive_extract.py index 688df7c..6c1b324 100644 --- a/tests/python/test_archive_extract.py +++ b/tests/python/test_archive_extract.py @@ -20,6 +20,8 @@ BackendError, CorruptDataError, ExtractionPolicyError, + ExtractJob, + ExtractOptions, IncompleteExtractionError, _core, ) @@ -213,6 +215,38 @@ def test_filtered_out_matches_are_not_reported( assert not [p for p in out.rglob("*") if p.is_file()] + @pytest.mark.parametrize( + ("files", "include", "exclude"), + [ + ([], [], ["sub"]), + ([], ["*.txt"], ["d/x.txt"]), + (["d/sub"], ["e.txt"], []), + (["d"], ["e.txt"], ["*x.txt"]), + (["d/x.txt"], [], ["*"]), + ], + ) + def test_plan_lists_what_is_extracted( + self, + archive: Path, + tmp_path: Path, + files: list[str], + include: list[str], + exclude: list[str], + ) -> None: + """Test that the plan's entries, and so the bar's total, match the run.""" + out = tmp_path / "out" + options = ExtractOptions(include=include, exclude=exclude) + job = ExtractJob.from_archive(archive, out, files, options) + + result = job.run() + + assert result.ok, result.error + written = sorted( + p.relative_to(out).as_posix() for p in out.rglob("*") if p.is_file() + ) + planned = sorted(e.path for e in job.plan.entries if not e.is_dir) + assert planned == written + class TestRecognisedButUnsupportedArchive: """Test that a detected archive format without a backend names itself.""" From 2e04079673187731bdf87ac245b0cfa8955d5389 Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Sat, 31 Oct 2026 22:04:45 +0000 Subject: [PATCH 23/31] feat: delete inputs once their output is written, with --rm flag - New '--rm' flag on 'compress'/'extract' deletes each input once its output is complete - New ''--keep/-k' flag maintains the default behaviour of retaining each input - New 'remove_sources' on create_archive() and 'ArchiveOptions' does the same for an archive - Removes deepest first, so only what was stored goes and a directory holding anything unstored stays - '--rm' is refused with '--on-conflict skip', and on extraction with '--entry', '--include' or '--exclude' - In each case the output holds less than the input, so deleting it would lose data - A source that isn't removed is a warning after the normal summary and a non-zero exit - New 'SourceRemovalError' carries the path written plus every path left behind with its reason - A special file or symlink the platform or user can't create now joins 'IncompleteExtractionError' instead of logging a warning - A run that couldn't write everything now fails and '--rm' can't take the archive with it - Windows now maps a missing privilege, an unsupported operation, and a volume that can't do it to EPERM - Anything else from a failed symlink is now EIO, so a full disk still fails the extraction rather than reading as a skipped link - '-q' no longer returns before the work after the summary, so a quiet run still removes its inputs --- src/compresso/__init__.py | 2 + src/compresso/_core.pyi | 17 +++- src/compresso/cli/_run.py | 56 +++++++++-- src/compresso/cli/compress.py | 52 ++++++---- src/compresso/cli/extract.py | 62 ++++++++---- src/compresso/csrc/_core.c | 14 ++- src/compresso/csrc/archives.h | 11 +- src/compresso/csrc/archives/archive_ops.c | 101 ++++++++++++++----- src/compresso/csrc/archives/extract.c | 71 ++++++++----- src/compresso/csrc/archives/select.c | 5 +- src/compresso/csrc/archives/select.h | 5 +- src/compresso/csrc/common.h | 1 + src/compresso/csrc/fsutil.h | 4 +- src/compresso/csrc/fsutil_win32.c | 8 +- src/compresso/frontend/archive_api.py | 10 ++ src/compresso/frontend/dispatch.py | 4 + tests/python/cli/test_compress.py | 111 ++++++++++++++++++++ tests/python/cli/test_extract.py | 115 ++++++++++++++++++++- tests/python/conftest.py | 14 +++ tests/python/test_archive_create.py | 117 +++++++++++++++++++++- tests/python/test_archive_extract.py | 58 ++++++++--- tests/python/test_core.py | 2 + 22 files changed, 720 insertions(+), 120 deletions(-) diff --git a/src/compresso/__init__.py b/src/compresso/__init__.py index 881c049..85a72c2 100644 --- a/src/compresso/__init__.py +++ b/src/compresso/__init__.py @@ -9,6 +9,7 @@ ExtractionPolicyError, HeaderError, IncompleteExtractionError, + SourceRemovalError, TrailingDataWarning, compress_file, decompress_file, @@ -67,6 +68,7 @@ "JobResult", "OverwriteMode", "ProgressCallback", + "SourceRemovalError", "SymlinkMode", "TrailingDataWarning", "benchmark_file", diff --git a/src/compresso/_core.pyi b/src/compresso/_core.pyi index a7aa8f9..025984b 100644 --- a/src/compresso/_core.pyi +++ b/src/compresso/_core.pyi @@ -27,6 +27,19 @@ class IncompleteExtractionError(Error): Raises on completion, anything extracted is kept. """ +class SourceRemovalError(Error): + """An archive was written, but some of its sources could not be deleted. + + The archive is complete, and every other source is deleted. + + Attributes: + output: The path the archive was written to. + failures: Each path left behind, with the reason. + """ + + output: str + failures: list[tuple[str, str]] + class CorruptDataError(Error): """The input is corrupt, truncated, or not the format it claims to be.""" @@ -155,11 +168,13 @@ def create_archive( cancel: CancelToken | None = ..., include: Sequence[str] = ..., exclude: Sequence[str] = ..., + remove_sources: bool = ..., ) -> str: """Create an archive from the given input paths in the given format. `overwrite` is 0 = error (default), 1 = skip, 2 = overwrite, 3 = rename; - applied to the single destination archive. + applied to the single destination archive; `remove_sources` with skip + raises ValueError. Returns the path actually written, which RENAME may have changed from `output_path`; SKIP returns `output_path` unchanged and writes nothing. diff --git a/src/compresso/cli/_run.py b/src/compresso/cli/_run.py index 52d4435..385a1cd 100644 --- a/src/compresso/cli/_run.py +++ b/src/compresso/cli/_run.py @@ -2,13 +2,14 @@ from __future__ import annotations +import os import time from collections.abc import Callable, Iterable from pathlib import Path from typing import Any, TypeVar -from .._core import Error -from ..frontend._job import ThreadedJob +from .._core import Error, SourceRemovalError +from ..frontend._job import JobResult, ThreadedJob from ..frontend.archive_api import OverwriteMode from ._render import ( BAR_THRESHOLD, @@ -19,6 +20,7 @@ fail, progress_bar, report_error, + report_warning, ) J = TypeVar("J") @@ -28,6 +30,10 @@ class InputFailed(Exception): """One input failed; the runner reports it and goes on to the next.""" +class NotRemoved(Exception): + """An output is written, but `--rm` left inputs; args are (path, reason).""" + + def require_files(paths: Iterable[Path]) -> None: """Exit 2 before any input runs if one is missing or not a file. @@ -41,17 +47,28 @@ def require_files(paths: Iterable[Path]) -> None: fail(f"Error: {path} does not exist or is not a file", EXIT_USAGE) -def conflict_mode(on_conflict: OverwriteMode | None, force: bool) -> OverwriteMode: +def conflict_mode( + on_conflict: OverwriteMode | None, force: bool, remove: bool +) -> OverwriteMode: """Resolve `--on-conflict` and `-f` into one mode, exiting 2 if they disagree. Args: on_conflict: The mode given, or None if it was not. force: Whether `-f`, shorthand for `--on-conflict overwrite`, was given. + remove: Whether `--rm`, which can't follow an output that was skipped, + was given. Returns: The mode to run with; RENAME if neither was given. """ if not force: + if remove and on_conflict is OverwriteMode.SKIP: + fail( + "Error: --rm contradicts --on-conflict skip; " + "use --on-conflict error to keep inputs whose output exists", + EXIT_USAGE, + ) + return on_conflict or OverwriteMode.RENAME if on_conflict not in (None, OverwriteMode.OVERWRITE): @@ -60,6 +77,25 @@ def conflict_mode(on_conflict: OverwriteMode | None, force: bool) -> OverwriteMo return OverwriteMode.OVERWRITE +def remove_source(source: Path, before: os.stat_result) -> None: + """Delete an input now its output is written when '--rm' flag is used. + + Args: + source: The input. + before: Its stat from before the job; anything else at that path now + is the output, written over it, so it stays. + + Raises: + NotRemoved: If the input can't be deleted. + """ + try: + if os.path.samestat(source.stat(), before): + source.unlink() + + except OSError as e: + raise NotRemoved((str(source), e.strerror or str(e))) from e + + def run_job( job: ThreadedJob[Any], source: Path, @@ -68,7 +104,7 @@ def run_job( *, quiet: bool, what: str, -) -> float: +) -> tuple[float, JobResult]: """Run `job` under a progress bar. Args: @@ -80,7 +116,8 @@ def run_job( what: Operation name for messages, e.g. "Compression". Returns: - Seconds the job took. + Seconds the job took, and its result; a `SourceRemovalError` in that + result is left for the caller, as the output is still there to report. Raises: InputFailed: If the job failed. @@ -97,10 +134,10 @@ def run_job( if result.cancelled: cancelled(what) - if not result.ok: + if not result.ok and not isinstance(result.error, SourceRemovalError): raise InputFailed(f"{source}: {what} failed: {result.error}") - return elapsed + return elapsed, result def run_all(jobs: Iterable[J], run_one: Callable[[J], None], what: str) -> int: @@ -124,6 +161,11 @@ def run_all(jobs: Iterable[J], run_one: Callable[[J], None], what: str) -> int: report_error(str(e)) failed = True + except NotRemoved as e: + for path, reason in e.args: + report_warning(f"{path}: not removed: {reason}") + failed = True + except Error as e: report_error(f"{what} error: {e}") failed = True diff --git a/src/compresso/cli/compress.py b/src/compresso/cli/compress.py index 2cff952..222af83 100644 --- a/src/compresso/cli/compress.py +++ b/src/compresso/cli/compress.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Annotated -from .._core import BackendError, Error, HeaderError +from .._core import BackendError, Error, HeaderError, SourceRemovalError from ..frontend.api import CompressionJob from ..frontend.archive_api import ArchiveJob, OverwriteMode from ..frontend.dispatch import CompressRequest, plan_compress @@ -20,16 +20,19 @@ skipped, summary, ) -from ._run import conflict_mode, run_all, run_job +from ._run import NotRemoved, conflict_mode, remove_source, run_all, run_job -def _compress_one_file(job: CompressionJob, quiet: bool, verbose: bool) -> None: +def _compress_one_file( + job: CompressionJob, quiet: bool, verbose: bool, remove: bool +) -> None: """Compress a single file into a one-file container. Args: job: The planned job. quiet: Whether to suppress all output. verbose: Whether to print the plan and stats blocks. + remove: Whether to delete the input once the output is written. """ plan = job.plan @@ -40,8 +43,9 @@ def _compress_one_file(job: CompressionJob, quiet: bool, verbose: bool) -> None: compress_plan(plan) skips: bool = plan.options.overwrite is OverwriteMode.SKIP and plan.dest.exists() + before = plan.src.stat() - elapsed: float = run_job( + elapsed, _ = run_job( job, plan.src, "Compressing", plan.input_size, quiet=quiet, what="Compression" ) @@ -51,16 +55,16 @@ def _compress_one_file(job: CompressionJob, quiet: bool, verbose: bool) -> None: if verbose: compress_result(plan, written, elapsed) - elif quiet: - return - - elif skips: + elif skips and not quiet: skipped(plan.dest) - else: + elif not quiet: renamed: Path | None = written if written != plan.dest else None summary(plan.dest, plan.input_size, written.stat().st_size, elapsed, renamed) + if remove: + remove_source(plan.src, before) + def _archive_many(job: ArchiveJob, quiet: bool, verbose: bool) -> None: """Pack several paths, or a directory, into one archive. @@ -82,7 +86,7 @@ def _archive_many(job: ArchiveJob, quiet: bool, verbose: bool) -> None: # A format with a codec stage reads the data twice, so the job reports a # larger total than this once it starts; the bar follows it - elapsed: float = run_job( + elapsed, result = run_job( job, plan.output, "Archiving", @@ -92,18 +96,15 @@ def _archive_many(job: ArchiveJob, quiet: bool, verbose: bool) -> None: ) # RENAME may have written somewhere other than `plan.output`; job.plan is - # replaced on success with the path actually written + # replaced with the path actually written once the archive is committed written: Path = job.plan.output if verbose: archive_result(plan, written, elapsed) - elif quiet: - return - - elif skips: + elif skips and not quiet: skipped(plan.output) - else: + elif not quiet: renamed: Path | None = written if written != plan.output else None summary( plan.output, @@ -113,6 +114,9 @@ def _archive_many(job: ArchiveJob, quiet: bool, verbose: bool) -> None: renamed, ) + if isinstance(result.error, SourceRemovalError): + raise NotRemoved(*result.error.failures) + def compress( inputs: Annotated[ @@ -189,6 +193,14 @@ def compress( bool, app.Option("--force", "-f", help="Replace an existing output"), ] = False, + remove: Annotated[ + bool, + app.Option( + "--rm/--keep", + " /-k", + help="Delete the input once the output is written, or keep it", + ), + ] = False, threads: Annotated[ int, app.Option( @@ -237,11 +249,14 @@ def compress( output refuses it. on_conflict: What to do when the output exists (default: rename). force: If True, replace an existing output. + remove: If True, delete the input once the output is written; an + archive deletes only what it stored, leaving anything else and + the directories holding it. threads: Worker threads for zstd and xz (default: one per core). verbose: If True, show the plan and full stats. quiet: If True, suppress all output. """ - overwrite_mode = conflict_mode(on_conflict, force) + overwrite_mode = conflict_mode(on_conflict, force, remove) if verbose and quiet: fail("Error: -v contradicts -q", EXIT_USAGE) @@ -268,6 +283,7 @@ def compress( threads=threads, include=include or (), exclude=exclude or (), + remove_sources=remove, ) try: @@ -280,7 +296,7 @@ def run_one(job: CompressionJob | ArchiveJob) -> None: if isinstance(job, ArchiveJob): _archive_many(job, quiet, verbose) else: - _compress_one_file(job, quiet, verbose) + _compress_one_file(job, quiet, verbose, remove) if not (quiet or verbose): print() diff --git a/src/compresso/cli/extract.py b/src/compresso/cli/extract.py index 88a92c5..d3e3c50 100644 --- a/src/compresso/cli/extract.py +++ b/src/compresso/cli/extract.py @@ -26,7 +26,14 @@ skipped, summary, ) -from ._run import InputFailed, conflict_mode, require_files, run_all, run_job +from ._run import ( + InputFailed, + conflict_mode, + remove_source, + require_files, + run_all, + run_job, +) _SIZE = re.compile(r"(\d+)(?:([kmgt])(?:i?b)?|b)?", re.IGNORECASE) _UNITS = {"": 1, "k": 1024, "m": 1024**2, "g": 1024**3, "t": 1024**4} @@ -54,13 +61,14 @@ def parse_size(value: str) -> int: return int(count) * _UNITS[(unit or "").lower()] -def _extract_archive(job: ExtractJob, quiet: bool, verbose: bool) -> None: +def _extract_archive(job: ExtractJob, quiet: bool, verbose: bool, remove: bool) -> None: """Unpack one archive. Args: job: The planned job. quiet: Whether to suppress all output. verbose: Whether to print the plan and stats blocks. + remove: Whether to delete the archive once it is extracted. """ plan = job.plan @@ -73,7 +81,8 @@ def _extract_archive(job: ExtractJob, quiet: bool, verbose: bool) -> None: # Progress covers the bytes written out; a compressed archive also reports # the pass that decompresses it, so the job's own total wins total: int = sum(entry.size for entry in plan.entries) - elapsed: float = run_job( + before = plan.archive.stat() + elapsed, _ = run_job( job, plan.archive, "Extracting", total, quiet=quiet, what="Extraction" ) @@ -85,18 +94,22 @@ def _extract_archive(job: ExtractJob, quiet: bool, verbose: bool) -> None: extract_result(plan, elapsed) elif not quiet: - summary( - plan.archive, plan.archive.stat().st_size, total, elapsed, None, ratio=False - ) + summary(plan.archive, before.st_size, total, elapsed, None, ratio=False) + + if remove: + remove_source(plan.archive, before) -def _decompress_one_file(job: DecompressionJob, quiet: bool, verbose: bool) -> None: +def _decompress_one_file( + job: DecompressionJob, quiet: bool, verbose: bool, remove: bool +) -> None: """Unpack one single-file container: a `.comp` or a standalone codec. Args: job: The planned job. quiet: Whether to suppress all output. verbose: Whether to print the plan and stats blocks. + remove: Whether to delete the input once the output is written. """ plan = job.plan @@ -108,10 +121,11 @@ def _decompress_one_file(job: DecompressionJob, quiet: bool, verbose: bool) -> N # Progress counts the compressed bytes read, so the source's size is the # denominator rather than the size it expands to - compressed_size: int = plan.src.stat().st_size + before = plan.src.stat() + compressed_size: int = before.st_size skips: bool = plan.overwrite is OverwriteMode.SKIP and plan.dest.exists() - elapsed: float = run_job( + elapsed, _ = run_job( job, plan.src, "Decompressing", @@ -126,13 +140,10 @@ def _decompress_one_file(job: DecompressionJob, quiet: bool, verbose: bool) -> N if verbose: decompress_result(plan, written, compressed_size, elapsed) - elif quiet: - return - - elif skips: + elif skips and not quiet: skipped(plan.dest) - else: + elif not quiet: renamed: Path | None = written if written != plan.dest else None summary( plan.src, @@ -143,6 +154,9 @@ def _decompress_one_file(job: DecompressionJob, quiet: bool, verbose: bool) -> N ratio=False, ) + if remove: + remove_source(plan.src, before) + def extract( inputs: Annotated[ @@ -203,6 +217,14 @@ def extract( bool, app.Option("--force", "-f", help="Replace files that already exist"), ] = False, + remove: Annotated[ + bool, + app.Option( + "--rm/--keep", + " /-k", + help="Delete the input once it is fully extracted, or keep it", + ), + ] = False, max_size: Annotated[ int | None, app.Option( @@ -311,6 +333,7 @@ def extract( run left with nothing to extract warns, but still succeeds. on_conflict: What to do when a file exists (default: rename). force: If True, replace files that already exist. + remove: If True, delete each input once it is fully written out. max_size: Cap on total extracted bytes (default: no cap). max_depth: Cap on an entry path's depth, or 0 for none. same_permissions: If True, restore archive entries' modes exactly; @@ -327,7 +350,12 @@ def extract( verbose: If True, show the plan and full stats. quiet: If True, suppress all output. """ - mode = conflict_mode(on_conflict, force) + mode = conflict_mode(on_conflict, force, remove) + + # Whatever these leave out would go with the archive + if remove and (entry or include or exclude): + fail("Error: --rm contradicts --entry, --include and --exclude", EXIT_USAGE) + if verbose and quiet: fail("Error: -v contradicts -q", EXIT_USAGE) @@ -370,14 +398,14 @@ def extract( def run_one(job: ExtractJob | DecompressionJob) -> None: if isinstance(job, ExtractJob): - _extract_archive(job, quiet, verbose) + _extract_archive(job, quiet, verbose, remove) else: # An archive creates a missing `-C` itself; a single file needs it made if directory: directory.mkdir(parents=True, exist_ok=True) - _decompress_one_file(job, quiet, verbose) + _decompress_one_file(job, quiet, verbose, remove) if not (quiet or verbose): print() diff --git a/src/compresso/csrc/_core.c b/src/compresso/csrc/_core.c index ac867a7..336b696 100644 --- a/src/compresso/csrc/_core.c +++ b/src/compresso/csrc/_core.c @@ -416,7 +416,7 @@ static PyObject *py_create_archive(PyObject *self, PyObject *args, static char *kwlist[] = {"output_path", "format", "input_paths", "compression_level", "overwrite", "threads", "progress", "cancel", "include", - "exclude", NULL}; + "exclude", "remove_sources", NULL}; PyObject *output_path_bytes = NULL; const char *format_name = NULL; @@ -428,12 +428,14 @@ static PyObject *py_create_archive(PyObject *self, PyObject *args, PyObject *cancel = NULL; PathList include = {0}; PathList exclude = {0}; + int remove_sources = 0; if (!PyArg_ParseTupleAndKeywords( - args, kwargs, "O&sO&|i$iiOOO&O&", kwlist, PyUnicode_FSConverter, + args, kwargs, "O&sO&|i$iiOOO&O&p", kwlist, PyUnicode_FSConverter, &output_path_bytes, &format_name, path_list_converter, &inputs, &compression_level, &overwrite_existing, &threads, &progress, &cancel, - pattern_list_converter, &include, pattern_list_converter, &exclude)) { + pattern_list_converter, &include, pattern_list_converter, &exclude, + &remove_sources)) { return NULL; // Error already set } PathFilter filter = {include.paths, include.count, exclude.paths, @@ -460,7 +462,8 @@ static PyObject *py_create_archive(PyObject *self, PyObject *args, char actual_path[FS_PATH_MAX]; result = create_archive( PyBytes_AS_STRING(output_path_bytes), &pipe, inputs.paths, inputs.count, - &filter, overwrite_existing, actual_path, sizeof(actual_path), &ctx); + &filter, overwrite_existing, remove_sources, actual_path, + sizeof(actual_path), &ctx); done: Py_DECREF(output_path_bytes); @@ -1046,6 +1049,7 @@ static int core_exec(PyObject *module) { &errors->error}, {&errors->incomplete_extraction, "compresso.IncompleteExtractionError", &errors->error}, + {&errors->source_removal, "compresso.SourceRemovalError", &errors->error}, {&errors->trailing_data_warning, "compresso.TrailingDataWarning", &PyExc_UserWarning}, }; @@ -1071,6 +1075,7 @@ static int core_traverse(PyObject *module, visitproc visit, void *arg) { Py_VISIT(errors->corrupt_data); Py_VISIT(errors->extraction_policy); Py_VISIT(errors->incomplete_extraction); + Py_VISIT(errors->source_removal); Py_VISIT(errors->cancelled); Py_VISIT(errors->trailing_data_warning); return 0; @@ -1084,6 +1089,7 @@ static int core_clear(PyObject *module) { Py_CLEAR(errors->corrupt_data); Py_CLEAR(errors->extraction_policy); Py_CLEAR(errors->incomplete_extraction); + Py_CLEAR(errors->source_removal); Py_CLEAR(errors->cancelled); Py_CLEAR(errors->trailing_data_warning); return 0; diff --git a/src/compresso/csrc/archives.h b/src/compresso/csrc/archives.h index fd4aba2..5d6db02 100644 --- a/src/compresso/csrc/archives.h +++ b/src/compresso/csrc/archives.h @@ -167,14 +167,15 @@ typedef struct { size_t num_exclude; } PathFilter; -// `overwrite_existing` follows the same 0-3 scheme as -// `ExtractionPolicy.overwrite_existing`; on success, the path actually written -// (which RENAME may have changed) is copied into `out_actual_path` +// `overwrite_existing` is as in `ExtractionPolicy`; `out_actual_path` receives +// the path written, which RENAME may change; `remove_sources` deletes only what +// was stored; with SKIP, a file appearing during the run stops the commit, but +// still returns success int create_archive(const char *output_path, const CompressionPipeline *pipeline, const char **input_paths, size_t num_paths, const PathFilter *filter, int overwrite_existing, - char *out_actual_path, size_t out_actual_path_size, - CoreContext *ctx); + int remove_sources, char *out_actual_path, + size_t out_actual_path_size, CoreContext *ctx); // Entries are checked against `policy`; any failure but a cancel undoes what // the extraction wrote, restoring what OVERWRITE replaced diff --git a/src/compresso/csrc/archives/archive_ops.c b/src/compresso/csrc/archives/archive_ops.c index 351542c..176fa5b 100644 --- a/src/compresso/csrc/archives/archive_ops.c +++ b/src/compresso/csrc/archives/archive_ops.c @@ -1,10 +1,10 @@ #define PY_SSIZE_T_CLEAN #include #include "archive_ops.h" -#include "paths.h" #include "../common.h" #include "../fsutil.h" #include "../standalone.h" +#include "paths.h" #include "walk.h" #include #include @@ -183,13 +183,64 @@ static int add_walked_entries(const CArchive *archive, void *writer, return 0; } +// Deepest first; a directory holding something unstored stays, and so does "."; +// a path that won't go lands in `failed`, and the rest still go +static int remove_walked(const WalkList *walked, PyObject *failed) { + for (size_t i = walked->count; i-- > 0;) { + const WalkItem *item = &walked->items[i]; + int is_dir = item->st.type == FS_TYPE_DIR; + if ((is_dir ? fs_rmdir(item->path) : fs_unlink(item->path)) == 0) + continue; + if (is_dir && (errno == ENOTEMPTY || errno == EEXIST || errno == EINVAL)) + continue; + + const char *reason = strerror(errno); + PyObject *failure = + Py_BuildValue("(Ns)", PyUnicode_DecodeFSDefault(item->path), reason); + int rc = failure ? PyList_Append(failed, failure) : -1; + Py_XDECREF(failure); + if (rc != 0) + return -1; + } + return 0; +} + +// The archive is committed by now, so what's raised also says where it went +static int remove_stored(const WalkList *walked, const char *output, + const CompErrors *errors) { + PyObject *failed = PyList_New(0); + int ret = failed ? remove_walked(walked, failed) : -1; + + if (ret == 0 && PyList_GET_SIZE(failed) > 0) { + ret = -1; + PyObject *error = PyObject_CallFunction( + errors->source_removal, "s", + "Archive written, but some of its sources could not be deleted"); + PyObject *path = PyUnicode_DecodeFSDefault(output); + if (error && path && PyObject_SetAttrString(error, "output", path) == 0 && + PyObject_SetAttrString(error, "failures", failed) == 0) + PyErr_SetObject(errors->source_removal, error); + Py_XDECREF(path); + Py_XDECREF(error); + } + + Py_XDECREF(failed); + return ret; +} + // ---- Archive Operations ---- int create_archive(const char *output_path, const CompressionPipeline *pipeline, const char **input_paths, size_t num_paths, const PathFilter *filter, int overwrite_existing, - char *out_actual_path, size_t out_actual_path_size, - CoreContext *ctx) { + int remove_sources, char *out_actual_path, + size_t out_actual_path_size, CoreContext *ctx) { + if (remove_sources && overwrite_existing == 1) { + PyErr_SetString(PyExc_ValueError, + "remove_sources can't be combined with skip"); + return -1; + } + if (!pipeline_is_valid(pipeline) || pipeline->archive == ARCHIVE_NONE) { char name[32]; pipeline_display_name(pipeline, name, sizeof(name)); @@ -261,7 +312,6 @@ int create_archive(const char *output_path, const CompressionPipeline *pipeline, ctx_begin_stage(ctx, walked.file_bytes); ret = add_walked_entries(archive, writer, &walked, ctx); } - walk_list_free(&walked); // libzip does all its compression inside zip_close, so that is where its // progress comes from; a run that already failed is discarded instead @@ -271,28 +321,33 @@ int create_archive(const char *output_path, const CompressionPipeline *pipeline, if (!stream) { if (ret == 0) - return output_commit(tmp_path, &out); - fs_unlink(tmp_path); - return ret; - } - - // libarchive only knows its write callback failed; failed streams return - // a failure to every write; empty ones swap in the cause (e.g. cancelled) - if (ret != 0) { - int stream_ret = codec_write(stream, NULL, 0); - if (stream_ret != 0) { - PyErr_Clear(); - ret = stream_ret; + ret = output_commit(tmp_path, &out); + else + fs_unlink(tmp_path); + } else { + // libarchive only knows its write callback failed; failed streams return + // a failure to every write; empty ones swap in the cause (e.g. cancelled) + if (ret != 0) { + int stream_ret = codec_write(stream, NULL, 0); + if (stream_ret != 0) { + PyErr_Clear(); + ret = stream_ret; + } } - } - int codec_ret = codec_writer_close(stream, ret != 0); - if (ret == 0) - ret = codec_ret; + int codec_ret = codec_writer_close(stream, ret != 0); + if (ret == 0) + ret = codec_ret; - char failure[64]; - snprintf(failure, sizeof(failure), "%s compression failed", fmt->name); - return output_finish(ret, NULL, tmp, tmp_path, &out, ctx->errors, failure); + char failure[64]; + snprintf(failure, sizeof(failure), "%s compression failed", fmt->name); + ret = output_finish(ret, NULL, tmp, tmp_path, &out, ctx->errors, failure); + } + + if (ret == 0 && remove_sources) + ret = remove_stored(&walked, out_actual_path, ctx->errors); + walk_list_free(&walked); + return ret; } FILE *open_archive_source(const char *archive_path, const CArchive **archive, diff --git a/src/compresso/csrc/archives/extract.c b/src/compresso/csrc/archives/extract.c index d50d3f3..899fda0 100644 --- a/src/compresso/csrc/archives/extract.c +++ b/src/compresso/csrc/archives/extract.c @@ -302,14 +302,27 @@ static int make_node(const void *entry, const char *out_path) { return fs_mknod(out_path, e->special, e->mode, e->rdev); } +// Appends "path (reason)" to `skipped_nodes`; returns 0, or -1 with an +// exception set +static int skip_node(PyObject *skipped_nodes, const char *path, + const char *reason) { + PyObject *node = PyUnicode_FromFormat("%s (%s)", path, reason); + int rc = node ? PyList_Append(skipped_nodes, node) : -1; + Py_XDECREF(node); + return rc; +} + // Runs `make(source, out_path)` under the overwrite mode, as for a file; -// returns 0, or -1 with an exception set +// returns 0, 1 if symlinks can't be created here, or -1 with an exception set static int place_link(int (*make)(const void *, const char *), const void *source, const char *out_path, const char *entry_path, int overwrite_existing, int64_t mtime, Changes *changes) { if (make(source, out_path) == 0) return link_made(changes, out_path, mtime); + // Only here, as nothing has been moved aside yet + if (make == make_symlink && errno == EPERM) + return 1; if (errno != EEXIST) { PyErr_SetFromErrnoWithFilename(PyExc_OSError, out_path); return -1; @@ -421,9 +434,11 @@ static int extract_hardlink(const CompErrors *errors, const char *resolved_root, policy->overwrite_existing, 0, changes); } +// A symlink this user or filesystem can't create is appended to `skipped_nodes` static int extract_symlink(const CompErrors *errors, const char *resolved_root, const char *out_path, const ArchiveEntry *entry, - const ExtractionPolicy *policy, Changes *changes) { + const ExtractionPolicy *policy, + PyObject *skipped_nodes, Changes *changes) { char parent[FS_PATH_MAX]; if (prepare_parent(errors, resolved_root, out_path, entry->path, changes, parent) != 0) @@ -432,26 +447,29 @@ static int extract_symlink(const CompErrors *errors, const char *resolved_root, if (check_target_from(errors, resolved_root, parent, entry) != 0) return -1; - return place_link(make_symlink, entry->link_target, out_path, entry->path, - policy->overwrite_existing, - policy->preserve_timestamps ? (int64_t)entry->mtime : 0, - changes); + int rc = place_link(make_symlink, entry->link_target, out_path, entry->path, + policy->overwrite_existing, + policy->preserve_timestamps ? (int64_t)entry->mtime : 0, + changes); + if (rc != 1) + return rc; + return skip_node(skipped_nodes, entry->path, + "this user or filesystem can't create symlinks"); } -// A node this platform or user can't create is skipped with a warning; returns -// 0, or -1 with an exception set +// A node this platform or user can't create is appended to `skipped_nodes` as +// "path (reason)"; returns 0, or -1 with an exception set static int extract_special(const CompErrors *errors, const char *resolved_root, const char *out_path, const ArchiveEntry *entry, const ExtractionPolicy *policy, uint32_t mode_mask, - CoreContext *ctx, Changes *changes) { - if (!fs_can_mknod(entry->special)) { - ctx_log(ctx, CTX_LOG_WARNING, "Skipped %s: %s", entry->path, - !entry->special ? "sockets are not extracted" - : entry->special == AE_IFIFO - ? "FIFOs can't be created on this platform" - : "devices are only created by root, and not on Windows"); - return 0; - } + PyObject *skipped_nodes, Changes *changes) { + if (!fs_can_mknod(entry->special)) + return skip_node( + skipped_nodes, entry->path, + !entry->special ? "sockets are not extracted" + : entry->special == AE_IFIFO + ? "FIFOs can't be created on this platform" + : "devices are only created by root, and not on Windows"); char parent[FS_PATH_MAX]; if (prepare_parent(errors, resolved_root, out_path, entry->path, changes, @@ -621,13 +639,14 @@ static int extract_file(const CArchive *archive, void *reader, // Extract each entry from an open reader into `changes` and `deferred`; each // hardlink whose target is missing is appended to `skipped_links` as -// "link -> target"; `data_ctx` counts bytes written, NULL callers count input +// "link -> target", and each node that can't be created to `skipped_nodes`; +// `data_ctx` counts bytes written, NULL callers count input static int extract_entries(const CArchive *archive, void *reader, const char *output_dir, const char *resolved_root, Selection *sel, const ExtractionPolicy *policy, CoreContext *ctx, CoreContext *data_ctx, Changes *changes, DeferredDirs *deferred, - PyObject *skipped_links) { + PyObject *skipped_links, PyObject *skipped_nodes) { ArchiveEntry entry = {0}; uint64_t written_total = 0; Renames renames = {0}; @@ -691,7 +710,7 @@ static int extract_entries(const CArchive *archive, void *reader, policy, mode_mask, &renames, changes, deferred); else if (entry.type == ENTRY_SYMLINK && policy->allow_symlinks == 1) rc = extract_symlink(ctx->errors, resolved_root, out_path, &entry, policy, - changes); + skipped_nodes, changes); else if (entry.type == ENTRY_FILE || entry.type == ENTRY_SYMLINK) rc = extract_file(archive, reader, ctx->errors, resolved_root, out_path, &entry, policy, mode_mask, data_ctx, &written_total, @@ -708,7 +727,7 @@ static int extract_entries(const CArchive *archive, void *reader, } } else if (entry.type == ENTRY_SPECIAL) rc = extract_special(ctx->errors, resolved_root, out_path, &entry, policy, - mode_mask, ctx, changes); + mode_mask, skipped_nodes, changes); if (rc < 0) { result = rc; goto cleanup; @@ -764,8 +783,11 @@ int extract_archive(const char *archive_path, const char *output_dir, Selection sel; PyObject *skipped_links = PyList_New(0); - if (!skipped_links || selection_init(&sel, files, num_files, filter) != 0) { + PyObject *skipped_nodes = PyList_New(0); + if (!skipped_links || !skipped_nodes || + selection_init(&sel, files, num_files, filter) != 0) { Py_XDECREF(skipped_links); + Py_XDECREF(skipped_nodes); if (src) fclose(src); return -1; @@ -801,7 +823,7 @@ int extract_archive(const char *archive_path, const char *output_dir, &sel, policy, ctx, &declared_total) : extract_entries(archive, reader, output_dir, resolved_root, &sel, policy, ctx, src ? NULL : ctx, &changes, - &deferred, skipped_links); + &deferred, skipped_links, skipped_nodes); // A failed pass keeps its own error if (archive->close_reader(reader, ret != 0) != 0 && ret == 0) @@ -826,8 +848,9 @@ int extract_archive(const char *archive_path, const char *output_dir, // After everything else has been extracted if (ret == 0) - ret = report_incomplete(&sel, skipped_links, ctx); + ret = report_incomplete(&sel, skipped_links, skipped_nodes, ctx); Py_DECREF(skipped_links); + Py_DECREF(skipped_nodes); free(sel.items); free(sel.include_found); return ret; diff --git a/src/compresso/csrc/archives/select.c b/src/compresso/csrc/archives/select.c index e44e79f..6d6fb46 100644 --- a/src/compresso/csrc/archives/select.c +++ b/src/compresso/csrc/archives/select.c @@ -177,7 +177,7 @@ static int append_part(PyObject *parts, const char *label, PyObject *items) { } int report_incomplete(const Selection *sel, PyObject *skipped_links, - CoreContext *ctx) { + PyObject *skipped_nodes, CoreContext *ctx) { PyObject *names = PyList_New(0); PyObject *parts = PyList_New(0); int ret = -1; @@ -203,7 +203,8 @@ int report_incomplete(const Selection *sel, PyObject *skipped_links, goto done; } if (append_part(parts, "Not found in archive", names) != 0 || - append_part(parts, "Hardlink target missing", skipped_links) != 0) + append_part(parts, "Hardlink target missing", skipped_links) != 0 || + append_part(parts, "Skipped, the rest was extracted", skipped_nodes) != 0) goto done; if (PyList_GET_SIZE(parts) == 0) { diff --git a/src/compresso/csrc/archives/select.h b/src/compresso/csrc/archives/select.h index ff5cabc..b1d20c3 100644 --- a/src/compresso/csrc/archives/select.h +++ b/src/compresso/csrc/archives/select.h @@ -37,8 +37,9 @@ int path_matches(const char *path, const char *pat); // Whether any of `count` patterns matches `path` int path_matches_any(const char *path, const char **pats, size_t count); -// Raises an error listing unmatched names and includes, and skipped hardlinks +// Raises an error listing unmatched names and includes, skipped hardlinks, and +// nodes that couldn't be created int report_incomplete(const Selection *sel, PyObject *skipped_links, - CoreContext *ctx); + PyObject *skipped_nodes, CoreContext *ctx); #endif // SELECT_H diff --git a/src/compresso/csrc/common.h b/src/compresso/csrc/common.h index 0e2619b..c15828a 100644 --- a/src/compresso/csrc/common.h +++ b/src/compresso/csrc/common.h @@ -149,6 +149,7 @@ struct CompErrors { PyObject *corrupt_data; PyObject *extraction_policy; PyObject *incomplete_extraction; + PyObject *source_removal; PyObject *cancelled; PyObject *trailing_data_warning; }; diff --git a/src/compresso/csrc/fsutil.h b/src/compresso/csrc/fsutil.h index 31c248a..b8ce574 100644 --- a/src/compresso/csrc/fsutil.h +++ b/src/compresso/csrc/fsutil.h @@ -182,8 +182,8 @@ int fs_rename(const char *from, const char *to, int replace); int fs_link(const char *existing, const char *new_path); // Symlink `link_path` to `target`, stored unresolved; -1 with errno set -// (EEXIST if `link_path` exists, EPERM on Windows without Developer Mode/admin -// rights); on Windows, a target that is a directory gets a directory link +// (EEXIST if `link_path` exists, EPERM if this user or filesystem can't create +// symlinks; on Windows, a target that is a directory gets a directory link int fs_symlink(const char *target, const char *link_path); // Whether this process can create a node of `type` (fs_stat.special's values): diff --git a/src/compresso/csrc/fsutil_win32.c b/src/compresso/csrc/fsutil_win32.c index a1e084c..620787d 100644 --- a/src/compresso/csrc/fsutil_win32.c +++ b/src/compresso/csrc/fsutil_win32.c @@ -436,6 +436,11 @@ static void set_errno_from_win32(int fallback) { case ERROR_ACCESS_DENIED: errno = EACCES; break; + case ERROR_PRIVILEGE_NOT_HELD: + case ERROR_NOT_SUPPORTED: + case ERROR_INVALID_FUNCTION: // e.g. a FAT volume + errno = EPERM; + break; default: errno = fallback; break; @@ -554,7 +559,8 @@ int fs_symlink(const char *target, const char *link_path) { if (ok) return 0; - set_errno_from_win32(EPERM); // e.g. ERROR_PRIVILEGE_NOT_HELD + // Not EPERM for anything else, e.g. a full disk, which must fail extraction + set_errno_from_win32(EIO); return -1; } diff --git a/src/compresso/frontend/archive_api.py b/src/compresso/frontend/archive_api.py index 643c9a5..92d36ba 100644 --- a/src/compresso/frontend/archive_api.py +++ b/src/compresso/frontend/archive_api.py @@ -13,6 +13,7 @@ from .._core import ( Cancelled, CancelToken, + SourceRemovalError, check_level, create_archive, extract_archive, @@ -125,6 +126,9 @@ class ArchiveOptions: plus the directories holding it. exclude: Globs on entry names that aren't stored or descended into, overriding `include`. + remove_sources: Whether to delete what was stored once the archive is + written; anything else stays, including parent directories; running + with `OverwriteMode.SKIP` fails, as a skipped archive holds nothing. """ format: str | None = None @@ -133,6 +137,7 @@ class ArchiveOptions: threads: int = 0 include: Sequence[str] = () exclude: Sequence[str] = () + remove_sources: bool = False @dataclass(frozen=True) @@ -593,6 +598,7 @@ def run( threads=self.plan.options.threads, include=self.plan.options.include, exclude=self.plan.options.exclude, + remove_sources=self.plan.options.remove_sources, progress=to_core_progress(progress, total), cancel=cancel, ) @@ -603,6 +609,10 @@ def run( self.plan = replace(self.plan, output=Path(actual_path)) return JobResult(ok=True, error=None, plan=self.plan) + except SourceRemovalError as e: + self.plan = replace(self.plan, output=Path(e.output)) + return JobResult(ok=False, error=e, plan=self.plan) + # Cancellation is a deliberate stop, not a failure; see # `CompressionJob.run`. except Cancelled: diff --git a/src/compresso/frontend/dispatch.py b/src/compresso/frontend/dispatch.py index 7b757d1..b4a922c 100644 --- a/src/compresso/frontend/dispatch.py +++ b/src/compresso/frontend/dispatch.py @@ -95,6 +95,8 @@ class CompressRequest: threads: Worker threads, or 0 for one per core. include: Globs an archive's entries must match; see `ArchiveOptions`. exclude: Globs an archive's entries must not match. + remove_sources: Whether an archive deletes what it stored once written; + see `ArchiveOptions`. """ inputs: list[Path] @@ -107,6 +109,7 @@ class CompressRequest: threads: int = 0 include: Sequence[str] = () exclude: Sequence[str] = () + remove_sources: bool = False @dataclass(frozen=True) @@ -172,6 +175,7 @@ def plan_compress(request: CompressRequest) -> CompressionJob | ArchiveJob: threads=request.threads, include=request.include, exclude=request.exclude, + remove_sources=request.remove_sources, ) archive_job = ArchiveJob.from_paths( diff --git a/tests/python/cli/test_compress.py b/tests/python/cli/test_compress.py index b2cde1d..18a9081 100644 --- a/tests/python/cli/test_compress.py +++ b/tests/python/cli/test_compress.py @@ -8,6 +8,7 @@ import pytest from compresso.cli._render import ( + EXIT_FAILED, EXIT_USAGE, ) @@ -112,6 +113,116 @@ def test_existing_output( assert (f"{t.dest} skipped" in result.output) == ("skip" in flags) +class TestRemoveInput: + """Test what `--rm` and `-k` do to the input.""" + + @pytest.mark.parametrize( + "flags,removes", + [ + ((), False), + (("--rm",), True), + (("-k",), False), + (("--rm", "--keep"), False), + (("-k", "--rm"), True), + ], + ids=["default", "rm", "keep", "rm-then-keep", "keep-then-rm"], + ) + def test_flags( + self, + compress_target: CompressTarget, + flags: tuple[str, ...], + removes: bool, + ) -> None: + """Test that the input goes only with --rm, the last flag winning.""" + t = compress_target + + invoke("compress", t.source, "-o", t.dest, *t.args, *flags, "-q") + + assert t.dest.is_file() + assert t.source.exists() != removes + + def test_a_failed_run_keeps_the_input( + self, compress_target: CompressTarget + ) -> None: + """Test that an output refused by --on-conflict leaves the input.""" + t = compress_target + t.dest.write_bytes(b"stale") + + invoke( + "compress", + t.source, + "-o", + t.dest, + *t.args, + "--on-conflict", + "error", + "--rm", + expect=EXIT_FAILED, + ) + + assert t.source.exists() + + def test_skip_is_refused(self, payload: Path) -> None: + """Test that --rm can't lose an input to an output that was skipped.""" + result = invoke( + "compress", payload, "--on-conflict", "skip", "--rm", expect=EXIT_USAGE + ) + + assert "--rm contradicts --on-conflict skip" in result.output + assert payload.exists() + + def test_undeletable_file_follows_the_summary( + self, locked: Path, tmp_path: Path + ) -> None: + """Test that an input that won't go is a warning after a normal summary.""" + source = locked / "a.txt" + source.write_text("a") + locked.chmod(0o555) + out = tmp_path / "a.gz" + + result = invoke("compress", source, "-o", out, "--rm", expect=EXIT_FAILED) + + assert out.is_file() + assert source.exists() + warning = f"! {source}: not removed: Permission denied" + assert result.output.index("→") < result.output.index(warning) + + @pytest.mark.parametrize("flags", [(), ("-q",)], ids=["default", "quiet"]) + def test_undeletable_archive_source_names_what_was_written( + self, + locked: Path, + source_tree: Path, + tmp_path: Path, + flags: tuple[str, ...], + ) -> None: + """Test that the archive isn't called failed, and a rename is reported.""" + stuck = locked / "a.txt" + stuck.write_text("a") + locked.chmod(0o555) + out = tmp_path / "out.tar" + out.write_bytes(b"stale") + + result = invoke( + "compress", + source_tree, + locked, + "-o", + out, + "--rm", + *flags, + expect=EXIT_FAILED, + ) + + written = tmp_path / renamed("out.tar") + assert tarfile.is_tarfile(written) + assert out.read_bytes() == b"stale" + assert not source_tree.exists() + assert stuck.exists() + assert f"! {stuck}: not removed: Permission denied" in result.output + assert "Archive failed" not in result.output + assert (written.name in result.output) != bool(flags) + + class TestSmartCompress: """Test `compress` runs the job dispatch plans; the rules are in frontend/test_dispatch.py.""" diff --git a/tests/python/cli/test_extract.py b/tests/python/cli/test_extract.py index b01e633..428541f 100644 --- a/tests/python/cli/test_extract.py +++ b/tests/python/cli/test_extract.py @@ -3,6 +3,7 @@ from __future__ import annotations import gzip +import os import sys import tarfile from collections.abc import Callable @@ -19,7 +20,7 @@ ) from compresso.frontend.dispatch import ExtractRequest -from ..helpers import file_entry, link_entry, make_tar, renamed, umask +from ..helpers import file_entry, link_entry, make_tar, node_entry, renamed, umask from .conftest import CONFLICT_MODES, invoke @@ -481,3 +482,115 @@ def test_missing_directory_is_created( assert (dest / "a").read_bytes() == payload.read_bytes() assert (dest / "tree").is_dir() + + +class TestRemoveInput: + """Test what `--rm` and `-k` do to the input.""" + + @pytest.mark.parametrize( + "flags,removes", + [((), False), (("--rm",), True), (("--rm", "-k"), False)], + ids=["default", "rm", "rm-then-keep"], + ) + @pytest.mark.parametrize("kind", ["file", "archive"]) + def test_flags( + self, + tmp_path: Path, + make_archive: Callable[[str], Path], + kind: str, + flags: tuple[str, ...], + removes: bool, + ) -> None: + """Test that a fully unpacked input goes only with --rm.""" + if kind == "file": + source = tmp_path / "a.gz" + source.write_bytes(gzip.compress(b"data")) + else: + source = make_archive("tar") + dest = tmp_path / "dest" + + invoke("extract", source, "-C", dest, *flags, "-q") + + assert source.exists() != removes + assert any(dest.iterdir()) + + def test_a_failed_decompression_keeps_the_input(self, tmp_path: Path) -> None: + """Test that a corrupt single-file input isn't deleted.""" + source = tmp_path / "a.gz" + source.write_bytes(gzip.compress(b"data")[:-4]) + + invoke( + "extract", source, "-C", tmp_path / "out", "--rm", "-q", expect=EXIT_FAILED + ) + + assert source.exists() + + def test_entry_written_over_the_archive_stays(self, tmp_path: Path) -> None: + """Test that --rm doesn't delete what replaced the archive at its path.""" + archive = tmp_path / "self.tar" + make_tar(archive, file_entry("self.tar", size=3)) + + invoke("extract", archive, "-f", "--rm", "-q") + + assert archive.stat().st_size == 3 + + @pytest.mark.parametrize( + "flags", + [ + ("-e", "x"), + ("--include", "x"), + ("--exclude", "x"), + ("--on-conflict", "skip"), + ], + ids=["entry", "include", "exclude", "skip"], + ) + def test_partial_extraction_is_refused( + self, make_archive: Callable[[str], Path], flags: tuple[str, ...] + ) -> None: + """Test that --rm can't take entries that were never extracted with it.""" + archive = make_archive("tar") + + result = invoke("extract", archive, *flags, "--rm", expect=EXIT_USAGE) + + assert "--rm contradicts" in result.output + assert archive.exists() + + @pytest.mark.parametrize("kind", ["archive", "single"]) + def test_undeletable_input_follows_the_summary( + self, + make_archive: Callable[[str], Path], + locked: Path, + tmp_path: Path, + kind: str, + ) -> None: + """Test that an input that won't go is a warning after a normal summary.""" + if kind == "archive": + source = make_archive("tar").rename(locked / "a.tar") + else: + source = locked / "a.gz" + source.write_bytes(gzip.compress(b"a")) + locked.chmod(0o555) + + result = invoke( + "extract", source, "-C", tmp_path / "unpacked", "--rm", expect=EXIT_FAILED + ) + + assert any((tmp_path / "unpacked").iterdir()) + assert source.exists() + warning = f"! {source}: not removed: Permission denied" + assert result.output.index("→") < result.output.index(warning) + + @pytest.mark.skipif( + sys.platform == "win32" or os.geteuid() == 0, reason="needs POSIX, not root" + ) + def test_uncreatable_node_keeps_the_archive(self, tmp_path: Path) -> None: + """Test that a device fails the input, so --rm can't take it with the archive.""" + archive = tmp_path / "nodes.tar" + make_tar(archive, file_entry("a"), node_entry("char", tarfile.CHRTYPE)) + out = tmp_path / "out" + + result = invoke("extract", archive, "-C", out, "--rm", expect=EXIT_FAILED) + + assert "Skipped, the rest was extracted: char" in result.output + assert (out / "a").is_file() + assert archive.exists() diff --git a/tests/python/conftest.py b/tests/python/conftest.py index 0623afa..0ae5987 100644 --- a/tests/python/conftest.py +++ b/tests/python/conftest.py @@ -2,6 +2,8 @@ import importlib import os +import sys +from collections.abc import Generator from pathlib import Path import pytest @@ -47,6 +49,18 @@ def needs_symlinks(tmp_path: Path) -> None: probe.unlink() +@pytest.fixture +def locked(tmp_path: Path) -> Generator[Path]: + """A directory whose files can't be deleted once a test makes it read-only.""" + if sys.platform == "win32" or os.geteuid() == 0: + pytest.skip("needs POSIX, not root") + + root = tmp_path / "locked" + root.mkdir() + yield root + root.chmod(0o755) + + @pytest.fixture def sample_text_file(tmp_path: Path) -> Path: """A small, compressible text file.""" diff --git a/tests/python/test_archive_create.py b/tests/python/test_archive_create.py index c68ac73..ca54bfa 100644 --- a/tests/python/test_archive_create.py +++ b/tests/python/test_archive_create.py @@ -1,6 +1,7 @@ """Tests for creating archives.""" import contextlib +import errno import logging import os import socket @@ -14,7 +15,13 @@ import orjson import pytest -from compresso import ArchiveJob, ArchiveOptions, BackendError, _core +from compresso import ( + ArchiveJob, + ArchiveOptions, + BackendError, + SourceRemovalError, + _core, +) from .helpers import ARCHIVE_AND_LIST, run_child @@ -391,6 +398,114 @@ def total(*sources: Path) -> int: assert total(source_link) == 0 +class TestRemoveSources: + """Test that `remove_sources` deletes exactly what the archive stored.""" + + @pytest.fixture + def tree(self, tmp_path: Path) -> Path: + """A small tree with a nested directory.""" + root = tmp_path / "tree" + (root / "sub").mkdir(parents=True) + (root / "a.txt").write_text("a") + (root / "sub" / "b.log").write_text("b") + + return root + + @pytest.mark.parametrize("fmt", ["tar", "tar.zst", "zip"]) + def test_stored_tree_is_deleted(self, tree: Path, tmp_path: Path, fmt: str) -> None: + """Test that a stored tree goes, leaving a complete archive.""" + output = tmp_path / f"out.{fmt}" + + _core.create_archive(str(output), fmt, [str(tree)], remove_sources=True) + + assert not tree.exists() + names = {e["path"].rstrip("/") for e in _core.list_archive_contents(output)} + assert names == {"tree", "tree/a.txt", "tree/sub", "tree/sub/b.log"} + + def test_what_a_filter_left_out_stays(self, tree: Path, tmp_path: Path) -> None: + """Test that an excluded file keeps itself and the directory holding it.""" + _core.create_archive( + str(tmp_path / "out.tar"), + "tar", + [str(tree)], + exclude=["b.log"], + remove_sources=True, + ) + + assert (tree / "sub" / "b.log").is_file() + assert not (tree / "a.txt").exists() + + def test_output_inside_the_source_stays(self, tree: Path) -> None: + """Test that the walk skipping the archive saves it from the removal.""" + output = tree / "out.tar" + + _core.create_archive(str(output), "tar", [str(tree)], remove_sources=True) + + assert [p.name for p in tree.iterdir()] == ["out.tar"] + assert len(_core.list_archive_contents(output)) == 4 + + @pytest.mark.skipif(sys.platform == "win32", reason="needs a FIFO") + def test_unstored_special_file_stays(self, tree: Path, tmp_path: Path) -> None: + """Test that a FIFO zip can't hold survives with its directory.""" + os.mkfifo(tree / "pipe") + + _core.create_archive( + str(tmp_path / "out.zip"), "zip", [str(tree)], remove_sources=True + ) + + assert [p.name for p in tree.iterdir()] == ["pipe"] + + @pytest.mark.usefixtures("needs_symlinks") + def test_symlink_goes_but_not_its_target(self, tree: Path, tmp_path: Path) -> None: + """Test that a source symlink, stored as a link, is all that's deleted.""" + link = tmp_path / "link" + link.symlink_to(tree, target_is_directory=True) + + _core.create_archive( + str(tmp_path / "out.tar"), "tar", [str(link)], remove_sources=True + ) + + assert not link.is_symlink() + assert (tree / "a.txt").is_file() + + def test_skip_is_refused(self, tree: Path, tmp_path: Path) -> None: + """Test that skip, which can drop a finished archive, can't delete sources.""" + with pytest.raises(ValueError, match="remove_sources"): + _core.create_archive( + str(tmp_path / "out.tar"), + "tar", + [str(tree)], + overwrite=1, + remove_sources=True, + ) + + assert (tree / "a.txt").is_file() + assert not (tmp_path / "out.tar").exists() + + def test_undeletable_sources_are_each_reported( + self, tree: Path, locked: Path, tmp_path: Path + ) -> None: + """Test that the rest still go, and the error covers all that stayed.""" + (locked / "c.txt").write_text("c") + (locked / "d.txt").write_text("d") + locked.chmod(0o555) + output = tmp_path / "out.tar" + + with pytest.raises(SourceRemovalError) as caught: + _core.create_archive( + str(output), "tar", [str(tree), str(locked)], remove_sources=True + ) + + reason = os.strerror(errno.EACCES) + assert caught.value.output == str(output) + assert sorted(caught.value.failures) == [ + (str(locked / "c.txt"), reason), + (str(locked / "d.txt"), reason), + ] + assert not tree.exists() + assert len(_core.list_archive_contents(output)) == 7 + + class TestSizeChange: """Test a file whose size changes between the walk and its copy.""" diff --git a/tests/python/test_archive_extract.py b/tests/python/test_archive_extract.py index 6c1b324..d818762 100644 --- a/tests/python/test_archive_extract.py +++ b/tests/python/test_archive_extract.py @@ -574,19 +574,32 @@ def test_fifo_is_created_with_its_metadata(self, tmp_path: Path) -> None: @pytest.mark.skipif( sys.platform == "win32" or os.geteuid() == 0, reason="needs POSIX, not root" ) - def test_device_is_skipped_without_root( - self, tmp_path: Path, caplog: pytest.LogCaptureFixture - ) -> None: - """Test that a device is skipped with a warning, the rest still extracted.""" + def test_device_is_reported_without_root(self, tmp_path: Path) -> None: + """Test that a device fails the extraction, the rest still extracted.""" archive = tmp_path / "nodes.tar" make_tar(archive, file_entry("a"), node_entry("char", tarfile.CHRTYPE)) out = tmp_path / "out" - with caplog.at_level(logging.WARNING, logger="compresso"): + with pytest.raises( + IncompleteExtractionError, + match=r"^Skipped, the rest was extracted: char \(devices are only created by root", + ): _core.extract_archive(str(archive), str(out), []) assert os.listdir(out) == ["a"] - assert "Skipped char: devices are only created by root" in caplog.text + + @pytest.mark.skipif( + sys.platform == "win32" or os.geteuid() == 0, reason="needs POSIX, not root" + ) + def test_excluded_device_is_not_reported(self, tmp_path: Path) -> None: + """Test that a device left out on purpose doesn't fail the extraction.""" + archive = tmp_path / "nodes.tar" + make_tar(archive, file_entry("a"), node_entry("char", tarfile.CHRTYPE)) + out = tmp_path / "out" + + _core.extract_archive(str(archive), str(out), [], exclude=["char"]) + + assert os.listdir(out) == ["a"] @pytest.mark.skipif( sys.platform == "win32" or os.geteuid() != 0, reason="needs root" @@ -603,19 +616,19 @@ def test_device_is_created_as_root(self, tmp_path: Path) -> None: assert st.st_rdev == os.makedev(1, 3) @pytest.mark.skipif(sys.platform != "win32", reason="Windows-only") - def test_windows_skips_a_fifo( - self, tmp_path: Path, caplog: pytest.LogCaptureFixture - ) -> None: - """Test that a FIFO is skipped with a warning, the rest still extracted.""" + def test_windows_reports_a_fifo(self, tmp_path: Path) -> None: + """Test that a FIFO fails the extraction, the rest still extracted.""" archive = tmp_path / "nodes.tar" make_tar(archive, file_entry("a"), node_entry("fifo", tarfile.FIFOTYPE)) out = tmp_path / "out" - with caplog.at_level(logging.WARNING, logger="compresso"): + with pytest.raises( + IncompleteExtractionError, + match=r"^Skipped, the rest was extracted: fifo \(FIFOs can't be created on this platform\)$", + ): _core.extract_archive(str(archive), str(out), []) assert os.listdir(out) == ["a"] - assert "Skipped fifo: FIFOs can't be created on this platform" in caplog.text class TestDirectoryMetadata: @@ -896,6 +909,27 @@ def test_target_text_counts_toward_the_size_cap(self, tmp_path: Path) -> None: class TestSymlinkExtraction: """Test allow_symlinks=1, which creates links whose targets stay inside.""" + def test_uncreatable_link_is_reported(self, tmp_path: Path) -> None: + """Test that a link this process can't make keeps the rest extracted.""" + try: + (tmp_path / "symlink-probe").symlink_to("x") + except OSError: + pass + else: + pytest.skip("symlinks are permitted here") + + archive = tmp_path / "links.tar" + make_tar(archive, file_entry("a"), link_entry("b", "a", kind=tarfile.SYMTYPE)) + out = tmp_path / "out" + + with pytest.raises( + IncompleteExtractionError, + match=r"^Skipped, the rest was extracted: b \(this user or filesystem", + ): + _core.extract_archive(str(archive), str(out), [], allow_symlinks=1) + + assert os.listdir(out) == ["a"] + @pytest.mark.usefixtures("needs_symlinks") def test_link_is_created_as_stored(self, tmp_path: Path) -> None: """Test that the link holds its target verbatim, and resolves to it.""" diff --git a/tests/python/test_core.py b/tests/python/test_core.py index 5d175e5..05e9b5e 100644 --- a/tests/python/test_core.py +++ b/tests/python/test_core.py @@ -19,6 +19,7 @@ ExtractionPolicyError, HeaderError, IncompleteExtractionError, + SourceRemovalError, _core, compress_file, decompress_file, @@ -55,6 +56,7 @@ class TestCoreExceptions: CorruptDataError, ExtractionPolicyError, IncompleteExtractionError, + SourceRemovalError, Cancelled, ], ) From e1474a7b7500c8ad4bcfb1f8d3e851268000599b Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Sun, 1 Nov 2026 01:08:50 +0000 Subject: [PATCH 24/31] feat: archive symlink target files, with --dereference - New '-h/--dereference' flag on 'compress', and 'dereference' on create_archive() and 'ArchiveOptions', stores a link's target in its place - A link to a directory between it and its source stays a link, so the walk can't loop, as does a dangling one, which has nothing to store - 'WalkItem' tracks each item's parent to make that check - Dangling links warn only where the data was wanted (named as a source, or followed), since otherwise it is a faithful copy and nothing is lost - '--rm' deletes the links it followed and leaves what they lead to, unlike GNU tar, so dereferencing can't delete the originals - New fs_stat_target() stats through a link - On Windows the stat splits so both paths share it, and a directory now gets an identity, which the ancestor check compares - Plans now follow the same rules, and a missing source is now 'lexists', so a dangling link is counted - Adds tests covering dereferencing, planning, combined with '--rm', links to ancestors, and dangling links --- src/compresso/_core.pyi | 5 + src/compresso/cli/compress.py | 13 ++ src/compresso/csrc/_core.c | 22 +-- src/compresso/csrc/archives.h | 5 +- src/compresso/csrc/archives/archive_ops.c | 11 +- src/compresso/csrc/archives/walk.c | 48 +++++- src/compresso/csrc/archives/walk.h | 19 ++- src/compresso/csrc/fsutil.h | 7 +- src/compresso/csrc/fsutil_posix.c | 8 + src/compresso/csrc/fsutil_win32.c | 66 +++++--- src/compresso/frontend/archive_api.py | 55 ++++++- src/compresso/frontend/dispatch.py | 4 + tests/python/cli/test_compress.py | 19 +++ tests/python/test_archive_create.py | 189 ++++++++++++++++++++++ 14 files changed, 406 insertions(+), 65 deletions(-) diff --git a/src/compresso/_core.pyi b/src/compresso/_core.pyi index 025984b..c8cd4d3 100644 --- a/src/compresso/_core.pyi +++ b/src/compresso/_core.pyi @@ -169,6 +169,7 @@ def create_archive( include: Sequence[str] = ..., exclude: Sequence[str] = ..., remove_sources: bool = ..., + dereference: bool = ..., ) -> str: """Create an archive from the given input paths in the given format. @@ -182,6 +183,10 @@ def create_archive( `include` and `exclude` are globs matched against entry names; an exclude wins and isn't descended into, and with any includes, only matching entries and the directories holding them are stored. + + `dereference` stores what symlinks lead to; one that's dangling, or that + leads to a directory above it, stays a link; `remove_sources` then deletes + the links, and not their targets. """ def extract_archive( diff --git a/src/compresso/cli/compress.py b/src/compresso/cli/compress.py index 222af83..ac68415 100644 --- a/src/compresso/cli/compress.py +++ b/src/compresso/cli/compress.py @@ -201,6 +201,14 @@ def compress( help="Delete the input once the output is written, or keep it", ), ] = False, + dereference: Annotated[ + bool, + app.Option( + "--dereference", + "-h", + help="Archive what symlinks lead to, rather than the links", + ), + ] = False, threads: Annotated[ int, app.Option( @@ -252,6 +260,10 @@ def compress( remove: If True, delete the input once the output is written; an archive deletes only what it stored, leaving anything else and the directories holding it. + dereference: If True, an archive stores symlink targets, and `remove` + deletes the links, not targets; a dangling link, or one leading to + a directory above it, stays a link; single-file output always reads + through a link. threads: Worker threads for zstd and xz (default: one per core). verbose: If True, show the plan and full stats. quiet: If True, suppress all output. @@ -284,6 +296,7 @@ def compress( include=include or (), exclude=exclude or (), remove_sources=remove, + dereference=dereference, ) try: diff --git a/src/compresso/csrc/_core.c b/src/compresso/csrc/_core.c index 336b696..54a984b 100644 --- a/src/compresso/csrc/_core.c +++ b/src/compresso/csrc/_core.c @@ -413,10 +413,11 @@ static PyObject *py_decompress_file(PyObject *self, PyObject *args, static PyObject *py_create_archive(PyObject *self, PyObject *args, PyObject *kwargs) { - static char *kwlist[] = {"output_path", "format", "input_paths", - "compression_level", "overwrite", "threads", - "progress", "cancel", "include", - "exclude", "remove_sources", NULL}; + static char *kwlist[] = { + "output_path", "format", "input_paths", "compression_level", + "overwrite", "threads", "progress", "cancel", + "include", "exclude", "remove_sources", "dereference", + NULL}; PyObject *output_path_bytes = NULL; const char *format_name = NULL; @@ -429,13 +430,14 @@ static PyObject *py_create_archive(PyObject *self, PyObject *args, PathList include = {0}; PathList exclude = {0}; int remove_sources = 0; + int dereference = 0; if (!PyArg_ParseTupleAndKeywords( - args, kwargs, "O&sO&|i$iiOOO&O&p", kwlist, PyUnicode_FSConverter, + args, kwargs, "O&sO&|i$iiOOO&O&pp", kwlist, PyUnicode_FSConverter, &output_path_bytes, &format_name, path_list_converter, &inputs, &compression_level, &overwrite_existing, &threads, &progress, &cancel, pattern_list_converter, &include, pattern_list_converter, &exclude, - &remove_sources)) { + &remove_sources, &dereference)) { return NULL; // Error already set } PathFilter filter = {include.paths, include.count, exclude.paths, @@ -460,10 +462,10 @@ static PyObject *py_create_archive(PyObject *self, PyObject *args, } char actual_path[FS_PATH_MAX]; - result = create_archive( - PyBytes_AS_STRING(output_path_bytes), &pipe, inputs.paths, inputs.count, - &filter, overwrite_existing, remove_sources, actual_path, - sizeof(actual_path), &ctx); + result = + create_archive(PyBytes_AS_STRING(output_path_bytes), &pipe, inputs.paths, + inputs.count, &filter, overwrite_existing, remove_sources, + dereference, actual_path, sizeof(actual_path), &ctx); done: Py_DECREF(output_path_bytes); diff --git a/src/compresso/csrc/archives.h b/src/compresso/csrc/archives.h index 5d6db02..7381570 100644 --- a/src/compresso/csrc/archives.h +++ b/src/compresso/csrc/archives.h @@ -170,11 +170,12 @@ typedef struct { // `overwrite_existing` is as in `ExtractionPolicy`; `out_actual_path` receives // the path written, which RENAME may change; `remove_sources` deletes only what // was stored; with SKIP, a file appearing during the run stops the commit, but -// still returns success +// still returns success; `follow_links` stores symlink targets, and +// `remove_sources` then deletes the links and not targets int create_archive(const char *output_path, const CompressionPipeline *pipeline, const char **input_paths, size_t num_paths, const PathFilter *filter, int overwrite_existing, - int remove_sources, char *out_actual_path, + int remove_sources, int follow_links, char *out_actual_path, size_t out_actual_path_size, CoreContext *ctx); // Entries are checked against `policy`; any failure but a cancel undoes what diff --git a/src/compresso/csrc/archives/archive_ops.c b/src/compresso/csrc/archives/archive_ops.c index 176fa5b..c97a7d5 100644 --- a/src/compresso/csrc/archives/archive_ops.c +++ b/src/compresso/csrc/archives/archive_ops.c @@ -184,11 +184,14 @@ static int add_walked_entries(const CArchive *archive, void *writer, } // Deepest first; a directory holding something unstored stays, and so does "."; -// a path that won't go lands in `failed`, and the rest still go +// a followed link goes, but not its target; a path that won't go lands in +// `failed`, and the rest still go static int remove_walked(const WalkList *walked, PyObject *failed) { for (size_t i = walked->count; i-- > 0;) { const WalkItem *item = &walked->items[i]; - int is_dir = item->st.type == FS_TYPE_DIR; + if (item->via_link) + continue; + int is_dir = item->st.type == FS_TYPE_DIR && !item->followed; if ((is_dir ? fs_rmdir(item->path) : fs_unlink(item->path)) == 0) continue; if (is_dir && (errno == ENOTEMPTY || errno == EEXIST || errno == EINVAL)) @@ -233,7 +236,7 @@ static int remove_stored(const WalkList *walked, const char *output, int create_archive(const char *output_path, const CompressionPipeline *pipeline, const char **input_paths, size_t num_paths, const PathFilter *filter, int overwrite_existing, - int remove_sources, char *out_actual_path, + int remove_sources, int follow_links, char *out_actual_path, size_t out_actual_path_size, CoreContext *ctx) { if (remove_sources && overwrite_existing == 1) { PyErr_SetString(PyExc_ValueError, @@ -307,7 +310,7 @@ int create_archive(const char *output_path, const CompressionPipeline *pipeline, WalkList walked = {0}; int ret = walk_sources(input_paths, num_paths, filter, &own, archive->stores_specials, archive->stores_hardlinks, - ctx, &walked); + follow_links, ctx, &walked); if (ret == 0) { ctx_begin_stage(ctx, walked.file_bytes); ret = add_walked_entries(archive, writer, &walked, ctx); diff --git a/src/compresso/csrc/archives/walk.c b/src/compresso/csrc/archives/walk.c index cbaa4e1..d043a38 100644 --- a/src/compresso/csrc/archives/walk.c +++ b/src/compresso/csrc/archives/walk.c @@ -67,8 +67,36 @@ static int is_skipped(const WalkItem *item, const OwnFiles *own, return 0; } -// Pushes the children of `dir` so they pop in the order readdir gave them -static int push_children(WalkList *pending, const WalkItem *dir) { +static void follow_link(WalkItem *item, const WalkList *walked) { + fs_stat target; + if (fs_stat_target(item->path, &target) != 0) + return; + + for (size_t i = item->parent; i > 0; i = walked->items[i - 1].parent) + if (fs_same_file(&target, &walked->items[i - 1].st)) + return; + + item->st = target; + item->followed = 1; +} + +// Only where the link was meant to give data: left in a tree it's a faithful +// copy, and such links are routine +static void warn_if_dangling(const WalkItem *item, int follow_links, + CoreContext *ctx) { + fs_stat target; + if (item->st.type != FS_TYPE_SYMLINK || !(follow_links || item->parent == 0)) + return; + + if (fs_stat_target(item->path, &target) != 0) + ctx_log(ctx, CTX_LOG_WARNING, + "Stored %s as a link: its target can't be reached", item->path); +} + +// Pushes the children of `dir`, whose index is `parent`, so they pop in the +// order readdir gave them +static int push_children(WalkList *pending, const WalkItem *dir, + size_t parent) { fs_dir *d = fs_opendir(dir->path); if (!d) { PyErr_SetFromErrnoWithFilename(PyExc_OSError, dir->path); @@ -89,6 +117,9 @@ static int push_children(WalkList *pending, const WalkItem *dir) { ret = -1; break; } + pending->items[pending->count - 1].parent = parent; + pending->items[pending->count - 1].via_link = + dir->followed || dir->via_link; } // A conversion failure also ends fs_readdir; unchecked, files would silently @@ -208,8 +239,8 @@ static void prune_provisional(WalkList *list) { int walk_sources(const char **input_paths, size_t num_paths, const PathFilter *filter, const OwnFiles *own, - int keep_specials, int link_hardlinks, CoreContext *ctx, - WalkList *out) { + int keep_specials, int link_hardlinks, int follow_links, + CoreContext *ctx, WalkList *out) { // Depth-first like the archive's order, on a heap stack rather than the C one WalkList pending = {0}; int ret = 0; @@ -256,20 +287,23 @@ int walk_sources(const char **input_paths, size_t num_paths, ret = -1; break; } + if (follow_links && item.st.type == FS_TYPE_SYMLINK) + follow_link(&item, out); if ((!included && item.st.type != FS_TYPE_DIR) || is_skipped(&item, own, keep_specials, ctx)) { free(item.path); continue; } + warn_if_dangling(&item, follow_links, ctx); ret = push(out, item.path, item.prefix_len); if (ret != 0) break; - out->items[out->count - 1].st = item.st; - out->items[out->count - 1].provisional = !included; + item.provisional = !included; + out->items[out->count - 1] = item; if (item.st.type == FS_TYPE_DIR) - ret = push_children(&pending, &item); + ret = push_children(&pending, &item, out->count); else if (item.st.type == FS_TYPE_FILE) out->file_bytes += item.st.size; } diff --git a/src/compresso/csrc/archives/walk.h b/src/compresso/csrc/archives/walk.h index 4e394ef..1d45860 100644 --- a/src/compresso/csrc/archives/walk.h +++ b/src/compresso/csrc/archives/walk.h @@ -22,6 +22,9 @@ typedef struct WalkItem { fs_stat st; const struct WalkItem *link; // The file's first name, if this is a later one int provisional; // A directory kept only if something below it is + size_t parent; // 1-based index of its directory, 0 for an input + int followed; // A symlink stored as what it leads to + int via_link; // Reached through a followed symlink } WalkItem; typedef struct { @@ -35,15 +38,17 @@ typedef struct { // NULL with an exception set char *walk_entry_name(const char *path, size_t prefix_len); -// Every input and everything below it that `filter` keeps, depth-first, -// symlinks not followed; an excluded directory isn't descended into; warns and -// skips `own` and sockets, and FIFOs and devices unless `keep_specials`; with -// `link_hardlinks`, a file's later names get `link` and aren't counted in -// `file_bytes`; returns COMP_CANCELLED, or -1 with an exception set +// Every input and everything below it that `filter` keeps, depth-first, not +// entering excluded directories; warns and skips `own` and sockets, and FIFOs +// and devices unless `keep_specials`; `link_hardlinks` gives a file's later +// names `link` and leaves them out of `file_bytes`; `follow_links` walks +// symlinks as their targets, except dangling ones and those to an ancestor; +// dangling ones warn with this flag or as inputs; returns COMP_CANCELLED, or -1 +// with an exception set int walk_sources(const char **input_paths, size_t num_paths, const PathFilter *filter, const OwnFiles *own, - int keep_specials, int link_hardlinks, CoreContext *ctx, - WalkList *out); + int keep_specials, int link_hardlinks, int follow_links, + CoreContext *ctx, WalkList *out); void walk_list_free(WalkList *list); diff --git a/src/compresso/csrc/fsutil.h b/src/compresso/csrc/fsutil.h index b8ce574..3b149b7 100644 --- a/src/compresso/csrc/fsutil.h +++ b/src/compresso/csrc/fsutil.h @@ -42,8 +42,8 @@ typedef struct { uint32_t special; uint64_t rdev; - // Identity, for fs_same_file; on Windows only filled for regular files (the - // volume serial and file index), and 0 wherever it is unknown + // Identity, for fs_same_file; on Windows only filled for regular files and + // directories (volume serial and file index), and 0 wherever it is unknown uint64_t dev, ino; uint32_t nlink; // Its names' count; filled wherever the identity is } fs_stat; @@ -73,6 +73,9 @@ int fs_join(char *out, size_t out_size, const char *dir, const char *name); // Windows reparse point reports FS_TYPE_SYMLINK rather than its target's type int fs_stat_path(const char *path, fs_stat *out); +// Stat what `path` leads to, following symlinks; -1 if the link is dangling +int fs_stat_target(const char *path, fs_stat *out); + // Stat an open file, so a path reached through a symlink reports its target; // the identity is left 0 on Windows int fs_fstat(FILE *f, fs_stat *out); diff --git a/src/compresso/csrc/fsutil_posix.c b/src/compresso/csrc/fsutil_posix.c index bfaacea..18db2ad 100644 --- a/src/compresso/csrc/fsutil_posix.c +++ b/src/compresso/csrc/fsutil_posix.c @@ -47,6 +47,14 @@ int fs_stat_path(const char *path, fs_stat *out) { return 0; } +int fs_stat_target(const char *path, fs_stat *out) { + struct stat st; + if (stat(path, &st) != 0) + return -1; + fs_fill_stat(&st, out); + return 0; +} + int fs_fstat(FILE *f, fs_stat *out) { struct stat st; if (fstat(fileno(f), &st) != 0) diff --git a/src/compresso/csrc/fsutil_win32.c b/src/compresso/csrc/fsutil_win32.c index 620787d..925b818 100644 --- a/src/compresso/csrc/fsutil_win32.c +++ b/src/compresso/csrc/fsutil_win32.c @@ -72,26 +72,10 @@ static HANDLE fs_open_for_metadata(const wchar_t *wpath) { NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); } -int fs_stat_path(const char *path, fs_stat *out) { - wchar_t wpath[FS_PATH_MAX]; - if (fs_widen(path, wpath, FS_PATH_MAX) != 0) - return -1; - - // _wstat64 follows reparse points and has no S_ISLNK equivalent, so without - // this archiver walks into any junction - DWORD attrs = GetFileAttributesW(wpath); - int is_reparse = attrs != INVALID_FILE_ATTRIBUTES && - (attrs & FILE_ATTRIBUTE_REPARSE_POINT) != 0; - +static int fs_stat_wide(const wchar_t *wpath, fs_stat *out) { struct __stat64 st; - if (_wstat64(wpath, &st) != 0) { - // A link is still a real entry when its target is missing - if (!is_reparse) - return -1; - memset(out, 0, sizeof(*out)); - out->type = FS_TYPE_SYMLINK; - return 0; - } + if (_wstat64(wpath, &st) != 0) + return -1; out->size = (uint64_t)st.st_size; out->mtime = (int64_t)st.st_mtime; @@ -102,18 +86,15 @@ int fs_stat_path(const char *path, fs_stat *out) { out->ino = 0; out->nlink = 0; - if (is_reparse) - out->type = FS_TYPE_SYMLINK; - else if (st.st_mode & _S_IFDIR) + if (st.st_mode & _S_IFDIR) out->type = FS_TYPE_DIR; else if (st.st_mode & _S_IFREG) out->type = FS_TYPE_FILE; else out->type = FS_TYPE_OTHER; - // _wstat64 leaves st_ino 0, so the identity needs a handle; only regular - // files get one, as nothing compares directories - if (out->type == FS_TYPE_FILE) { + // _wstat64 leaves st_ino 0, so the identity needs a handle + if (out->type == FS_TYPE_FILE || out->type == FS_TYPE_DIR) { HANDLE h = fs_open_for_metadata(wpath); BY_HANDLE_FILE_INFORMATION info; if (h != INVALID_HANDLE_VALUE) { @@ -129,6 +110,41 @@ int fs_stat_path(const char *path, fs_stat *out) { return 0; } +int fs_stat_target(const char *path, fs_stat *out) { + wchar_t wpath[FS_PATH_MAX]; + if (fs_widen(path, wpath, FS_PATH_MAX) != 0) + return -1; + return fs_stat_wide(wpath, out); +} + +int fs_stat_path(const char *path, fs_stat *out) { + wchar_t wpath[FS_PATH_MAX]; + if (fs_widen(path, wpath, FS_PATH_MAX) != 0) + return -1; + + // _wstat64 follows reparse points and has no S_ISLNK equivalent, so without + // this archiver walks into any junction + DWORD attrs = GetFileAttributesW(wpath); + int is_reparse = attrs != INVALID_FILE_ATTRIBUTES && + (attrs & FILE_ATTRIBUTE_REPARSE_POINT) != 0; + + if (fs_stat_wide(wpath, out) != 0) { + // A link is still a real entry when its target is missing + if (!is_reparse) + return -1; + memset(out, 0, sizeof(*out)); + } + + if (is_reparse) { + out->type = FS_TYPE_SYMLINK; + out->dev = 0; + out->ino = 0; + out->nlink = 0; + } + + return 0; +} + int fs_fstat(FILE *f, fs_stat *out) { struct __stat64 st; if (_fstat64(_fileno(f), &st) != 0) diff --git a/src/compresso/frontend/archive_api.py b/src/compresso/frontend/archive_api.py index 92d36ba..6bade6c 100644 --- a/src/compresso/frontend/archive_api.py +++ b/src/compresso/frontend/archive_api.py @@ -129,6 +129,9 @@ class ArchiveOptions: remove_sources: Whether to delete what was stored once the archive is written; anything else stays, including parent directories; running with `OverwriteMode.SKIP` fails, as a skipped archive holds nothing. + dereference: Whether to store what symlinks lead to; one that's + dangling, or that leads to a directory above it, stays a link; + `remove_sources` then deletes the links, not what they lead to. """ format: str | None = None @@ -138,6 +141,7 @@ class ArchiveOptions: include: Sequence[str] = () exclude: Sequence[str] = () remove_sources: bool = False + dereference: bool = False @dataclass(frozen=True) @@ -279,7 +283,7 @@ def plan_archive( reason_if_unavailable="No source paths provided", ) - missing: list[Path] = [p for p in source_paths if not p.exists()] + missing: list[Path] = [p for p in source_paths if not os.path.lexists(p)] if missing: return ArchivePlan( sources=source_paths, @@ -359,21 +363,23 @@ def _stored_size(sources: list[Path], options: ArchiveOptions) -> int: def _stored_files(source: Path, options: ArchiveOptions) -> Iterator[os.stat_result]: - """Yield the lstat of each regular file the C walk keeps from a source. + """Yield the stat of each regular file the C walk keeps from a source. Only `exclude` prunes: a directory no include matches can still hold files that one does. Args: - source: The source path; a symlink stays a link, not its target. + source: The source path; a symlink stays a link, not its target, + unless `options.dereference` is set. options: Archive options. Yields: - Each kept regular file's lstat. + Each kept regular file's stat. """ + follow = options.dereference name = source.relative_to(source.parent).as_posix() - if source.is_symlink() or not source.is_dir(): + if not source.is_dir() or (source.is_symlink() and not follow): candidates = [(source, name)] elif path_matches(name, options.exclude): @@ -389,11 +395,37 @@ def _stored_files(source: Path, options: ArchiveOptions) -> Iterator[os.stat_res if options.include and not path_matches(entry, options.include): continue - info = os.lstat(path) + # A dangling link stays a link, which holds no bytes + if follow and not os.path.exists(path): + continue + + info = os.stat(path, follow_symlinks=follow) if stat.S_ISREG(info.st_mode): yield info +def _leads_above(path: Path, source: Path) -> bool: + """Return whether a symlink leads to a directory between it and a source. + + Args: + path: The path found below `source`. + source: The directory being walked. + + Returns: + True if `path` is a link the C walk wouldn't follow. + """ + if not path.is_symlink(): + return False + + above = path.parent + while not os.path.samefile(path, above): + if above == source: + return False + above = above.parent + + return True + + def _walk_files(source: Path, options: ArchiveOptions) -> Iterator[tuple[str, str]]: """Yield each non-directory under a source, pruning excluded directories. @@ -404,9 +436,15 @@ def _walk_files(source: Path, options: ArchiveOptions) -> Iterator[tuple[str, st Yields: Each path on disk with its entry name. """ - for top, dirs, files in os.walk(source): + follow = options.dereference + for top, dirs, files in os.walk(source, followlinks=follow): rel = Path(top).relative_to(source.parent).as_posix() - dirs[:] = [d for d in dirs if not path_matches(f"{rel}/{d}", options.exclude)] + dirs[:] = [ + d + for d in dirs + if not path_matches(f"{rel}/{d}", options.exclude) + and not (follow and _leads_above(Path(top, d), source)) + ] for f in files: yield os.path.join(top, f), f"{rel}/{f}" @@ -599,6 +637,7 @@ def run( include=self.plan.options.include, exclude=self.plan.options.exclude, remove_sources=self.plan.options.remove_sources, + dereference=self.plan.options.dereference, progress=to_core_progress(progress, total), cancel=cancel, ) diff --git a/src/compresso/frontend/dispatch.py b/src/compresso/frontend/dispatch.py index b4a922c..af9eaed 100644 --- a/src/compresso/frontend/dispatch.py +++ b/src/compresso/frontend/dispatch.py @@ -97,6 +97,8 @@ class CompressRequest: exclude: Globs an archive's entries must not match. remove_sources: Whether an archive deletes what it stored once written; see `ArchiveOptions`. + dereference: Whether an archive stores what symlinks lead to; see + `ArchiveOptions`. """ inputs: list[Path] @@ -110,6 +112,7 @@ class CompressRequest: include: Sequence[str] = () exclude: Sequence[str] = () remove_sources: bool = False + dereference: bool = False @dataclass(frozen=True) @@ -176,6 +179,7 @@ def plan_compress(request: CompressRequest) -> CompressionJob | ArchiveJob: include=request.include, exclude=request.exclude, remove_sources=request.remove_sources, + dereference=request.dereference, ) archive_job = ArchiveJob.from_paths( diff --git a/tests/python/cli/test_compress.py b/tests/python/cli/test_compress.py index 18a9081..61c021a 100644 --- a/tests/python/cli/test_compress.py +++ b/tests/python/cli/test_compress.py @@ -113,6 +113,25 @@ def test_existing_output( assert (f"{t.dest} skipped" in result.output) == ("skip" in flags) +class TestDereference: + """Test `-h`.""" + + @pytest.mark.usefixtures("needs_symlinks") + @pytest.mark.parametrize("flag", ["-h", "--dereference"]) + def test_archive_holds_the_target(self, tmp_path: Path, flag: str) -> None: + """Test that the flag reaches the archive.""" + tree = tmp_path / "tree" + tree.mkdir() + (tree / "a").write_text("hello") + (tree / "link").symlink_to("a") + output = tmp_path / "out.tar" + + invoke("compress", tree, "-o", output, flag, "-q") + + with tarfile.open(output) as tar: + assert tar.getmember("tree/link").isfile() + + class TestRemoveInput: """Test what `--rm` and `-k` do to the input.""" diff --git a/tests/python/test_archive_create.py b/tests/python/test_archive_create.py index ca54bfa..6c105d5 100644 --- a/tests/python/test_archive_create.py +++ b/tests/python/test_archive_create.py @@ -398,6 +398,195 @@ def total(*sources: Path) -> int: assert total(source_link) == 0 +@pytest.mark.usefixtures("needs_symlinks") +class TestDereference: + """Test that `dereference` stores what symlinks lead to, as bsdtar `-h`.""" + + @pytest.fixture + def tree(self, tmp_path: Path) -> Path: + """A tree with links to a file, and to a directory outside it.""" + other = tmp_path / "other" + other.mkdir() + (other / "f.txt").write_text("other") + + root = tmp_path / "tree" + root.mkdir() + (root / "a.txt").write_text("hello") + (root / "flink").symlink_to("a.txt") + (root / "dlink").symlink_to(other, target_is_directory=True) + + return root + + @staticmethod + def entries(output: Path) -> dict[str, "_core.ArchiveEntryDict"]: + return {e["path"].rstrip("/"): e for e in _core.list_archive_contents(output)} + + @pytest.mark.parametrize("fmt", ["tar", "tar.zst", "zip"]) + def test_links_become_their_targets( + self, tree: Path, tmp_path: Path, fmt: str + ) -> None: + """Test that a link to a file or a directory holds what it leads to.""" + output = tmp_path / f"out.{fmt}" + + _core.create_archive(str(output), fmt, [str(tree)], dereference=True) + + entries = self.entries(output) + assert entries["tree/flink"]["type"] == "file" + assert entries["tree/flink"]["size"] == 5 + assert entries["tree/dlink"]["type"] == "dir" + assert entries["tree/dlink/f.txt"]["size"] == 5 + + def test_links_stay_links_by_default(self, tree: Path, tmp_path: Path) -> None: + """Test that nothing is followed unless asked.""" + output = tmp_path / "out.tar" + + _core.create_archive(str(output), "tar", [str(tree)]) + + entries = self.entries(output) + assert entries["tree/dlink"]["type"] == "symlink" + assert "tree/dlink/f.txt" not in entries + + def test_source_link_is_followed(self, tree: Path, tmp_path: Path) -> None: + """Test that a link given as a source is stored under its own name.""" + link = tmp_path / "top" + link.symlink_to(tree, target_is_directory=True) + output = tmp_path / "out.tar" + + _core.create_archive(str(output), "tar", [str(link)], dereference=True) + + entries = self.entries(output) + assert entries["top"]["type"] == "dir" + assert entries["top/a.txt"]["size"] == 5 + + def test_dangling_link_stays_a_link(self, tree: Path, tmp_path: Path) -> None: + """Test that a link leading nowhere, or to itself, is stored as a link.""" + (tree / "dangling").symlink_to("missing") + (tree / "self").symlink_to("self") + output = tmp_path / "out.tar" + + _core.create_archive(str(output), "tar", [str(tree)], dereference=True) + + entries = self.entries(output) + assert entries["tree/dangling"]["type"] == "symlink" + assert entries["tree/dangling"]["link_target"] == "missing" + assert entries["tree/self"]["type"] == "symlink" + + @pytest.mark.parametrize( + ("as_source", "dereference", "warns"), + [ + (True, False, True), + (True, True, True), + (False, True, True), + (False, False, False), + ], + ) + def test_dangling_link_warns_where_data_was_expected( + self, + tree: Path, + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + as_source: bool, + dereference: bool, + warns: bool, + ) -> None: + """Test that only a named or followed dangling link is reported.""" + link = tree / "dangling" + link.symlink_to("missing") + source = link if as_source else tree + + with caplog.at_level(logging.WARNING, logger="compresso"): + _core.create_archive( + str(tmp_path / "out.tar"), "tar", [str(source)], dereference=dereference + ) + + expected = f"Stored {link} as a link: its target can't be reached" + assert (expected in caplog.text) is warns + + def test_link_to_an_ancestor_stays_a_link( + self, tree: Path, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + """Test that a loop, even through another directory, ends the walk quietly.""" + (tree / "up").symlink_to(".", target_is_directory=True) + (tree / "sub").mkdir() + (tree / "sub" / "up").symlink_to("..", target_is_directory=True) + (tmp_path / "other" / "back").symlink_to(tree, target_is_directory=True) + output = tmp_path / "out.tar" + + with caplog.at_level(logging.WARNING, logger="compresso"): + _core.create_archive(str(output), "tar", [str(tree)], dereference=True) + + entries = self.entries(output) + for name in ("tree/up", "tree/sub/up", "tree/dlink/back"): + assert entries[name]["type"] == "symlink" + assert "as a link" not in caplog.text + + def test_link_to_the_output_is_skipped(self, tree: Path, tmp_path: Path) -> None: + """Test that the archive being written isn't stored through a link.""" + output = tmp_path / "out.tar" + output.write_text("old") + (tree / "own").symlink_to(output) + + _core.create_archive( + str(output), "tar", [str(tree)], overwrite=2, dereference=True + ) + + assert "tree/own" not in self.entries(output) + + def test_excluded_link_is_not_walked(self, tree: Path, tmp_path: Path) -> None: + """Test that an exclude still prunes what a link leads to.""" + output = tmp_path / "out.tar" + + _core.create_archive( + str(output), "tar", [str(tree)], exclude=["*/dlink"], dereference=True + ) + + assert not any(name.startswith("tree/dlink") for name in self.entries(output)) + + def test_rm_deletes_links_but_not_targets(self, tree: Path, tmp_path: Path) -> None: + """Test that a link goes, source or not; its target stays, unlike GNU tar's.""" + other = tmp_path / "other" + top = tmp_path / "top" + top.symlink_to(other, target_is_directory=True) + + _core.create_archive( + str(tmp_path / "out.tar"), + "tar", + [str(tree), str(top)], + dereference=True, + remove_sources=True, + ) + + assert not os.path.lexists(tree) + assert not os.path.lexists(top) + assert (other / "f.txt").read_text() == "other" + + def test_plan_counts_what_the_walk_stores(self, tree: Path, tmp_path: Path) -> None: + """Test that the planned total follows links, loops and dangling ones.""" + (tree / "dangling").symlink_to("missing") + (tree / "up").symlink_to(".", target_is_directory=True) + (tmp_path / "other" / "back").symlink_to(tree, target_is_directory=True) + + def total(dereference: bool) -> int: + options = ArchiveOptions(format="tar", dereference=dereference) + job = ArchiveJob.from_paths([tree], tmp_path / "o.tar", options) + assert job.plan.can_run + return job.plan.total_input_size + + assert total(False) == 5 + assert total(True) == 15 + + def test_plan_keeps_a_dangling_source(self, tree: Path, tmp_path: Path) -> None: + """Test that a dangling link exists as a source, as it does to the walk.""" + link = tree / "dangling" + link.symlink_to("missing") + output = tmp_path / "o.tar" + + job = ArchiveJob.from_paths([link], output, ArchiveOptions(format="tar")) + assert job.run().ok + + assert self.entries(output)["dangling"]["type"] == "symlink" + + class TestRemoveSources: """Test that `remove_sources` deletes exactly what the archive stored.""" From b857d8dd909227b055235dbe9720bdeb46c67c0a Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Sun, 1 Nov 2026 23:52:16 +0000 Subject: [PATCH 25/31] feat: 'test' command to check archives and compressed files for damage - New 'test' command decodes each input in full and writes nothing, checking integrity based on format - A plain '.tar' has no checksum over its data, so only its headers and the data's presence are checked - Trailing data fails, by raising 'TrailingDataWarning' for the run - New '_core.verify_file' checks it, with 'VerificationPlan', plan_verification() and 'VerificationJob' added to the frontend - An unrecognised file is refused at the plan rather than part-way through - list_archive_contents() becomes a generic run_reader_pass() that the check now reuses - A plain archive now gets a headers-only pass first, so progress counts what its entries hold rather than the file's size - A NULL 'OutputTarget.path' now means discard, writing to the null device and committing nothing - Zip now reads one byte past a link target, since libzip only checks the CRC once a read reaches the end - A cancel while the reader opens is now reported as cancelled rather than an unexplained failure - Adds new test_verify.py and test_verification.py covering all test command behaviour --- src/compresso/__init__.py | 6 + src/compresso/_core.pyi | 13 ++ src/compresso/cli/__init__.py | 3 +- src/compresso/cli/verify.py | 88 ++++++++ src/compresso/csrc/_core.c | 28 +++ src/compresso/csrc/archives/archive_ops.c | 116 +++++++++- src/compresso/csrc/archives/extract.c | 3 +- src/compresso/csrc/archives/zip.c | 4 +- src/compresso/csrc/common.c | 19 +- src/compresso/csrc/common.h | 5 + src/compresso/csrc/fsutil.h | 6 +- src/compresso/frontend/__init__.py | 6 + src/compresso/frontend/api.py | 111 ++++++++++ src/compresso/meson.build | 1 + tests/python/cli/test_app.py | 15 +- tests/python/cli/test_verify.py | 62 ++++++ tests/python/frontend/test_verification.py | 238 +++++++++++++++++++++ tests/python/helpers.py | 16 +- tests/python/test_core.py | 16 ++ 19 files changed, 732 insertions(+), 24 deletions(-) create mode 100644 src/compresso/cli/verify.py create mode 100644 tests/python/cli/test_verify.py create mode 100644 tests/python/frontend/test_verification.py diff --git a/src/compresso/__init__.py b/src/compresso/__init__.py index 85a72c2..6238d1a 100644 --- a/src/compresso/__init__.py +++ b/src/compresso/__init__.py @@ -21,8 +21,11 @@ CompressionPlan, DecompressionJob, DecompressionPlan, + VerificationJob, + VerificationPlan, plan_compression, plan_decompression, + plan_verification, ) from .frontend.archive_api import ( ArchiveEntry, @@ -71,6 +74,8 @@ "SourceRemovalError", "SymlinkMode", "TrailingDataWarning", + "VerificationJob", + "VerificationPlan", "benchmark_file", "compress_file", "decompress_file", @@ -81,4 +86,5 @@ "plan_compression", "plan_decompression", "plan_extraction", + "plan_verification", ] diff --git a/src/compresso/_core.pyi b/src/compresso/_core.pyi index c8cd4d3..4265faa 100644 --- a/src/compresso/_core.pyi +++ b/src/compresso/_core.pyi @@ -229,6 +229,19 @@ def extract_archive( as a name in `files` does, after the rest is extracted. """ +def verify_file( + path: StrOrBytesPath, + *, + threads: int = ..., + progress: ProgressFn | None = ..., + cancel: CancelToken | None = ..., +) -> None: + """Decode all of an archive or compressed file, writing nothing. + + Raises what extracting it would for damaged data, so it proves only what + the format checks: a plain tar's file data carries no checksum. + """ + def detect_format(file_path: StrOrBytesPath) -> str: """Name a file's format, from its magic bytes and then its extension.""" diff --git a/src/compresso/cli/__init__.py b/src/compresso/cli/__init__.py index 93c565c..cb8829c 100644 --- a/src/compresso/cli/__init__.py +++ b/src/compresso/cli/__init__.py @@ -2,7 +2,7 @@ from __future__ import annotations -from . import benchmark, compress, extract, formats, info +from . import benchmark, compress, extract, formats, info, verify from . import list as list_ from ._app import app @@ -12,6 +12,7 @@ _COMMAND_MODULES = ( compress, extract, + verify, list_, info, formats, diff --git a/src/compresso/cli/verify.py b/src/compresso/cli/verify.py new file mode 100644 index 0000000..99502a1 --- /dev/null +++ b/src/compresso/cli/verify.py @@ -0,0 +1,88 @@ +"""The `test` command, checking the integrity of archives and compressed files.""" + +from __future__ import annotations + +import sys +import warnings +from pathlib import Path +from typing import Annotated + +from .._core import TrailingDataWarning +from ..frontend.api import VerificationJob +from ._app import app +from ._render import cancelled, fail, succeed +from ._run import InputFailed, require_files, run_all, run_job + + +def _verify_one(job: VerificationJob, quiet: bool) -> None: + """Check one input. + + Args: + job: The planned job. + quiet: Whether to suppress all output. + """ + plan = job.plan + + if not plan.can_run: + raise InputFailed(f"{plan.src}: {plan.reason_if_unavailable}") + + size: int = plan.src.stat().st_size + run_job(job, plan.src, "Testing", size, quiet=quiet, what="Test") + + if not quiet: + succeed(f"OK: {plan.src}") + + +def verify( + inputs: Annotated[ + list[Path], + app.Argument(help="Archives or compressed files", readable=False), + ], + threads: Annotated[ + int, + app.Option( + "--threads", + "-T", + min=0, + help="Worker threads for xz, or 0 for one per core", + ), + ] = 0, + quiet: Annotated[ + bool, app.Option("--quiet", "-q", help="Suppress all output") + ] = False, +) -> None: + """Check archives and compressed files for damage, writing nothing. + \f + + Each input is decoded in full, as `extract` would, so this catches whatever + its format can detect: a checksum or length that doesn't match, or a file + cut short. Trailing data fails too, as `gzip -t` treats it, though `extract` + only warns. A plain `.tar` has no checksum over its file data, so only its + headers are checked, and that the data they declare is there. + + Args: + inputs: The archives or compressed files to check. + threads: Worker threads for xz (default: one per core). + quiet: If True, suppress all output. + """ + require_files(inputs) + + try: + jobs = (VerificationJob.from_file(src=path, threads=threads) for path in inputs) + + with warnings.catch_warnings(): + warnings.simplefilter("error", TrailingDataWarning) + code = run_all(jobs, lambda job: _verify_one(job, quiet), "Test") + + sys.exit(code) + + except KeyboardInterrupt: + cancelled("Test") + + except Exception as e: + fail(f"Unexpected error: {e}") + + +def register() -> None: + """Attach this command to the app.""" + app.command(name="test", aliases=["t"])(verify) diff --git a/src/compresso/csrc/_core.c b/src/compresso/csrc/_core.c index 54a984b..9903d65 100644 --- a/src/compresso/csrc/_core.c +++ b/src/compresso/csrc/_core.c @@ -765,6 +765,31 @@ static PyObject *py_decompress_standalone(PyObject *self, PyObject *args, return finish_call(self, rc, &ctx, actual_path); } +static PyObject *py_verify_file(PyObject *self, PyObject *args, + PyObject *kwargs) { + static char *kwlist[] = {"path", "threads", "progress", "cancel", NULL}; + + PyObject *path_bytes = NULL; + int threads = 0; // One per core + PyObject *progress = NULL; + PyObject *cancel = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&|$iOO", kwlist, + PyUnicode_FSConverter, &path_bytes, &threads, + &progress, &cancel)) { + return NULL; // Error already set + } + + CoreContext ctx; + int result = -1; + if (core_context_init(&ctx, self, progress, cancel, threads) == 0) { + result = verify_file(PyBytes_AS_STRING(path_bytes), &ctx); + } + + Py_DECREF(path_bytes); + return finish_call(self, result, &ctx, NULL); +} + // ---- Format Detection Methods ---- static PyObject *py_detect_format(PyObject *Py_UNUSED(self), PyObject *args) { @@ -995,6 +1020,9 @@ static PyMethodDef CoreMethods[] = { {"decompress_standalone", (PyCFunction)(void (*)(void))py_decompress_standalone, METH_VARARGS | METH_KEYWORDS, "Decompress a standalone format file."}, + {"verify_file", (PyCFunction)(void (*)(void))py_verify_file, + METH_VARARGS | METH_KEYWORDS, + "Decode an archive or compressed file, writing nothing."}, {"detect_format", (PyCFunction)py_detect_format, METH_VARARGS, "Detect the format of a file."}, diff --git a/src/compresso/csrc/archives/archive_ops.c b/src/compresso/csrc/archives/archive_ops.c index c97a7d5..e8a433b 100644 --- a/src/compresso/csrc/archives/archive_ops.c +++ b/src/compresso/csrc/archives/archive_ops.c @@ -505,12 +505,19 @@ static PyObject *read_archive_entries(const CArchive *archive, void *reader, return list; } -PyObject *list_archive_contents(const char *archive_path, CoreContext *ctx) { +// One read of an archive; `data_ctx` takes the progress of its entries' data, +// and is NULL when a codec stage already counts the input; returns 0 or a +// negative code with an exception set +typedef int (*ReaderPass)(const CArchive *archive, void *reader, void *state, + CoreContext *data_ctx, CoreContext *ctx); + +static int run_reader_pass(const char *archive_path, ReaderPass pass, + void *state, CoreContext *ctx) { const CArchive *archive; CompressionPipeline pipe; FILE *src = open_archive_source(archive_path, &archive, &pipe, ctx); if (!src) - return NULL; + return -1; // Decoded as libarchive reads CodecStream *stream = NULL; @@ -527,24 +534,111 @@ PyObject *list_archive_contents(const char *archive_path, CoreContext *ctx) { reader = archive->create_reader(archive_path, ctx); } - PyObject *list = reader ? read_archive_entries(archive, reader, ctx) : NULL; - int failed = list == NULL; - // A close failure after a full listing would otherwise return the list - if (reader && archive->close_reader(reader, failed) != 0) - failed = 1; + // A cancel while opening fails the open, which can only return NULL + int ret = reader ? pass(archive, reader, state, stream ? NULL : ctx, ctx) + : ctx_cancelled(ctx) ? COMP_CANCELLED + : -1; + + // A close failure after a full pass would otherwise go unreported + if (reader && archive->close_reader(reader, ret != 0) != 0 && ret == 0) + ret = -1; - if (stream && close_codec_source(stream, src, !failed) != 0) - failed = 1; - else if (!stream && src) + if (stream) { + int rc = close_codec_source(stream, src, ret == 0); + if (ret == 0) + ret = rc; + } else if (src) { fclose(src); + } + return ret; +} + +static int list_pass(const CArchive *archive, void *reader, void *state, + CoreContext *data_ctx, CoreContext *ctx) { + (void)data_ctx; + PyObject **list = state; + *list = read_archive_entries(archive, reader, ctx); + return *list ? 0 : -1; +} - if (failed) { +PyObject *list_archive_contents(const char *archive_path, CoreContext *ctx) { + PyObject *list = NULL; + if (run_reader_pass(archive_path, list_pass, &list, ctx) != 0) { Py_XDECREF(list); return NULL; } return list; } +typedef struct { + FILE *sink; // Takes each file's data; NULL to only add up `declared` + uint64_t declared; +} VerifyState; + +static int verify_pass(const CArchive *archive, void *reader, void *state, + CoreContext *data_ctx, CoreContext *ctx) { + VerifyState *verify = state; + ArchiveEntry entry = {0}; + int ret; + + // NULL: the data pass reads the same headers again, and warns once there + while ((ret = archive->get_next_entry(reader, &entry, + verify->sink ? ctx : NULL)) == 1) { + int rc = ctx_advance(ctx, 0); + if (rc == 0 && entry.type == ENTRY_FILE) { + uint64_t written; + if (verify->sink) + rc = archive->extract_entry_data(reader, verify->sink, UINT64_MAX, + &written, data_ctx); + else + verify->declared += entry.size; + } + entry_reset(&entry); + if (rc != 0) + return rc; + } + return ret; +} + +static int verify_archive(const char *archive_path, + const CompressionPipeline *pipeline, + CoreContext *ctx) { + FILE *sink = fs_fopen(FS_NULL_DEVICE, "wb"); + if (!sink) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, FS_NULL_DEVICE); + return -1; + } + + // A codec stream is decoded once, counting its compressed bytes; a plain file + // gets a headers pass first, for the total its data counts towards + VerifyState state = {NULL, 0}; + int ret = 0; + if (pipeline->codec == FORMAT_UNKNOWN) { + ret = run_reader_pass(archive_path, verify_pass, &state, ctx); + ctx_begin_stage(ctx, state.declared); + } + if (ret == 0) { + state.sink = sink; + ret = run_reader_pass(archive_path, verify_pass, &state, ctx); + } + + fclose(sink); + return ret; +} + +int verify_file(const char *path, CoreContext *ctx) { + Format format; + FILE *src = open_source(path, &format); + if (!src) + return -1; + fclose(src); + + CompressionPipeline pipe = pipeline_from_format(format, path); + if (pipe.archive != ARCHIVE_NONE) + return verify_archive(path, &pipe, ctx); + return decompress_file(path, NULL, ALGO_NONE, 0, NULL, 0, ctx); +} + // ---- Archive Capabilities ---- PyObject *get_archive_capabilities(void) { diff --git a/src/compresso/csrc/archives/extract.c b/src/compresso/csrc/archives/extract.c index 899fda0..75ee161 100644 --- a/src/compresso/csrc/archives/extract.c +++ b/src/compresso/csrc/archives/extract.c @@ -814,7 +814,8 @@ int extract_archive(const char *archive_path, const char *output_dir, reader = archive->create_reader(archive_path, ctx); } if (!reader) { - ret = -1; + // A cancel while opening fails the open, which can only return NULL + ret = ctx_cancelled(ctx) ? COMP_CANCELLED : -1; break; } diff --git a/src/compresso/csrc/archives/zip.c b/src/compresso/csrc/archives/zip.c index 18116c0..d528bcf 100644 --- a/src/compresso/csrc/archives/zip.c +++ b/src/compresso/csrc/archives/zip.c @@ -524,9 +524,11 @@ static int zip_read_link_target(ZipReader *reader, zip_int64_t idx, return -1; } + // The CRC is only checked by a read that reaches the end char target[FS_PATH_MAX]; + char past_end; zip_int64_t n = zip_fread(zf, target, entry->size); - if (n < 0) { + if (n < 0 || zip_fread(zf, &past_end, 1) < 0) { set_zip_error(reader->errors, zip_file_get_error(zf), "Error reading from archive", reader->input_path, 1, 0); zip_fclose(zf); diff --git a/src/compresso/csrc/common.c b/src/compresso/csrc/common.c index ccf4156..f3e7914 100644 --- a/src/compresso/csrc/common.c +++ b/src/compresso/csrc/common.c @@ -198,6 +198,10 @@ static void report_actual(const OutputTarget *out, const char *path) { } int output_check(const char *src_path, const OutputTarget *out) { + if (!out->path) { + return 0; + } + char resolved[FS_PATH_MAX]; int rc = fs_resolve_conflict(out->path, out->overwrite, resolved, sizeof(resolved)); @@ -222,6 +226,14 @@ int output_check(const char *src_path, const OutputTarget *out) { } FILE *output_open(const OutputTarget *out, char *temp) { + if (!out->path) { + FILE *sink = fs_fopen(FS_NULL_DEVICE, "wb"); + if (!sink) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, FS_NULL_DEVICE); + } + return sink; + } + static const char SUFFIX[] = ".compresso-XXXXXX"; const char *slash = fs_last_sep(out->path); size_t dir_len = slash ? (size_t)(slash - out->path + 1) : 0; @@ -275,15 +287,16 @@ int output_finish(int err, FILE *input, FILE *output, const char *temp, // Flushed first, as a later write would move the mtime again; a failed flush // is left for codec_finish_file to report; best-effort, as in extraction fs_stat src; - if (err == 0 && out->keep_source_metadata && input && output && + int writes = output && out->path; + if (err == 0 && out->keep_source_metadata && input && writes && fflush(output) == 0 && fs_fstat(input, &src) == 0) { fs_futimens(output, src.mtime); fs_fchmod(output, src.mode); } - err = codec_finish_file(err, input, output, output ? temp : NULL, errors, + err = codec_finish_file(err, input, output, writes ? temp : NULL, errors, failure_message); - if (err != 0 || !output) { + if (err != 0 || !writes) { return err; } return output_commit(temp, out); diff --git a/src/compresso/csrc/common.h b/src/compresso/csrc/common.h index c15828a..b00c504 100644 --- a/src/compresso/csrc/common.h +++ b/src/compresso/csrc/common.h @@ -191,4 +191,9 @@ int decompress_file(const char *src_path, const char *dst_path, AlgoID algo, int overwrite_existing, char *out_actual_path, size_t out_actual_path_size, CoreContext *ctx); +// Decode all of an archive or compressed file, writing nothing, so only what +// the format checks can fail it: a plain tar's data is proved to be there, but +// not to be right; returns 0, -1 with an exception set, or COMP_CANCELLED +int verify_file(const char *path, CoreContext *ctx); + #endif // COMMON_H diff --git a/src/compresso/csrc/fsutil.h b/src/compresso/csrc/fsutil.h index 3b149b7..f12b19e 100644 --- a/src/compresso/csrc/fsutil.h +++ b/src/compresso/csrc/fsutil.h @@ -14,6 +14,7 @@ // so prefix comparisons must test both #if defined(_WIN32) || defined(_WIN64) #define FS_IS_SEP(c) ((c) == '/' || (c) == '\\') +#define FS_NULL_DEVICE "NUL" #include @@ -22,6 +23,7 @@ int fs_widen(const char *utf8, wchar_t *out, int out_count); #else #define FS_IS_SEP(c) ((c) == '/') +#define FS_NULL_DEVICE "/dev/null" #endif typedef enum { @@ -155,8 +157,8 @@ int fs_commit_temp(const char *temp, const char *dst, int overwrite_existing, // A file output, written to a temp beside `path` and committed on success typedef struct { - const char *path; - int overwrite; // The 0-3 scheme above + const char *path; // NULL discards the output, leaving only its failures + int overwrite; // The 0-3 scheme above char *actual; // Optional; receives the path written, or skipped size_t actual_size; int owner_only; // 0600, for an intermediate that holds a whole archive diff --git a/src/compresso/frontend/__init__.py b/src/compresso/frontend/__init__.py index dd08af8..dc7c47a 100644 --- a/src/compresso/frontend/__init__.py +++ b/src/compresso/frontend/__init__.py @@ -8,8 +8,11 @@ CompressionPlan, DecompressionJob, DecompressionPlan, + VerificationJob, + VerificationPlan, plan_compression, plan_decompression, + plan_verification, ) from .archive_api import ( ArchiveEntry, @@ -45,8 +48,11 @@ "OverwriteMode", "ProgressCallback", "SymlinkMode", + "VerificationJob", + "VerificationPlan", "plan_archive", "plan_compression", "plan_decompression", "plan_extraction", + "plan_verification", ] diff --git a/src/compresso/frontend/api.py b/src/compresso/frontend/api.py index 585adf0..ef72bef 100644 --- a/src/compresso/frontend/api.py +++ b/src/compresso/frontend/api.py @@ -18,6 +18,7 @@ detect_format, format_is_archive, gzip_stored_name, + verify_file, ) from .._core import get_default_backend_for_strategy as default_backend from .._levels import to_core_level @@ -120,6 +121,23 @@ class DecompressionPlan: mtime: int | None = None +@dataclass(frozen=True) +class VerificationPlan: + """Holds an integrity check of an archive or compressed file. + + Attributes: + src: Source file path. + can_run: Whether the check can proceed. + reason_if_unavailable: Reason it cannot, if applicable. + threads: Worker threads for xz/lzma, or 0 for one per core. + """ + + src: Path + can_run: bool = True + reason_if_unavailable: str | None = None + threads: int = 0 + + def plan_compression( src: str | Path, dest: str | Path | None = None, @@ -395,6 +413,32 @@ def plan( return plan(None, True) +def plan_verification(src: str | Path, threads: int = 0) -> VerificationPlan: + """Plan an integrity check of an archive or compressed file. + + Args: + src: Source file path. + threads: Worker threads for xz/lzma, or 0 for one per core. + + Returns: + VerificationPlan: The resulting plan. + """ + src_path = Path(src) + + try: + recognised: bool = detect_format(src_path) != "unknown" + + # The caller names the file with the error + except OSError as e: + return VerificationPlan(src_path, False, e.strerror or str(e), threads) + + if not recognised: + reason = "not a recognised archive or compressed file" + return VerificationPlan(src_path, False, reason, threads) + + return VerificationPlan(src_path, threads=threads) + + class CompressionJob(ThreadedJob[CompressionPlan]): """Compression job high-level wrapper.""" @@ -648,3 +692,70 @@ def run( error=e, plan=self.plan, ) + + +class VerificationJob(ThreadedJob[VerificationPlan]): + """Integrity check high-level wrapper.""" + + def __init__(self, plan: VerificationPlan) -> None: + """Initialise the verification job. + + Args: + plan: The verification plan to execute. + """ + self.plan: VerificationPlan = plan + + @classmethod + def from_file(cls, src: str | Path, threads: int = 0) -> Self: + """Create a VerificationJob from a file path. + + Args: + src: Source file path. + threads: Worker threads for xz/lzma, or 0 for one per core. + + Returns: + VerificationJob: The created job. + """ + return cls(plan=plan_verification(src, threads)) + + def run( + self, + progress: ProgressCallback | None = None, + cancel: CancelToken | None = None, + ) -> JobResult: + """Decode the whole file, writing nothing. + + Args: + progress: Optional progress callback, invoked with + `(fraction, done_bytes, total_bytes)`; counts the bytes a plain + archive's entries hold, or else the compressed bytes consumed. + cancel: Optional `CancelToken`. + + Returns: + JobResult: The result, whose `error` says what is damaged. + """ + if not self.plan.can_run: + return JobResult( + ok=False, + error=RuntimeError(self.plan.reason_if_unavailable or "Cannot test"), + plan=self.plan, + ) + + try: + verify_file( + self.plan.src, + threads=self.plan.threads, + progress=to_core_progress(progress, 0), + cancel=cancel, + ) + + return JobResult(ok=True, error=None, plan=self.plan) + + # Cancellation is a deliberate stop, not a failure; see + # `CompressionJob.run`. + except Cancelled: + return JobResult(ok=False, error=None, plan=self.plan, cancelled=True) + + # `run` reports failure through JobResult rather than raising + except Exception as e: # noqa: BLE001 + return JobResult(ok=False, error=e, plan=self.plan) diff --git a/src/compresso/meson.build b/src/compresso/meson.build index c3b12c3..1bd25bd 100644 --- a/src/compresso/meson.build +++ b/src/compresso/meson.build @@ -59,6 +59,7 @@ py.install_sources( 'cli/formats.py', 'cli/info.py', 'cli/list.py', + 'cli/verify.py', subdir: 'compresso/cli', ) diff --git a/tests/python/cli/test_app.py b/tests/python/cli/test_app.py index bacaa9f..ebe6e81 100644 --- a/tests/python/cli/test_app.py +++ b/tests/python/cli/test_app.py @@ -23,7 +23,7 @@ format_size, format_time, ) -from compresso.frontend.api import CompressionJob, DecompressionJob +from compresso.frontend.api import CompressionJob, DecompressionJob, VerificationJob from compresso.frontend.archive_api import ( ArchiveJob, ExtractJob, @@ -88,6 +88,7 @@ def test_commands_are_listed_in_workflow_order(self) -> None: for aliases in [ "(c)", "(x)", + "(t)", "(l)", "(i)", "List the available", @@ -101,6 +102,7 @@ def test_commands_are_listed_in_workflow_order(self) -> None: [ ("c", "compress"), ("x", "extract"), + ("t", "test"), ("l", "list"), ("i", "info"), ("b", "benchmark"), @@ -301,6 +303,7 @@ class TestExitCodes: ("info", "nope.comp"), ("extract", "nope.gz"), ("list", "nope.zip"), + ("test", "nope.gz"), ], ) def test_bad_input_is_usage(self, payload: Path, command: str, name: str) -> None: @@ -402,6 +405,7 @@ class TestInterruption: (["compress"], "Compression cancelled"), (["info"], "Inspection cancelled"), (["list"], "Listing cancelled"), + (["test"], "Test cancelled"), (["formats"], "Format listing cancelled"), ], ) @@ -419,6 +423,7 @@ def interrupt(*args: object, **kwargs: object) -> None: raise KeyboardInterrupt monkeypatch.setattr(CompressionJob, "from_file", interrupt) + monkeypatch.setattr(VerificationJob, "from_file", interrupt) monkeypatch.setattr(info_cmd, "detect_format", interrupt) monkeypatch.setattr(list_cmd, "detect_format", interrupt) monkeypatch.setattr(formats_cmd, "list_capabilities", interrupt) @@ -442,7 +447,7 @@ def test_threads_reach_every_job( make_archive: Callable[[str], Path], monkeypatch: pytest.MonkeyPatch, ) -> None: - """Test that compress and extract pass -T on, one file or many.""" + """Test that compress, extract and test pass -T on, one file or many.""" archive = make_archive("tar") seen: list[int] = [] @@ -456,18 +461,20 @@ def record(**kwargs: Any) -> None: monkeypatch.setattr(ArchiveJob, "from_paths", record) monkeypatch.setattr(DecompressionJob, "from_file", record) monkeypatch.setattr(ExtractJob, "from_archive", record) + monkeypatch.setattr(VerificationJob, "from_file", record) for command, source in [ ("compress", payload), ("compress", source_tree), ("extract", payload), ("extract", archive), + ("test", archive), ]: invoke(command, source, "-T", "3", expect=EXIT_CANCELLED) - assert seen == [3, 3, 3, 3] + assert seen == [3, 3, 3, 3, 3] - @pytest.mark.parametrize("command", ["compress", "extract"]) + @pytest.mark.parametrize("command", ["compress", "extract", "test"]) def test_negative_threads_is_usage(self, payload: Path, command: str) -> None: """Test that a count below 0 is refused before any work starts.""" invoke(command, payload, "-T", "-1", expect=EXIT_USAGE) diff --git a/tests/python/cli/test_verify.py b/tests/python/cli/test_verify.py new file mode 100644 index 0000000..c586837 --- /dev/null +++ b/tests/python/cli/test_verify.py @@ -0,0 +1,62 @@ +"""Tests for `compresso test`.""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +import pytest + +from compresso.cli._render import EXIT_FAILED + +from ..helpers import flip_middle_byte +from .conftest import invoke + + +@pytest.fixture +def packed(payload: Path, tmp_path: Path) -> Path: + """`payload` as a gzip file.""" + path = tmp_path / "p.gz" + invoke("compress", payload, "-o", path, "-q") + + return path + + +class TestVerify: + """Test the `test` command.""" + + def test_good_input_says_ok(self, packed: Path) -> None: + """Test that each input gets one line naming it.""" + assert f"OK: {packed}" in invoke("test", packed).output + + def test_quiet_prints_nothing(self, packed: Path) -> None: + """Test that -q leaves only the exit code.""" + assert invoke("test", "-q", packed).output == "" + + def test_damage_fails_and_the_rest_still_run( + self, packed: Path, tmp_path: Path + ) -> None: + """Test that a bad input is named, and the next input is still checked.""" + bad = tmp_path / "bad.gz" + shutil.copy(packed, bad) + flip_middle_byte(bad) + + output = invoke("test", bad, packed, expect=EXIT_FAILED).output + + assert "Test failed" in output + assert f"OK: {packed}" in output + + def test_trailing_data_fails(self, packed: Path) -> None: + """Test that it fails here, as with `gzip -t`, though `extract` only warns.""" + packed.write_bytes(packed.read_bytes() + b"not gzip data") + + output = invoke("test", packed, expect=EXIT_FAILED).output + + assert "trailing data" in output + + def test_unrecognised_file_fails(self, tmp_path: Path) -> None: + """Test that a file of no known format is an input failure.""" + notes = tmp_path / "notes.txt" + notes.write_text("just some text") + + assert "not a recognised" in invoke("test", notes, expect=EXIT_FAILED).output diff --git a/tests/python/frontend/test_verification.py b/tests/python/frontend/test_verification.py new file mode 100644 index 0000000..7bccdcb --- /dev/null +++ b/tests/python/frontend/test_verification.py @@ -0,0 +1,238 @@ +"""Tests for the frontend integrity check.""" + +import os +import stat +import tarfile +import zipfile +from pathlib import Path + +import pytest + +from compresso import CancelToken, TrailingDataWarning +from compresso.frontend.api import ( + CompressionJob, + CompressionOptions, + VerificationJob, + plan_verification, +) +from compresso.frontend.archive_api import ArchiveJob, ArchiveOptions + +from ..helpers import Recorder, flip_middle_byte + +CODECS = ["gz", "bz2", "xz", "zst", "lz4", "sz"] +ARCHIVES = ["tar.gz", "tar.bz2", "tar.xz", "tar.zst", "tar.lz4", "zip"] + + +@pytest.fixture +def source(tmp_path: Path) -> Path: + """A file holding enough incompressible data to survive one flipped byte.""" + path = tmp_path / "source.bin" + path.write_bytes(os.urandom(64 * 1024)) + + return path + + +@pytest.fixture +def tree(tmp_path: Path) -> Path: + """A directory of two files.""" + root = tmp_path / "tree" + root.mkdir() + (root / "a.bin").write_bytes(os.urandom(32 * 1024)) + (root / "b.bin").write_bytes(os.urandom(32 * 1024)) + + return root + + +def compressed(source: Path, suffix: str, algo: str | None = None) -> Path: + """Compress `source` beside itself.""" + output = source.with_suffix(f".{suffix}") + fmt = None if suffix == "comp" else suffix + options = CompressionOptions(algo=algo, format=fmt) + result = CompressionJob.from_file(source, output, options).run() + assert result.ok, result.error + + return output + + +def archived(tree: Path, fmt: str) -> Path: + """Archive `tree` beside itself.""" + output = tree.with_name(f"out.{fmt}") + result = ArchiveJob.from_paths([tree], output, ArchiveOptions(format=fmt)).run() + assert result.ok, result.error + + return output + + +def cut_short(path: Path) -> None: + """Drop the last third of `path`.""" + data = path.read_bytes() + path.write_bytes(data[: len(data) * 2 // 3]) + + +def assert_damage_caught(path: Path) -> None: + """Assert that `path` passes, then fails once flipped or cut short.""" + good = path.read_bytes() + assert VerificationJob.from_file(path).run().ok + + flip_middle_byte(path) + assert not VerificationJob.from_file(path).run().ok + + path.write_bytes(good) + cut_short(path) + assert not VerificationJob.from_file(path).run().ok + + +class TestSingleFile: + """Test standalone codecs and the `.comp` container.""" + + @pytest.mark.parametrize("suffix", CODECS) + def test_each_codec(self, source: Path, suffix: str) -> None: + """Test that a good file passes, and each kind of damage fails it.""" + assert_damage_caught(compressed(source, suffix)) + + def test_each_comp_algorithm(self, source: Path, compression_algo: str) -> None: + """Test that `.comp` damage is caught through every backend.""" + assert_damage_caught(compressed(source, "comp", compression_algo)) + + def test_trailing_data_only_warns_here(self, source: Path) -> None: + """Test that the job leaves the verdict on trailing data to its caller.""" + path = compressed(source, "gz") + path.write_bytes(path.read_bytes() + b"not gzip data") + + with pytest.warns(TrailingDataWarning): + assert VerificationJob.from_file(path).run().ok + + +class TestArchives: + """Test archives, whose entries are decoded too.""" + + @pytest.mark.parametrize("fmt", ARCHIVES) + def test_each_format(self, tree: Path, fmt: str) -> None: + """Test that a good archive passes, and each kind of damage fails it.""" + assert_damage_caught(archived(tree, fmt)) + + def test_plain_tar_has_no_data_checksum(self, tree: Path) -> None: + """Test that only what tar can detect fails it, unlike every other format.""" + path = archived(tree, "tar") + good = path.read_bytes() + with tarfile.open(path) as tar: + data_at = tar.getmembers()[-1].offset_data + + damaged = bytearray(good) + damaged[data_at] ^= 0xFF + path.write_bytes(damaged) + assert VerificationJob.from_file(path).run().ok + + # A header has a checksum of its own + damaged = bytearray(good) + damaged[data_at - 512] ^= 0xFF + path.write_bytes(damaged) + assert not VerificationJob.from_file(path).run().ok + + path.write_bytes(good) + cut_short(path) + assert not VerificationJob.from_file(path).run().ok + + def test_plain_tar_cut_between_entries_passes(self, tree: Path) -> None: + """Test that a cut leaving whole entries behind passes, as bsdtar has it.""" + path = archived(tree, "tar") + with tarfile.open(path) as tar: + last = tar.getmembers()[-1] + + path.write_bytes(path.read_bytes()[: last.offset]) + + assert VerificationJob.from_file(path).run().ok + + def test_stored_zip_entry_checks_its_crc(self, tmp_path: Path) -> None: + """Test that an uncompressed entry, which no decoder would catch, fails.""" + path = tmp_path / "stored.zip" + with zipfile.ZipFile(path, "w", zipfile.ZIP_STORED) as archive: + archive.writestr("data", os.urandom(4096)) + + flip_middle_byte(path) + + assert "CRC" in str(VerificationJob.from_file(path).run().error) + + def test_zip_symlink_target_checks_its_crc(self, tmp_path: Path) -> None: + """Test that a link, whose data is read with its header, is checked too.""" + path = tmp_path / "link.zip" + info = zipfile.ZipInfo("link") + info.create_system = 3 # Unix, or the mode below would be ignored + info.external_attr = (stat.S_IFLNK | 0o777) << 16 + with zipfile.ZipFile(path, "w") as archive: + archive.writestr(info, "some-link-target") + + assert VerificationJob.from_file(path).run().ok + + data = path.read_bytes() + path.write_bytes(data.replace(b"some-link-target", b"sOme-link-target")) + + assert "CRC" in str(VerificationJob.from_file(path).run().error) + + @pytest.mark.parametrize("fmt", ["tar", "zip"]) + def test_plain_archive_progress_counts_its_data(self, tree: Path, fmt: str) -> None: + """Test that the total is what the entries hold, not the archive's size.""" + recorder = Recorder() + + result = VerificationJob.from_file(archived(tree, fmt)).run(progress=recorder) + + assert result.ok + assert recorder.calls[-1] == (64 * 1024, 64 * 1024) + + def test_compressed_archive_progress_counts_its_input(self, tree: Path) -> None: + """Test that a codec stage reports the compressed bytes it consumes.""" + path = archived(tree, "tar.gz") + recorder = Recorder() + + VerificationJob.from_file(path).run(progress=recorder) + + assert recorder.calls[-1] == (path.stat().st_size, path.stat().st_size) + + +class TestRun: + """Test what holds for every input.""" + + @pytest.mark.parametrize("damaged", [False, True]) + @pytest.mark.parametrize("build", ["gz", "comp", "tar.gz", "zip"]) + def test_writes_nothing( + self, source: Path, tree: Path, build: str, damaged: bool + ) -> None: + """Test that the input and its directory are left as they were.""" + is_archive = build in ARCHIVES + path = archived(tree, build) if is_archive else compressed(source, build) + if damaged: + flip_middle_byte(path) + + before = sorted(path.parent.rglob("*")) + content = path.read_bytes() + + result = VerificationJob.from_file(path).run() + + assert result.ok is not damaged + assert sorted(path.parent.rglob("*")) == before + assert path.read_bytes() == content + + @pytest.mark.parametrize("fmt", ["tar", "tar.gz", "zip"]) + def test_cancelled_archive(self, tree: Path, fmt: str) -> None: + """Test that a cancel is reported as one rather than as damage.""" + token = CancelToken() + token.cancel() + + result = VerificationJob.from_file(archived(tree, fmt)).run(cancel=token) + + assert result.cancelled + assert result.error is None + + def test_unrecognised_file_cannot_run(self, tmp_path: Path) -> None: + """Test that a file of no known format is refused at the plan.""" + path = tmp_path / "notes.txt" + path.write_text("just some text") + + plan = plan_verification(path) + + assert not plan.can_run + assert not VerificationJob(plan).run().ok + + def test_missing_file_cannot_run(self, tmp_path: Path) -> None: + """Test that a missing input surfaces as an unavailable plan.""" + assert not plan_verification(tmp_path / "nope.gz").can_run diff --git a/tests/python/helpers.py b/tests/python/helpers.py index e3c867d..bc57ec4 100644 --- a/tests/python/helpers.py +++ b/tests/python/helpers.py @@ -12,7 +12,7 @@ from typing import Any, NamedTuple from compresso.frontend._job import ThreadedJob -from compresso.frontend.api import CompressionJob, DecompressionJob +from compresso.frontend.api import CompressionJob, DecompressionJob, VerificationJob from compresso.frontend.archive_api import ArchiveJob, ExtractJob # Crosses the 1 MiB report floor (`CTX_MIN_INTERVAL`) several times over @@ -45,6 +45,17 @@ def renamed(name: str, n: int = 2) -> str: return f"{stem}{_CONFLICT_SUFFIX.format(n)}{dot}{extension}" +def flip_middle_byte(path: Path) -> None: + """Damage a file while leaving its headers alone. + + Args: + path: The file to damage. + """ + data = bytearray(path.read_bytes()) + data[len(data) // 2] ^= 0xFF + path.write_bytes(data) + + class Recorder: """Collects progress callbacks for assertions.""" @@ -119,6 +130,9 @@ class JobFactory(NamedTuple): "source_tar", lambda src, out: ExtractJob.from_archive(archive=src, output_dir=out), ), + "verify": JobFactory( + "compressed_file", lambda src, _: VerificationJob.from_file(src=src) + ), } diff --git a/tests/python/test_core.py b/tests/python/test_core.py index 05e9b5e..c08d1e5 100644 --- a/tests/python/test_core.py +++ b/tests/python/test_core.py @@ -85,6 +85,22 @@ def test_each_module_raises_its_own(self, tmp_path: Path) -> None: fresh.compress_file(source, tmp_path / "out.comp", cancel=token) +class TestCancelledBeforeReading: + """Test a cancel landing while a compressed archive's reader opens.""" + + def test_extract_reports_the_cancel(self, tmp_path: Path) -> None: + """Test that the cancel is raised, not lost with the failed open.""" + source = tmp_path / "f" + source.write_bytes(b"data") + archive = tmp_path / "a.tar.gz" + _core.create_archive(archive, "tar.gz", [source]) + token = _core.CancelToken() + token.cancel() + + with pytest.raises(Cancelled): + _core.extract_archive(archive, tmp_path / "out", [], cancel=token) + + class TestCapabilities: """Test the get_capabilities function.""" From 78334cc78d510bc75ba460c8d33923269524a0aa Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Wed, 4 Nov 2026 21:10:41 +0000 Subject: [PATCH 26/31] feat(info): report what a compressed file's own metadata states - A standalone gzip codec file now shows the name and mtime its header stores - Also shows the uncompressed size an xz index or a zstd, lz4 or snappy header states, and the ratio that size gives - stored_content_size() reads those without decoding, so a damaged file can still state a size - It answers None unless the structures account for the whole file - zstd frames are walked by RFC 8878 rather than through 'ZSTD_getFrameHeader', which needs static linking - xz goes through lzma_file_info_decoder(), so concatenated streams and their padding add up - 'lz4' now records the content size when compressing a file of known length - Every lz4 codec fixture hash is updated to match - Moves gzip_stored_mtime() into frontend/_formats.py - Splits fs_fseek() out of fs_stream_size() for the seeking the walks do - The gzip-header fuzz target now also drives all four size readers --- src/compresso/_core.pyi | 7 + src/compresso/cli/_render/info.py | 56 ++++++- src/compresso/cli/info.py | 19 ++- src/compresso/csrc/_core.c | 27 ++++ src/compresso/csrc/codec/lz4.c | 9 +- src/compresso/csrc/fsutil.c | 27 ++-- src/compresso/csrc/fsutil.h | 3 + src/compresso/csrc/standalone.c | 252 +++++++++++++++++++++++++++++ src/compresso/csrc/standalone.h | 6 + src/compresso/frontend/_formats.py | 14 ++ src/compresso/frontend/api.py | 8 +- tests/c/fuzz/gzip_name.c | 12 +- tests/fixtures/codec_manifest.json | 64 ++++---- tests/python/cli/test_info.py | 52 ++++++ tests/python/test_core.py | 223 +++++++++++++++++++++++++ 15 files changed, 713 insertions(+), 66 deletions(-) diff --git a/src/compresso/_core.pyi b/src/compresso/_core.pyi index 4265faa..a5e65a5 100644 --- a/src/compresso/_core.pyi +++ b/src/compresso/_core.pyi @@ -251,6 +251,13 @@ def gzip_stored_name(file_path: StrOrBytesPath) -> bytes | None: Always None for a non-gzip file. """ +def stored_content_size(file_path: StrOrBytesPath) -> int | None: + """Return the uncompressed size an xz, zstd, lz4 or snappy file states, or None. + + Reads no compressed data, so a damaged file can still state a size; None for + any other format, a zstd or lz4 frame written from a stream, or trailing data. + """ + def format_is_archive(format: str) -> bool: """Return whether a format's container can hold more than one entry.""" diff --git a/src/compresso/cli/_render/info.py b/src/compresso/cli/_render/info.py index 3a85e43..52d2507 100644 --- a/src/compresso/cli/_render/info.py +++ b/src/compresso/cli/_render/info.py @@ -14,6 +14,18 @@ from . import format_size, format_time, succeed +def _format_mtime(mtime: float) -> str: + """Format a modification time for display. + + Args: + mtime: Seconds since the epoch. + + Returns: + The local date and time, to the minute. + """ + return time.strftime("%Y-%m-%d %H:%M", time.localtime(mtime)) + + def list_entries(entries: list[ArchiveEntry], *, long: bool = False) -> None: """Print one line per entry: its size, then its path. @@ -69,7 +81,7 @@ def _entry_detail_line(entry: ArchiveEntry) -> str: type_bit = stat.S_IFREG mode = stat.filemode(type_bit | entry.mode) - mtime = time.strftime("%Y-%m-%d %H:%M", time.localtime(entry.mtime)) + mtime = _format_mtime(entry.mtime) has_no_data = entry.is_dir or entry.is_symlink or entry.is_hardlink size = "" if has_no_data else format_size(size_bytes=entry.size) @@ -152,30 +164,62 @@ def archive_info( return data -def codec_info(file: Path, fmt: str, *, as_json: bool) -> dict[str, object]: +def codec_info( + file: Path, + fmt: str, + *, + stored_name: str | None = None, + mtime: int | None = None, + orig_size: int | None = None, + as_json: bool, +) -> dict[str, object]: """Describe a standalone codec file, e.g. a `.gz`, by its format and size. Args: file: The file's path. fmt: The file's detected format. + stored_name: The name its header stores, if any. + mtime: The modification time its header stores, if any. + orig_size: The uncompressed size its metadata states, if any. as_json: If True, print nothing; the caller prints the returned item. Returns: The file's JSON item. """ size: int = file.stat().st_size - data: dict[str, object] = {"path": str(file), "format": fmt, "size": size} + data: dict[str, object] = { + "path": str(file), + "format": fmt, + "size": size, + "stored_name": stored_name, + "mtime": mtime, + "orig_size": orig_size, + } + + if as_json: + return data lines: list[str] = [ "", f"File: {file}", f"Format: {fmt}", f"Size: {format_size(size_bytes=size)}", - "", ] - if not as_json: - print("\n".join(lines)) + if stored_name is not None: + lines.append(f"Stored name: {stored_name}") + + if mtime is not None: + lines.append(f"Modified: {_format_mtime(mtime)}") + + if orig_size is not None: + lines.append(f"Original size: {format_size(size_bytes=orig_size)}") + + if orig_size: + lines.append(f"Compression: {size / orig_size * 100:.1f}% of original") + + print("\n".join(lines)) + print() return data diff --git a/src/compresso/cli/info.py b/src/compresso/cli/info.py index c075db1..cbb6e0f 100644 --- a/src/compresso/cli/info.py +++ b/src/compresso/cli/info.py @@ -8,7 +8,8 @@ from pathlib import Path from typing import Annotated -from .._core import detect_format +from .._core import detect_format, gzip_stored_name, stored_content_size +from ..frontend._formats import gzip_stored_mtime from ..frontend.archive_api import plan_extraction as plan_archive_extraction from ..frontend.dispatch import is_archive_format from ..introspect.file_inspect import inspect as inspect_file @@ -50,7 +51,17 @@ def _describer(path: Path) -> Callable[..., dict[str, object]]: if fmt == "unknown": raise InputFailed(f"{path}: not a recognised archive or compressed file") - return partial(codec_info, path, fmt) + stored: bytes | None = gzip_stored_name(path) + + return partial( + codec_info, + path, + fmt, + # Only for display, so a name that isn't UTF-8 can stay approximate + stored_name=stored.decode(errors="replace") if stored else None, + mtime=gzip_stored_mtime(path) if fmt == "gzip" else None, + orig_size=stored_content_size(path), + ) def info( @@ -66,7 +77,9 @@ def info( An archive reports its format, entry count, and total size; `list` shows its entries. A Compresso `.comp` reports its algorithm and decompression - info, and a standalone codec file its format and size. + info, and a standalone codec file its format and size, with whatever its + header stores: a gzip's name and mtime, an xz, zstd, lz4 or snappy file's + original size. Args: inputs: The archives or compressed files to describe. diff --git a/src/compresso/csrc/_core.c b/src/compresso/csrc/_core.c index 9903d65..46aa857 100644 --- a/src/compresso/csrc/_core.c +++ b/src/compresso/csrc/_core.c @@ -835,6 +835,30 @@ static PyObject *py_gzip_stored_name(PyObject *Py_UNUSED(self), return result; } +static PyObject *py_stored_content_size(PyObject *Py_UNUSED(self), + PyObject *args) { + PyObject *file_path_bytes = NULL; + + if (!PyArg_ParseTuple(args, "O&", PyUnicode_FSConverter, &file_path_bytes)) { + return NULL; // Error already set + } + + Format format; + FILE *f = open_source(PyBytes_AS_STRING(file_path_bytes), &format); + Py_DECREF(file_path_bytes); + if (!f) { + return NULL; + } + + uint64_t size; + int stated = standalone_content_size(f, format, &size) == 0; + fclose(f); + if (!stated) { + Py_RETURN_NONE; + } + return PyLong_FromUnsignedLongLong(size); +} + static PyObject *py_format_is_archive(PyObject *Py_UNUSED(self), PyObject *args) { const char *format_name = NULL; @@ -1028,6 +1052,9 @@ static PyMethodDef CoreMethods[] = { "Detect the format of a file."}, {"gzip_stored_name", (PyCFunction)py_gzip_stored_name, METH_VARARGS, "Return the name a gzip header stores (FNAME), or None."}, + {"stored_content_size", (PyCFunction)py_stored_content_size, METH_VARARGS, + "Return the uncompressed size an xz, zstd, lz4 or snappy file's own " + "metadata states, or None."}, {"format_stores_hardlinks", (PyCFunction)py_format_stores_hardlinks, METH_VARARGS, "Return whether an archive format stores a file's later names as links."}, diff --git a/src/compresso/csrc/codec/lz4.c b/src/compresso/csrc/codec/lz4.c index 6d5b310..f39dc7d 100644 --- a/src/compresso/csrc/codec/lz4.c +++ b/src/compresso/csrc/codec/lz4.c @@ -6,7 +6,8 @@ #include // At the HC levels the output depends on how the input is split across -// LZ4F_compressUpdate calls, so it is fed a fixed amount whatever CODEC_CHUNK is +// LZ4F_compressUpdate calls, so it is fed a fixed amount whatever CODEC_CHUNK +// is #define LZ4_FEED 65536 // Room for every block one LZ4F_compressUpdate call can complete: with the @@ -45,6 +46,12 @@ static int lz4_begin(void *state, const CodecParams *params, int decompress, s->prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; } + // Lets `info` read the size back; liblz4 omits a zero, and fails at the end + // if the input isn't this long + if (params->exact_size) { + s->prefs.frameInfo.contentSize = params->orig_size; + } + return 0; } diff --git a/src/compresso/csrc/fsutil.c b/src/compresso/csrc/fsutil.c index 2b05e51..66ec124 100644 --- a/src/compresso/csrc/fsutil.c +++ b/src/compresso/csrc/fsutil.c @@ -15,30 +15,21 @@ int64_t fs_ftell(FILE *stream) { #endif } -int64_t fs_stream_size(FILE *stream) { - int64_t original = fs_ftell(stream); - if (original < 0) - return -1; - +int fs_fseek(FILE *stream, int64_t offset, int whence) { #if defined(_WIN32) || defined(_WIN64) - if (_fseeki64(stream, 0, SEEK_END) != 0) - return -1; + return _fseeki64(stream, offset, whence); #else - if (fseeko(stream, 0, SEEK_END) != 0) - return -1; + return fseeko(stream, (off_t)offset, whence); #endif +} - int64_t size = fs_ftell(stream); - -#if defined(_WIN32) || defined(_WIN64) - if (_fseeki64(stream, original, SEEK_SET) != 0) - return -1; -#else - if (fseeko(stream, (off_t)original, SEEK_SET) != 0) +int64_t fs_stream_size(FILE *stream) { + int64_t original = fs_ftell(stream); + if (original < 0 || fs_fseek(stream, 0, SEEK_END) != 0) return -1; -#endif - return size; + int64_t size = fs_ftell(stream); + return fs_fseek(stream, original, SEEK_SET) == 0 ? size : -1; } int fs_is_absolute(const char *path) { diff --git a/src/compresso/csrc/fsutil.h b/src/compresso/csrc/fsutil.h index f12b19e..0d06b1d 100644 --- a/src/compresso/csrc/fsutil.h +++ b/src/compresso/csrc/fsutil.h @@ -203,6 +203,9 @@ int fs_mknod(const char *path, uint32_t type, uint32_t mode, uint64_t rdev); // platform (plain ftell() is 32-bit on Windows); returns -1 on failure int64_t fs_ftell(FILE *stream); +// fseek() with a 64-bit offset on every platform; returns 0 on success +int fs_fseek(FILE *stream, int64_t offset, int whence); + // Size of an open stream in bytes, leaving the offset where it found it; // returns -1 on failure int64_t fs_stream_size(FILE *stream); diff --git a/src/compresso/csrc/standalone.c b/src/compresso/csrc/standalone.c index b4b26fe..8b39738 100644 --- a/src/compresso/csrc/standalone.c +++ b/src/compresso/csrc/standalone.c @@ -2,8 +2,12 @@ #include #include "standalone.h" #include "common.h" +#include +#include +#include #include #include +#include const FormatDesc *find_standalone_format(Format format) { const FormatDesc *desc = format_by_id(format); @@ -81,3 +85,251 @@ char *gzip_original_name(const char *compressed_path) { return c == 0 && i > 0 ? strdup(name_buf) : NULL; } + +static int xz_content_size(FILE *f, uint64_t *size) { + int64_t file_size = fs_stream_size(f); + lzma_stream strm = LZMA_STREAM_INIT; + lzma_index *index = NULL; + if (file_size < 0 || lzma_file_info_decoder(&strm, &index, UINT64_MAX, + (uint64_t)file_size) != LZMA_OK) { + return -1; + } + + uint8_t buf[BUFSIZ]; + lzma_ret ret = LZMA_OK; + while (ret == LZMA_OK) { + if (strm.avail_in == 0) { + strm.next_in = buf; + strm.avail_in = fread(buf, 1, sizeof(buf), f); + } + ret = lzma_code(&strm, LZMA_RUN); + if (ret == LZMA_SEEK_NEEDED) { + strm.avail_in = 0; + ret = fs_fseek(f, (int64_t)strm.seek_pos, SEEK_SET) == 0 ? LZMA_OK + : LZMA_DATA_ERROR; + } + } + + if (ret == LZMA_STREAM_END) { + *size = lzma_index_uncompressed_size(index); + } + lzma_index_end(index, NULL); + lzma_end(&strm); + return ret == LZMA_STREAM_END ? 0 : -1; +} + +// ZSTD_getFrameHeader needs static linking, so frames are walked by RFC 8878 +#define ZSTD_HEADER_MAX 18 // ZSTD_FRAMEHEADERSIZE_MAX, also static-only +#define ZSTD_BLOCK_HEADER 3 +#define ZSTD_SKIPPABLE_HEADER 8 +#define ZSTD_CHECKSUM 4 + +#define ZSTD_FHD_CHECKSUM 0x04 +#define ZSTD_FHD_SINGLE_SEGMENT 0x20 +#define ZSTD_BLOCK_RLE 1 + +// The offset past the blocks at `pos`, or -1 +static int64_t zstd_skip_blocks(FILE *f, int64_t pos, int64_t end) { + for (;;) { + unsigned char header[ZSTD_BLOCK_HEADER]; + if (fs_fseek(f, pos, SEEK_SET) != 0 || + fread(header, sizeof(header), 1, f) != 1) { + return -1; + } + + // Bit 0 ends the frame, 1-2 are the type, and the rest the size; an RLE + // block stores just the byte it repeats + uint32_t block = (uint32_t)get_le(header, ZSTD_BLOCK_HEADER); + pos += ZSTD_BLOCK_HEADER; + pos += ((block >> 1) & 3) == ZSTD_BLOCK_RLE ? 1 : block >> 3; + if (pos > end) { + return -1; + } + if (block & 1) { + return pos; + } + } +} + +// Adds the content size of the frame whose first `got` bytes are `header` to +// `size`, returning the offset past it, or -1 +typedef int64_t (*FrameWalker)(FILE *f, const unsigned char *header, + size_t got, int64_t pos, int64_t end, + uint64_t *size); + +static int64_t zstd_frame(FILE *f, const unsigned char *header, size_t got, + int64_t pos, int64_t end, uint64_t *size) { + static const uint8_t dict_id_size[] = {0, 1, 2, 4}; + static const uint8_t content_size_size[] = {0, 2, 4, 8}; + + // Both failures sit at the top of the range, and reject a bad magic + unsigned long long frame = ZSTD_getFrameContentSize(header, got); + if (frame >= ZSTD_CONTENTSIZE_ERROR || frame > UINT64_MAX - *size) { + return -1; + } + *size += frame; + + // Magic and descriptor, then the window byte unless the frame's single + // segment, whose content size is never absent + uint8_t fhd = header[4]; + int single = (fhd & ZSTD_FHD_SINGLE_SEGMENT) != 0; + int content_field = content_size_size[fhd >> 6]; + int header_size = 5 + !single + dict_id_size[fhd & 3] + + (single && !content_field ? 1 : content_field); + + pos = zstd_skip_blocks(f, pos + header_size, end); + return pos >= 0 && (fhd & ZSTD_FHD_CHECKSUM) ? pos + ZSTD_CHECKSUM : pos; +} + +#define LZ4_FLG_VERSION_MASK 0xC0 +#define LZ4_FLG_VERSION 0x40 +#define LZ4_FLG_BLOCK_CHECKSUM 0x10 +#define LZ4_FLG_CONTENT_SIZE 0x08 +#define LZ4_FLG_CONTENT_CHECKSUM 0x04 +#define LZ4_FLG_DICT_ID 0x01 +#define LZ4_BLOCK_SIZE_MASK 0x7FFFFFFF // The top bit marks a stored block + +static int64_t lz4_frame(FILE *f, const unsigned char *header, size_t got, + int64_t pos, int64_t end, uint64_t *size) { + // The minimum is magic, FLG, BD and the header checksum; the content size + // and dictionary ID sit between BD and the checksum + uint8_t flg = header[4]; + size_t header_size = + LZ4F_HEADER_SIZE_MIN + 8 + (flg & LZ4_FLG_DICT_ID ? 4 : 0); + + // A frame written from a stream, the lz4 CLI's default, states no size + if (get_le(header, 4) != LZ4F_MAGICNUMBER || + (flg & LZ4_FLG_VERSION_MASK) != LZ4_FLG_VERSION || + !(flg & LZ4_FLG_CONTENT_SIZE) || got < header_size) { + return -1; + } + uint64_t frame = get_le(header + 6, 8); + if (frame > UINT64_MAX - *size) { + return -1; + } + *size += frame; + + int64_t checksum = flg & LZ4_FLG_BLOCK_CHECKSUM ? LZ4F_BLOCK_CHECKSUM_SIZE : 0; + pos += (int64_t)header_size; + for (;;) { + unsigned char block[LZ4F_BLOCK_HEADER_SIZE]; + if (fs_fseek(f, pos, SEEK_SET) != 0 || + fread(block, sizeof(block), 1, f) != 1) { + return -1; + } + pos += LZ4F_BLOCK_HEADER_SIZE; + + // liblz4 ends the frame on a zero size, whatever the top bit + int64_t len = (int64_t)(get_le(block, 4) & LZ4_BLOCK_SIZE_MASK); + if (len == 0) { + break; + } + pos += len + checksum; + if (pos > end) { + return -1; + } + } + + return flg & LZ4_FLG_CONTENT_CHECKSUM ? pos + LZ4F_CONTENT_CHECKSUM_SIZE + : pos; +} + +// zstd and lz4 share skippable frames, so only the other frames differ +static int frames_content_size(FILE *f, FrameWalker frame, uint64_t *size) { + int64_t end = fs_stream_size(f); + int64_t pos = 0; + *size = 0; + + while (pos >= 0 && pos < end) { + unsigned char header[LZ4F_HEADER_SIZE_MAX]; // Over ZSTD_HEADER_MAX too + if (fs_fseek(f, pos, SEEK_SET) != 0) { + return -1; + } + size_t got = fread(header, 1, sizeof(header), f); + if (got < ZSTD_SKIPPABLE_HEADER) { + return -1; + } + + if ((get_le(header, 4) & ZSTD_MAGIC_SKIPPABLE_MASK) == + ZSTD_MAGIC_SKIPPABLE_START) { + pos += ZSTD_SKIPPABLE_HEADER + (int64_t)get_le(header + 4, 4); + } else { + pos = frame(f, header, got, pos, end, size); + } + } + + return pos == end && end > 0 ? 0 : -1; +} + +// Snappy's framing format, as codec/snappy.c reads it +#define SNAPPY_COMPRESSED 0x00 +#define SNAPPY_UNCOMPRESSED 0x01 +#define SNAPPY_SKIPPABLE 0x80 // 0x80-0xFE, then the stream identifier at 0xFF +#define SNAPPY_CHUNK 65536 // The most a chunk may decode to +#define SNAPPY_CHECKSUM 4 +#define SNAPPY_VARINT_MAX 3 // Enough to state SNAPPY_CHUNK + +// Every snappy block opens with its decoded length, so no chunk is decoded +static int snappy_content_size(FILE *f, uint64_t *size) { + int64_t end = fs_stream_size(f); + int64_t pos = 0; + *size = 0; + + while (pos < end) { + // Type and LE24 length, then a data chunk's checksum and the start of its + // body + unsigned char chunk[4 + SNAPPY_CHECKSUM + SNAPPY_VARINT_MAX]; + if (fs_fseek(f, pos, SEEK_SET) != 0) { + return -1; + } + size_t got = fread(chunk, 1, sizeof(chunk), f); + if (got < 4) { + return -1; + } + size_t len = (size_t)get_le(chunk + 1, 3); + size_t body = got - 4 < len ? got - 4 : len; + pos += 4 + (int64_t)len; + + size_t decoded = 0; + if (chunk[0] == SNAPPY_COMPRESSED) { + if (body < SNAPPY_CHECKSUM || + snappy_uncompressed_length((const char *)chunk + 4 + SNAPPY_CHECKSUM, + body - SNAPPY_CHECKSUM, + &decoded) != SNAPPY_OK) { + return -1; + } + } else if (chunk[0] == SNAPPY_UNCOMPRESSED) { + if (len < SNAPPY_CHECKSUM) { + return -1; + } + decoded = len - SNAPPY_CHECKSUM; + } else if (chunk[0] < SNAPPY_SKIPPABLE) { + return -1; // Reserved unskippable + } + + if (decoded > SNAPPY_CHUNK) { + return -1; + } + *size += decoded; + } + + return pos == end && end > 0 ? 0 : -1; +} + +int standalone_content_size(FILE *f, Format format, uint64_t *size) { + if (fs_fseek(f, 0, SEEK_SET) != 0) { + return -1; + } + switch (format) { + case FORMAT_XZ: + return xz_content_size(f, size); + case FORMAT_ZSTD: + return frames_content_size(f, zstd_frame, size); + case FORMAT_LZ4: + return frames_content_size(f, lz4_frame, size); + case FORMAT_SNAPPY: + return snappy_content_size(f, size); + default: + return -1; + } +} diff --git a/src/compresso/csrc/standalone.h b/src/compresso/csrc/standalone.h index 0bea8e6..b957b7a 100644 --- a/src/compresso/csrc/standalone.h +++ b/src/compresso/csrc/standalone.h @@ -23,4 +23,10 @@ int standalone_decompress(const FormatDesc *fmt, const char *input_path, // The original filename a gzip header stores, or NULL char *gzip_original_name(const char *compressed_path); +// The uncompressed size an xz index, the zstd or lz4 frame headers, or the +// snappy block headers of `f` state, without decoding; -1 if they don't state +// one, a zstd or lz4 frame written from a stream, or a file those structures +// don't fully describe +int standalone_content_size(FILE *f, Format format, uint64_t *size); + #endif // STANDALONE_H diff --git a/src/compresso/frontend/_formats.py b/src/compresso/frontend/_formats.py index 45645cc..5b56ed0 100644 --- a/src/compresso/frontend/_formats.py +++ b/src/compresso/frontend/_formats.py @@ -65,3 +65,17 @@ def infer_format(output: Path | None) -> str | None: return suffix return None + + +def gzip_stored_mtime(path: Path) -> int | None: + """Return the modification time a gzip header stores. + + Args: + path: A gzip file. + + Returns: + Its MTIME, in seconds since the epoch, or None if none is stored. + """ + with path.open("rb") as f: + # MTIME, bytes 4-7; zero means none was stored + return int.from_bytes(f.read(8)[4:], "little") or None diff --git a/src/compresso/frontend/api.py b/src/compresso/frontend/api.py index ef72bef..4be9c3e 100644 --- a/src/compresso/frontend/api.py +++ b/src/compresso/frontend/api.py @@ -29,6 +29,7 @@ STANDALONE_FORMATS, STANDALONE_SUFFIXES, canonical_standalone_format, + gzip_stored_mtime, ) from ._job import JobResult, ProgressCallback, ThreadedJob, to_core_progress from .archive_api import _OVERWRITE_CODES, OverwriteMode @@ -380,11 +381,8 @@ def plan( if name not in ("", ".."): dest_path = src_path.parent / name - mtime: int | None = None - if restore_name and detected == "gzip": - with src_path.open("rb") as f: - # MTIME, bytes 4-7; zero means none was stored - mtime = int.from_bytes(f.read(8)[4:], "little") or None + restores_mtime: bool = restore_name and detected == "gzip" + mtime: int | None = gzip_stored_mtime(src_path) if restores_mtime else None return replace(plan(detected, True), mtime=mtime) diff --git a/tests/c/fuzz/gzip_name.c b/tests/c/fuzz/gzip_name.c index 339b8e2..502611b 100644 --- a/tests/c/fuzz/gzip_name.c +++ b/tests/c/fuzz/gzip_name.c @@ -1,4 +1,4 @@ -// libFuzzer target for the gzip header's stored name, which is read by path +// libFuzzer target for what `info` reads from a header without decoding #include "standalone.h" #include @@ -31,5 +31,15 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { fwrite(data, 1, size, f); fclose(f); free(gzip_original_name(path)); + + f = fopen(path, "rb"); + if (f) { + uint64_t content_size; + standalone_content_size(f, FORMAT_XZ, &content_size); + standalone_content_size(f, FORMAT_ZSTD, &content_size); + standalone_content_size(f, FORMAT_LZ4, &content_size); + standalone_content_size(f, FORMAT_SNAPPY, &content_size); + fclose(f); + } return 0; } diff --git a/tests/fixtures/codec_manifest.json b/tests/fixtures/codec_manifest.json index 76c8721..e3f043e 100644 --- a/tests/fixtures/codec_manifest.json +++ b/tests/fixtures/codec_manifest.json @@ -26,26 +26,26 @@ "comp/bzip2/9/one_byte": "d9b98e06af4dd4815871946591eb980386839d9efa98ced727efc17b9422032a", "comp/bzip2/9/random": "170882d521c2b4c4ead01c3b9dc52a43721b4df544305127bf329ec2616e049c", "comp/bzip2/9/xargs": "82d5e670f90516791b7685288fa5d1fcdc6829ed27ae86453fa9f7d2a59c27c7", - "comp/lz4/-1/alice": "dadf4529b2493f5af03f2bbffe38a2ae8012f320d549f748690ffbf33ccc5f03", + "comp/lz4/-1/alice": "4752183bd94c55c284404b6b906002fbf176aa29d98a84faa9e6b64be636d6ce", "comp/lz4/-1/empty": "87c4277150bed1c418e64b0009a2922f7dafa818005a110f37dbeafe8df4af4d", - "comp/lz4/-1/one_byte": "7f388b5a3d640aa9e835dc174eb587a2c97a35be9f9f08c9668acb619e7089bb", - "comp/lz4/-1/random": "0756ec5722605a41013e956f7bbd42999fe20b6328ba5a74cb2befe99769fcbc", - "comp/lz4/-1/xargs": "5feaea7f0a22bf57d6587f38822504510dfee7289d29df1cd3f5faf9290749fb", - "comp/lz4/1/alice": "35c40b750ed0535f5ddf0d5175141bceaee03b8dc0851411bb2de9992e9e1e38", + "comp/lz4/-1/one_byte": "10d50f1be4f7c3b56ab4ac1a91e210b267ff9c825ce4a7126afb979135a31f12", + "comp/lz4/-1/random": "de5e823868af8ba53eab8d85fd2ae99783925aef690a387234b2d211dfdb9ffe", + "comp/lz4/-1/xargs": "ac765b6e5d85abda7e6b14aacd03ad5303b8cd0e3cf251d176c8baafc6aec271", + "comp/lz4/1/alice": "c822b3c34f8427501632e0e33c5c08252148de5ae3f10903f70207d95258e49c", "comp/lz4/1/empty": "59294d19c0874a3de85777b90fc4280488cac9fedfd1d4d7060262e933e5ae9b", - "comp/lz4/1/one_byte": "aae68f72724a5fcdb7c2a41835ae258750f011873ff9e6161ad5f3b61d34b503", - "comp/lz4/1/random": "35fa40e904baaf2a54a6782aa411b1b63edb95a0d026ffaa0770ee146775b7b9", - "comp/lz4/1/xargs": "28caf84c9ce033dd178eab509faab6aeac1cb45ee9ca3af554fa2421672b9ab0", - "comp/lz4/6/alice": "9bf7f8d9ee787edd4164131de727dcb0cc67077e4a0ebbc94139a196ac2b09fd", + "comp/lz4/1/one_byte": "7a0fea797622add91d202ae632c03ec9394a6402e7e0190c024d7aa68d1008ce", + "comp/lz4/1/random": "f0963e22cf45c7262d6219942aae482cbb968531de081da94446046f7091c266", + "comp/lz4/1/xargs": "d2198f98e911e4094f414758e146c96af3fb411df5a5657710e46bb5449b4464", + "comp/lz4/6/alice": "9269642f00ec505d38775186659876b1bc44e5ee0dc53c1a5a464ed240e7d575", "comp/lz4/6/empty": "4d33412556dd207e7095d4abd663b8a5fd30197f067edd75c081f3cbd53d00f6", - "comp/lz4/6/one_byte": "2eaff8da9cbe1124e3d8246b96f1267466fed99460121f36bddfad5a2bab7ae6", - "comp/lz4/6/random": "c12ce5e372d69def88e168895e8cc80cbf520123c88a66f10f91e716e3908349", - "comp/lz4/6/xargs": "ade7d1cf51b36670ca238ff76674a774f517e505ab329f7606c8f879f304ba82", - "comp/lz4/9/alice": "dac4b3721e86d24c3e42117295b17270733a2682d35791b1c75da7f1a6e583b1", + "comp/lz4/6/one_byte": "99ab50b6f1011730f61df8a8cd825e9bd624e1b0e6c54026a26a0aa2bfaa2b67", + "comp/lz4/6/random": "8566754262e336d1df411ba9b42a2fd3621449543b8c11509f975eeeefa8b326", + "comp/lz4/6/xargs": "49b715cc01ddd18fe56a8878463dc11879655efc8111e4c0ade479b8ac00d544", + "comp/lz4/9/alice": "e1aa1b0bb9216d14480fecf8d736632bc4b85aeeeda7c0d926a10f38189f36ad", "comp/lz4/9/empty": "e2571af42b428be1440e850df924fac6f5fef61862153692148a6a1ffef77759", - "comp/lz4/9/one_byte": "943242246d2988a4856ae59ab4911046731866b07f7168fd4a343bbe2f026428", - "comp/lz4/9/random": "9015cae80d4521bcf67822e12153984aefe188cbe5d7315cc1c824c5fb3bc954", - "comp/lz4/9/xargs": "3ed465a5f86a4c1040d9d37cc89c60c246225872a277ecda2d7b74a44df99777", + "comp/lz4/9/one_byte": "2ea5fc615396c4eaa1253d7e522c1e2e88a1af52c5f87d0b8b25a24938f8ad6d", + "comp/lz4/9/random": "047b03ed7762ea8600dd4d4483da70b17765a6bd9121e028a68206713980ba58", + "comp/lz4/9/xargs": "22da1815391fc4da766a8b83aabe10eed224cea1fab10bda051d4258b5e98fbd", "comp/lzma/-1/alice": "a56ad264fdfb57a4b1cd7175b12493a7f290ab891f191fe9cd471e062bc613c4", "comp/lzma/-1/empty": "a51c97d296ef49be2dc972aa578f088a10b8dc4fbe9409040f1bb61402a1532a", "comp/lzma/-1/one_byte": "29bd38e9b06cc5798d4c116d8930b7580858c2ec766a39e4fbc558563252740b", @@ -166,26 +166,26 @@ "standalone/gzip/9/one_byte": "676a53aff05d0332d03ae91ae54e0ab34a8a260e2b1840d71fb9d39980648883", "standalone/gzip/9/random": "a36344d3edbf8e9fa14c70239ad107a85049072706d92b89e66180d0025ef3b4", "standalone/gzip/9/xargs": "f6e6121a7577021494e0569d8bef58fc1065727afa714f863957b3b191ae17a3", - "standalone/lz4/-1/alice": "f7c4a5ac4e8cf94f855921922ab4ef3c2b93527674c8ddf3a3bcc6908289fa39", + "standalone/lz4/-1/alice": "36fca6c16ada62893ffa7ccce735e71266acc51e6cb7d10e0beadd33d309a77c", "standalone/lz4/-1/empty": "ff2d4fa25b032b9e88f09b73d8b58bf9b139945bbbe1735d10732636a36844be", - "standalone/lz4/-1/one_byte": "019638231ad533b09ce864640f065779885d89cb8896e7f1d4c828e6d06d8cb1", - "standalone/lz4/-1/random": "29f800ff0713579c3480cea009be421c4bc2ad6f512f8b8ed9f139d764bd5437", - "standalone/lz4/-1/xargs": "5f2484fb3e58f3d617f57908742043360f1eaf6f8b0393cda046695521728dce", - "standalone/lz4/1/alice": "f7c4a5ac4e8cf94f855921922ab4ef3c2b93527674c8ddf3a3bcc6908289fa39", + "standalone/lz4/-1/one_byte": "9c81485a8465e3a8436c50b2f576fabd179682e12fb1129f15b8f025604dfbba", + "standalone/lz4/-1/random": "3688f304f6edf20fa5b94d063b2a3f570ee228fab1b0c68d8c7782f9f548e00a", + "standalone/lz4/-1/xargs": "011b5c117af1061ba6be06e846727a112356df56404b2f41a04a3d24d5ea6e99", + "standalone/lz4/1/alice": "36fca6c16ada62893ffa7ccce735e71266acc51e6cb7d10e0beadd33d309a77c", "standalone/lz4/1/empty": "ff2d4fa25b032b9e88f09b73d8b58bf9b139945bbbe1735d10732636a36844be", - "standalone/lz4/1/one_byte": "019638231ad533b09ce864640f065779885d89cb8896e7f1d4c828e6d06d8cb1", - "standalone/lz4/1/random": "29f800ff0713579c3480cea009be421c4bc2ad6f512f8b8ed9f139d764bd5437", - "standalone/lz4/1/xargs": "5f2484fb3e58f3d617f57908742043360f1eaf6f8b0393cda046695521728dce", - "standalone/lz4/6/alice": "2c551f7b5e65e8214644da8c5bf0b7a4601b137906f64d4661cac95d43fbf801", + "standalone/lz4/1/one_byte": "9c81485a8465e3a8436c50b2f576fabd179682e12fb1129f15b8f025604dfbba", + "standalone/lz4/1/random": "3688f304f6edf20fa5b94d063b2a3f570ee228fab1b0c68d8c7782f9f548e00a", + "standalone/lz4/1/xargs": "011b5c117af1061ba6be06e846727a112356df56404b2f41a04a3d24d5ea6e99", + "standalone/lz4/6/alice": "015e6d6cb674c907220d90bf4f9bc10968ea85c85dc0ce5d653dd00676120898", "standalone/lz4/6/empty": "ff2d4fa25b032b9e88f09b73d8b58bf9b139945bbbe1735d10732636a36844be", - "standalone/lz4/6/one_byte": "019638231ad533b09ce864640f065779885d89cb8896e7f1d4c828e6d06d8cb1", - "standalone/lz4/6/random": "29f800ff0713579c3480cea009be421c4bc2ad6f512f8b8ed9f139d764bd5437", - "standalone/lz4/6/xargs": "46475505866608d6425f137ddb07033864e3d96c915925a100d3cacada5659d1", - "standalone/lz4/9/alice": "32db841e8169ce678805c7a52af8fb3bbb0ee480aa0097e2759c94c5bf44355c", + "standalone/lz4/6/one_byte": "9c81485a8465e3a8436c50b2f576fabd179682e12fb1129f15b8f025604dfbba", + "standalone/lz4/6/random": "3688f304f6edf20fa5b94d063b2a3f570ee228fab1b0c68d8c7782f9f548e00a", + "standalone/lz4/6/xargs": "f9b062b20d56e42aeb8626b50453fb0c7bfac0e53ef6434dc16ee3ee99472b32", + "standalone/lz4/9/alice": "b7a1c0cd137bf2ca55f19758b2f1460264cdef1f3588fc959942e33a7520cc7a", "standalone/lz4/9/empty": "ff2d4fa25b032b9e88f09b73d8b58bf9b139945bbbe1735d10732636a36844be", - "standalone/lz4/9/one_byte": "019638231ad533b09ce864640f065779885d89cb8896e7f1d4c828e6d06d8cb1", - "standalone/lz4/9/random": "29f800ff0713579c3480cea009be421c4bc2ad6f512f8b8ed9f139d764bd5437", - "standalone/lz4/9/xargs": "46475505866608d6425f137ddb07033864e3d96c915925a100d3cacada5659d1", + "standalone/lz4/9/one_byte": "9c81485a8465e3a8436c50b2f576fabd179682e12fb1129f15b8f025604dfbba", + "standalone/lz4/9/random": "3688f304f6edf20fa5b94d063b2a3f570ee228fab1b0c68d8c7782f9f548e00a", + "standalone/lz4/9/xargs": "f9b062b20d56e42aeb8626b50453fb0c7bfac0e53ef6434dc16ee3ee99472b32", "standalone/snappy/-1/alice": "9166e19e9ae120ab62af436e99ad4a59eeb25f1bc57cd1746c64fe24afb6fb48", "standalone/snappy/-1/empty": "75605fd35293ed52e316629e8d3971affee9fee08518e6657188c9e51450a66f", "standalone/snappy/-1/one_byte": "c4b4d76f5c545fa9c0ff12a3315caa3122a7e3222fd05d28a34b3c6969d281a2", diff --git a/tests/python/cli/test_info.py b/tests/python/cli/test_info.py index fb96449..6951410 100644 --- a/tests/python/cli/test_info.py +++ b/tests/python/cli/test_info.py @@ -2,10 +2,13 @@ from __future__ import annotations +import gzip +import time from collections.abc import Callable from pathlib import Path import orjson +import pytest from compresso.cli._render import EXIT_FAILED @@ -43,6 +46,55 @@ def test_standalone_codec_shows_format_and_size( assert "Format: gzip" in output assert "Size:" in output + def test_gzip_shows_its_stored_name_and_mtime(self, tmp_path: Path) -> None: + """Test that a gzip's header detail is shown, and is in the JSON.""" + packed = tmp_path / "p.gz" + with ( + packed.open("wb") as raw, + gzip.GzipFile("stored.txt", "wb", fileobj=raw, mtime=86400 * 365) as f, + ): + f.write(b"data") + + output = invoke("info", packed).output + item = orjson.loads(invoke("info", packed, "--json").output)[0] + + assert "Stored name: stored.txt" in output + assert ( + f"Modified: {time.strftime('%Y-%m-%d', time.localtime(item['mtime']))}" + in output + ) + assert "Original size" not in output + assert item["stored_name"] == "stored.txt" + assert item["mtime"] == 86400 * 365 + assert item["orig_size"] is None + + def test_gzip_name_need_not_be_utf8(self, tmp_path: Path) -> None: + """Test that a name of raw bytes can still be printed and put in JSON.""" + packed = tmp_path / "p.gz" + flagged = bytearray(gzip.compress(b"data", mtime=0)) + flagged[3] |= 0x08 # FNAME + packed.write_bytes(flagged[:10] + b"caf\xe9.txt\0" + flagged[10:]) + + items = orjson.loads(invoke("info", packed, "--json").output) + + assert items[0]["stored_name"] == "caf\ufffd.txt" + + @pytest.mark.parametrize("suffix", [".xz", ".zst", ".lz4", ".sz"]) + def test_stated_original_size_is_shown( + self, payload: Path, tmp_path: Path, suffix: str + ) -> None: + """Test that the size comes from the file, for the ratio to use too.""" + packed = tmp_path / f"p{suffix}" + invoke("compress", payload, "-o", packed, "-q") + + output = invoke("info", packed).output + item = orjson.loads(invoke("info", packed, "--json").output)[0] + + assert "Original size:" in output + assert "Compression:" in output + assert item["orig_size"] == payload.stat().st_size + assert item["stored_name"] is None + def test_several_inputs_carry_on_past_an_unrecognised_one( self, payload: Path, tmp_path: Path, make_archive: Callable[[str], Path] ) -> None: diff --git a/tests/python/test_core.py b/tests/python/test_core.py index c08d1e5..f251a29 100644 --- a/tests/python/test_core.py +++ b/tests/python/test_core.py @@ -1,5 +1,6 @@ """Tests for the core compression/decompression functionality.""" +import gzip import importlib.util import lzma import os @@ -708,3 +709,225 @@ def test_negative_count_is_refused( str(sample_text_file), str(tmp_path / "out.zst"), "zstd", threads=-1 ) assert not (tmp_path / "out.zst").exists() + + +def _zstd_frame(*blocks: bytes, size: int | None = None) -> bytes: + """Build a zstd frame from its encoded blocks, stating `size` if given.""" + magic = b"\x28\xb5\x2f\xfd" + if size is None: + return magic + b"\x00\x50" + b"".join(blocks) # Descriptor, then window + + return magic + b"\xe0" + size.to_bytes(8, "little") + b"".join(blocks) + + +def _zstd_block(payload: bytes, *, last: bool = True) -> bytes: + """Encode a raw zstd block.""" + return ((len(payload) << 3) | last).to_bytes(3, "little") + payload + + +def _lz4_frame( + *blocks: bytes, size: int | None = 3, flg: int = 0, extra: bytes = b"" +) -> bytes: + """Build an lz4 frame of stored blocks, stating `size` if given. + + `extra` follows the content size, for a dictionary ID; every checksum `flg` + asks for is left as zero, as nothing here checks one. + """ + flg |= 0x60 | (0x08 if size is not None else 0) # Version 01, independent + header = bytes([flg, 0x40]) + if size is not None: + header += size.to_bytes(8, "little") + block_checksum = b"\0" * 4 if flg & 0x10 else b"" + stored = b"".join( + (len(block) | 0x8000_0000).to_bytes(4, "little") + block + block_checksum + for block in blocks + ) + content_checksum = b"\0" * 4 if flg & 0x04 else b"" + return ( + b"\x04\x22\x4d\x18" + + header + + extra + + b"\x00" + + stored + + b"\0" * 4 + + content_checksum + ) + + +def _snappy_chunk(kind: int, body: bytes) -> bytes: + """Frame a snappy chunk, its checksum left as zero.""" + return bytes([kind]) + (4 + len(body)).to_bytes(3, "little") + b"\0" * 4 + body + + +SNAPPY_ID = b"\xff\x06\x00\x00sNaPpY" +# A block of one literal, which opens with its decoded length +SNAPPY_ABC = _snappy_chunk(0x00, b"\x03\x08abc") +SNAPPY_DE = _snappy_chunk(0x01, b"de") + + +class TestStoredContentSize: + """Test the size read from a file's metadata, which nothing decodes.""" + + FRAME = _zstd_frame(_zstd_block(b"abc"), size=3) + SKIPPABLE = b"\x50\x2a\x4d\x18" + (2).to_bytes(4, "little") + b"xx" + LZ4 = _lz4_frame(b"abc") + + @pytest.mark.parametrize( + ("data", "size"), + [ + (LZ4, 3), + (LZ4 + LZ4, 6), + (SKIPPABLE + LZ4 + SKIPPABLE, 3), + (_lz4_frame(b"ab", b"c"), 3), + # Block and content checksums, and a dictionary ID + (_lz4_frame(b"ab", b"c", flg=0x15, extra=b"\x01\0\0\0"), 3), + ], + ) + def test_lz4_frames_are_walked( + self, tmp_path: Path, data: bytes, size: int + ) -> None: + """Test the sum over every frame, whatever each one's layout.""" + path = tmp_path / "f.lz4" + path.write_bytes(data) + + assert _core.stored_content_size(path) == size + + @pytest.mark.parametrize( + "data", + [ + _lz4_frame(b"abc", size=None), # Written from a stream + LZ4 + _lz4_frame(b"abc", size=None), + LZ4[:-1], + LZ4 + b"junk", + LZ4 + LZ4[:10], + ], + ) + def test_lz4_states_no_size_unless_fully_described( + self, tmp_path: Path, data: bytes + ) -> None: + """Test that a partial sum is never reported as the size.""" + path = tmp_path / "f.lz4" + path.write_bytes(data) + + assert _core.stored_content_size(path) is None + + @pytest.mark.parametrize( + ("data", "size"), + [ + (SNAPPY_ID + SNAPPY_ABC, 3), + (SNAPPY_ID + SNAPPY_ABC + SNAPPY_DE, 5), + (SNAPPY_ID + b"\xfe\x02\x00\x00xx" + SNAPPY_DE, 2), # Padding + (SNAPPY_ID + SNAPPY_ABC + SNAPPY_ID + SNAPPY_DE, 5), # Concatenated + (SNAPPY_ID, 0), + ], + ) + def test_snappy_chunks_are_walked( + self, tmp_path: Path, data: bytes, size: int + ) -> None: + """Test the sum over every data chunk, skipping the rest.""" + path = tmp_path / "f.sz" + path.write_bytes(data) + + assert _core.stored_content_size(path) == size + + @pytest.mark.parametrize( + "data", + [ + SNAPPY_ID + SNAPPY_ABC[:-1], + SNAPPY_ID + SNAPPY_ABC + b"j", + SNAPPY_ID + _snappy_chunk(0x02, b""), # Reserved unskippable + SNAPPY_ID + _snappy_chunk(0x00, b"\x81\x80\x04"), # States 65537 + SNAPPY_ID + _snappy_chunk(0x01, b"x" * 65537), + ], + ) + def test_snappy_states_no_size_unless_fully_described( + self, tmp_path: Path, data: bytes + ) -> None: + """Test that a partial sum, or one no decoder accepts, is never stated.""" + path = tmp_path / "f.sz" + path.write_bytes(data) + + assert _core.stored_content_size(path) is None + + @pytest.mark.parametrize( + ("data", "size"), + [ + (FRAME, 3), + (FRAME + FRAME, 6), + (SKIPPABLE + FRAME + SKIPPABLE, 3), + (_zstd_frame(_zstd_block(b"ab", last=False), _zstd_block(b"c"), size=3), 3), + (_zstd_frame(b"\x0b\x00\x00a", size=1), 1), # RLE, stores one byte + # Single segment with a one-byte size field + (b"\x28\xb5\x2f\xfd\x20\x03" + _zstd_block(b"abc"), 3), + # Content checksum follows the blocks + ( + b"\x28\xb5\x2f\xfd\x24\x03" + _zstd_block(b"abc") + b"\xaa" * 4, + 3, + ), + # Window byte, then a one-byte dictionary ID, then an eight-byte size + ( + b"\x28\xb5\x2f\xfd\xc1\x50\x07" + + (3).to_bytes(8, "little") + + _zstd_block(b"abc"), + 3, + ), + ], + ) + def test_zstd_frames_are_walked( + self, tmp_path: Path, data: bytes, size: int | None + ) -> None: + """Test the sum over every frame, whatever each one's layout.""" + path = tmp_path / "f.zst" + path.write_bytes(data) + + assert _core.stored_content_size(path) == size + + @pytest.mark.parametrize( + "data", + [ + _zstd_frame(_zstd_block(b"abc")), # Written from a stream + FRAME + _zstd_frame(_zstd_block(b"abc")), + FRAME[:-1], + FRAME + b"junk", + FRAME + FRAME[:6], + FRAME + SKIPPABLE[:-1], + ], + ) + def test_zstd_states_no_size_unless_fully_described( + self, tmp_path: Path, data: bytes + ) -> None: + """Test that a partial sum is never reported as the size.""" + path = tmp_path / "f.zst" + path.write_bytes(data) + + assert _core.stored_content_size(path) is None + + def test_xz_sums_every_stream(self, tmp_path: Path) -> None: + """Test concatenated streams and the padding allowed between them.""" + path = tmp_path / "f.xz" + path.write_bytes( + lzma.compress(b"a" * 100) + b"\0" * 4 + lzma.compress(b"b" * 200) + ) + + assert _core.stored_content_size(path) == 300 + + @pytest.mark.parametrize( + "damage", + [lambda d: d[:-1], lambda d: d + b"junk"], + ids=["truncated", "trailing"], + ) + def test_xz_states_no_size_unless_fully_described( + self, tmp_path: Path, damage: Callable[[bytes], bytes] + ) -> None: + """Test that a cut or extended file has no footer where it should be.""" + path = tmp_path / "f.xz" + path.write_bytes(damage(lzma.compress(b"a" * 100))) + + assert _core.stored_content_size(path) is None + + def test_other_formats_state_nothing(self, tmp_path: Path) -> None: + """Test that a format with no such metadata gets no answer.""" + path = tmp_path / "f.gz" + path.write_bytes(gzip.compress(b"data")) + + assert _core.stored_content_size(path) is None From bf3b30bd45d08a24c63155aa7c2fb951bd99d947 Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Thu, 5 Nov 2026 13:31:06 +0000 Subject: [PATCH 27/31] feat(cli): validate -a, -s and -F against what the build supports - 'Strategy' becomes a 'StrEnum' in the frontend, exported and used for 'CompressionOptions.strategy' - '-s' and '-a' are now enum options, so a typo is refused while parsing with the valid names listed, and '--help' shows them - 'Algo' is now built from the core's capabilities, so only compiled backends are included - '-F comp' now names the Compresso container, as '.comp' on the output already did, and writes it whatever the file is called - An unknown '-F' now lists available formats, instead of refusing without saying what is valid - archive_formats() moves into frontend/_formats.py, where the 'formats' command and the plan both share it - plan_compression() now runs the core's check even with no level given, so an unknown algorithm is caught by the plan rather than at run time --- src/compresso/__init__.py | 2 + src/compresso/cli/_render/benchmark.py | 5 ++- src/compresso/cli/benchmark.py | 22 ++++++++-- src/compresso/cli/compress.py | 27 +++++++------ src/compresso/cli/formats.py | 17 ++------ src/compresso/frontend/__init__.py | 2 + src/compresso/frontend/_formats.py | 26 ++++++++++-- src/compresso/frontend/api.py | 56 +++++++++++++++++--------- src/compresso/frontend/dispatch.py | 31 +++++++++++++- tests/python/cli/test_compress.py | 22 +++++++++- tests/python/frontend/test_api.py | 12 +++++- tests/python/frontend/test_dispatch.py | 34 +++++++++++++++- 12 files changed, 192 insertions(+), 64 deletions(-) diff --git a/src/compresso/__init__.py b/src/compresso/__init__.py index 6238d1a..1752459 100644 --- a/src/compresso/__init__.py +++ b/src/compresso/__init__.py @@ -21,6 +21,7 @@ CompressionPlan, DecompressionJob, DecompressionPlan, + Strategy, VerificationJob, VerificationPlan, plan_compression, @@ -72,6 +73,7 @@ "OverwriteMode", "ProgressCallback", "SourceRemovalError", + "Strategy", "SymlinkMode", "TrailingDataWarning", "VerificationJob", diff --git a/src/compresso/cli/_render/benchmark.py b/src/compresso/cli/_render/benchmark.py index 765a6d1..a96864e 100644 --- a/src/compresso/cli/_render/benchmark.py +++ b/src/compresso/cli/_render/benchmark.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Sequence from pathlib import Path from ...introspect.benchmark import BenchmarkResult @@ -14,8 +15,8 @@ def _level_name(level: int | None) -> str | int: def benchmark_header( file: Path, repeats: int, - algos: list[str] | None, - strategies: list[str] | None, + algos: Sequence[str] | None, + strategies: Sequence[str] | None, levels: list[int | None], ) -> None: """Print what the run covers, then a blank line. diff --git a/src/compresso/cli/benchmark.py b/src/compresso/cli/benchmark.py index 19d1f52..59a9270 100644 --- a/src/compresso/cli/benchmark.py +++ b/src/compresso/cli/benchmark.py @@ -2,14 +2,20 @@ from __future__ import annotations +from enum import StrEnum from pathlib import Path from typing import Annotated +from ..frontend.api import Strategy from ..introspect.benchmark import benchmark_file +from ..introspect.capabilities import list_capabilities from ._app import app from ._render import EXIT_USAGE, cancelled, fail, succeed from ._render.benchmark import benchmark_header, benchmark_results +# Built from the core so `--help` lists only what this build compiled in +Algo = StrEnum("Algo", [c.name for c in list_capabilities()]) + def _parse_level(value: str) -> int | None: """Parse one `-l` value, exiting 2 if it's neither a number nor `auto`. @@ -33,13 +39,21 @@ def _parse_level(value: str) -> int | None: def benchmark( file: Annotated[Path, app.Argument(help="File to benchmark")], algo: Annotated[ - list[str] | None, - app.Option("--algo", "-a", help="Algorithm to run; repeatable (default: all)"), + list[Algo] | None, + app.Option( + "--algo", + "-a", + case_sensitive=False, + help="Algorithm to run; repeatable (default: all)", + ), ] = None, strategy: Annotated[ - list[str] | None, + list[Strategy] | None, app.Option( - "--strategy", "-s", help="Strategy to run; repeatable (default: all)" + "--strategy", + "-s", + case_sensitive=False, + help="Strategy to run; repeatable (default: all)", ), ] = None, level: Annotated[ diff --git a/src/compresso/cli/compress.py b/src/compresso/cli/compress.py index ac68415..a247594 100644 --- a/src/compresso/cli/compress.py +++ b/src/compresso/cli/compress.py @@ -3,13 +3,15 @@ from __future__ import annotations import sys +from enum import StrEnum from pathlib import Path from typing import Annotated from .._core import BackendError, Error, HeaderError, SourceRemovalError -from ..frontend.api import CompressionJob +from ..frontend.api import CompressionJob, Strategy from ..frontend.archive_api import ArchiveJob, OverwriteMode from ..frontend.dispatch import CompressRequest, plan_compress +from ..introspect.capabilities import list_capabilities from ._app import app from ._render import EXIT_USAGE, cancelled, fail from ._render.jobs import ( @@ -22,6 +24,9 @@ ) from ._run import NotRemoved, conflict_mode, remove_source, run_all, run_job +# Built from the core so `--help` lists only what this build compiled in +Algo = StrEnum("Algo", ["auto"] + [c.name for c in list_capabilities()]) + def _compress_one_file( job: CompressionJob, quiet: bool, verbose: bool, remove: bool @@ -134,11 +139,11 @@ def compress( "--format", "-F", case_sensitive=False, - help="Container to write, e.g. gz, zst, tar.zst, zip", + help="Container to write, e.g. comp, gz, zst, tar.zst, zip", ), ] = None, algo: Annotated[ - str | None, + Algo, app.Option( "--algo", "-a", @@ -146,14 +151,14 @@ def compress( help="Compression algorithm to use", show_default=False, ), - ] = "auto", + ] = Algo.auto, strategy: Annotated[ - str | None, + Strategy | None, app.Option( "--strategy", "-s", case_sensitive=False, - help="Compression strategy to use (fast/balanced/max_ratio)", + help="Compression strategy to use", show_default="balanced", ), ] = None, @@ -276,20 +281,18 @@ def compress( if not inputs: fail("Error: No input paths given", EXIT_USAGE) - algo_lower: str | None = algo.lower() if algo else None - if algo_lower == "auto": - algo_lower = None + algo_name: str | None = None if algo == "auto" else algo # -a names the algorithm that -s would otherwise pick - if algo_lower and strategy: + if algo_name and strategy: fail("Error: -a contradicts -s", EXIT_USAGE) request = CompressRequest( inputs=inputs, output=output, format=format, - algo=algo_lower, - strategy=strategy.lower() if strategy else None, + algo=algo_name, + strategy=strategy, level=level, overwrite=overwrite_mode, threads=threads, diff --git a/src/compresso/cli/formats.py b/src/compresso/cli/formats.py index 0d9dd17..f065349 100644 --- a/src/compresso/cli/formats.py +++ b/src/compresso/cli/formats.py @@ -2,7 +2,8 @@ from __future__ import annotations -from .._core import archive_capabilities, format_is_archive, standalone_formats +from .._core import standalone_formats +from ..frontend._formats import archive_formats from ..introspect.capabilities import list_capabilities from ._app import app from ._render import cancelled, fail @@ -17,19 +18,7 @@ def formats() -> None: if not caps: fail("No compression backends available") - standalone = standalone_formats() - # Asked of the core, which names a variant by any of the codec's - # names; listed by the extension an output is given - archives = { - archive["name"]: [ - f"{archive['name']}.{fmt['exts'][0]}" - for fmt in standalone - if format_is_archive(f"{archive['name']}.{fmt['exts'][0]}") - ] - for archive in archive_capabilities() - } - - list_formats(caps, standalone, archives) + list_formats(caps, standalone_formats(), archive_formats()) except KeyboardInterrupt: cancelled("Format listing") diff --git a/src/compresso/frontend/__init__.py b/src/compresso/frontend/__init__.py index dc7c47a..ecaf28e 100644 --- a/src/compresso/frontend/__init__.py +++ b/src/compresso/frontend/__init__.py @@ -8,6 +8,7 @@ CompressionPlan, DecompressionJob, DecompressionPlan, + Strategy, VerificationJob, VerificationPlan, plan_compression, @@ -47,6 +48,7 @@ "JobResult", "OverwriteMode", "ProgressCallback", + "Strategy", "SymlinkMode", "VerificationJob", "VerificationPlan", diff --git a/src/compresso/frontend/_formats.py b/src/compresso/frontend/_formats.py index 5b56ed0..6b6311c 100644 --- a/src/compresso/frontend/_formats.py +++ b/src/compresso/frontend/_formats.py @@ -4,7 +4,7 @@ from pathlib import Path -from .._core import format_is_archive, standalone_formats +from .._core import archive_capabilities, format_is_archive, standalone_formats # What an archive is written as when neither the caller nor its name says DEFAULT_ARCHIVE_FORMAT = "zip" @@ -24,6 +24,24 @@ } +def archive_formats() -> dict[str, list[str]]: + """Return each available archive format with its compressed variants. + + Returns: + Each archive format (e.g. "tar") mapped to the variants the core + writes, named by the extension an output is given (e.g. "tar.gz"). + """ + # Asked of the core, which names a variant by any of the codec's names + return { + archive["name"]: [ + f"{archive['name']}.{fmt['exts'][0]}" + for fmt in _STANDALONE + if format_is_archive(f"{archive['name']}.{fmt['exts'][0]}") + ] + for archive in archive_capabilities() + } + + def canonical_standalone_format(name: str | None) -> str | None: """Return the canonical name for a single-file format, or None. @@ -47,8 +65,8 @@ def infer_format(output: Path | None) -> str | None: output: The path the user asked to write, or None. Returns: - The format named by the suffix, or None if it names none; includes - `.comp`, whose container is the default anyway. + The format named by the suffix, "comp" for the Compresso container, + or None if it names none. """ if output is None: return None @@ -61,7 +79,7 @@ def infer_format(output: Path | None) -> str | None: ): return f"tar.{suffix}" - if suffix in STANDALONE_FORMATS or format_is_archive(suffix): + if suffix == "comp" or suffix in STANDALONE_FORMATS or format_is_archive(suffix): return suffix return None diff --git a/src/compresso/frontend/api.py b/src/compresso/frontend/api.py index 4be9c3e..0240c7e 100644 --- a/src/compresso/frontend/api.py +++ b/src/compresso/frontend/api.py @@ -4,6 +4,7 @@ import os from dataclasses import dataclass, replace +from enum import StrEnum from pathlib import Path from typing import Self @@ -37,13 +38,27 @@ MB = 1024 * 1024 +class Strategy(StrEnum): + """Which backend the Compresso container picks when no algorithm is given. + + Attributes: + FAST: Favour speed. + BALANCED: Trade speed against ratio. + MAX_RATIO: Favour the smallest output. + """ + + FAST = "fast" + BALANCED = "balanced" + MAX_RATIO = "max_ratio" + + @dataclass(frozen=True) class CompressionOptions: """User-facing compression options. Attributes: algo: Compression algorithm name, or None for auto. - strategy: Compression strategy - "fast", "balanced", or "max_ratio". + strategy: Compression strategy; ignored when `algo` is given. level: Compression level, or None for the backend's default; the range depends on the backend (see `BackendCapabilities`). format: Single-file container to write, e.g. "gz". None writes the @@ -56,7 +71,7 @@ class CompressionOptions: """ algo: str | None = None - strategy: str = "balanced" + strategy: Strategy = Strategy.BALANCED level: int | None = None format: str | None = None overwrite: OverwriteMode = OverwriteMode.RENAME @@ -252,25 +267,26 @@ def plan_compression( reason_if_unavailable="No suitable backend found for the selected strategy", ) - if options.level is not None: - try: - # The core's own check, so each backend's range lives in one place - if standalone: - check_level(options.level, format=standalone) - else: - check_level(options.level, algo=backend_name) + try: + # The core's own check, so each backend's range lives in one place; + # with no level it still refuses an unknown algorithm + level = to_core_level(options.level) + if standalone: + check_level(level, format=standalone) + else: + check_level(level, algo=backend_name) - except ValueError as e: - return CompressionPlan( - src=src_path, - dest=dest_path, - options=options, - input_size=input_size, - backend_name=backend_name, - estimated_seconds=None, - can_compress=False, - reason_if_unavailable=str(e), - ) + except ValueError as e: + return CompressionPlan( + src=src_path, + dest=dest_path, + options=options, + input_size=input_size, + backend_name=backend_name, + estimated_seconds=None, + can_compress=False, + reason_if_unavailable=str(e), + ) mb_s: int | float = get_estimated_speeds(algo=backend_name, operation="compress") estimated_seconds: int | float = (input_size / MB) / mb_s if input_size > 0 else 0.0 diff --git a/src/compresso/frontend/dispatch.py b/src/compresso/frontend/dispatch.py index af9eaed..f7d8594 100644 --- a/src/compresso/frontend/dispatch.py +++ b/src/compresso/frontend/dispatch.py @@ -10,10 +10,12 @@ from .._core import detect_format, format_is_archive from ._formats import ( DEFAULT_ARCHIVE_FORMAT, + STANDALONE_SUFFIXES, + archive_formats, canonical_standalone_format, infer_format, ) -from .api import CompressionJob, CompressionOptions, DecompressionJob +from .api import CompressionJob, CompressionOptions, DecompressionJob, Strategy from .archive_api import ( ArchiveJob, ArchiveOptions, @@ -105,7 +107,7 @@ class CompressRequest: output: Path | None = None format: str | None = None algo: str | None = None - strategy: str | None = None + strategy: Strategy | None = None level: int | None = None overwrite: OverwriteMode = OverwriteMode.RENAME threads: int = 0 @@ -162,6 +164,16 @@ def plan_compress(request: CompressRequest) -> CompressionJob | ArchiveJob: # Only the .comp container records its codec; the others fix their own comp_flag: str | None = "-a" if request.algo else "-s" if request.strategy else None + # The Compresso container is what no format writes, so naming it is the same + if fmt and fmt.lower() == "comp": + if many: + raise ValueError( + "comp holds a single file; " + f"use an archive format such as {DEFAULT_ARCHIVE_FORMAT} for several" + ) + + fmt = None + if format_is_single_file(fmt) and many: raise ValueError( f"{fmt} holds a single file; " @@ -216,6 +228,21 @@ def plan_compress(request: CompressRequest) -> CompressionJob | ArchiveJob: reason_if_unavailable=f"{comp_flag} applies only to .comp, not {fmt}", ) + # Archive formats returned above, so this is one the core doesn't know; + # the plan's own message lists only the single-file formats + elif fmt and not format_is_single_file(fmt): + known = ["comp", *(s.lstrip(".") for s in STANDALONE_SUFFIXES.values())] + for archive, variants in archive_formats().items(): + known += [archive, *variants] + + job.plan = replace( + job.plan, + can_compress=False, + reason_if_unavailable=( + f"Unknown format: {fmt} (expected one of {', '.join(known)})" + ), + ) + return job diff --git a/tests/python/cli/test_compress.py b/tests/python/cli/test_compress.py index 61c021a..e157875 100644 --- a/tests/python/cli/test_compress.py +++ b/tests/python/cli/test_compress.py @@ -32,8 +32,13 @@ def test_compress_then_decompress(self, payload: Path, tmp_path: Path) -> None: # The backends are covered in core and C tests; this checks the flags @pytest.mark.parametrize( "args", - [("-a", "bzip2"), ("-a", "zstd", "-l", "19"), ("-s", "fast")], - ids=["algo", "level", "strategy"], + [ + ("-a", "bzip2"), + ("-a", "zstd", "-l", "19"), + ("-s", "fast"), + ("-s", "MAX_RATIO"), + ], + ids=["algo", "level", "strategy", "strategy-any-case"], ) def test_algorithms_round_trip( self, payload: Path, tmp_path: Path, args: tuple[str, ...] @@ -59,6 +64,19 @@ def test_algo_with_strategy(self, payload: Path) -> None: assert "-a contradicts -s" in result.output + @pytest.mark.parametrize( + ("args", "choices"), + [(("-s", "fsat"), ("fast", "max_ratio")), (("-a", "zsdt"), ("auto", "zstd"))], + ids=["strategy", "algo"], + ) + def test_unknown_choice( + self, payload: Path, args: tuple[str, ...], choices: tuple[str, ...] + ) -> None: + """Test that a strategy or algorithm that isn't one is refused while parsing.""" + result = invoke("compress", payload, *args, expect=EXIT_USAGE) + + assert all(choice in result.output for choice in choices) + @pytest.mark.parametrize( ("args", "message"), [ diff --git a/tests/python/frontend/test_api.py b/tests/python/frontend/test_api.py index 4d1f876..1397b48 100644 --- a/tests/python/frontend/test_api.py +++ b/tests/python/frontend/test_api.py @@ -11,6 +11,7 @@ CompressionJob, CompressionOptions, DecompressionJob, + Strategy, plan_compression, plan_decompression, ) @@ -160,8 +161,15 @@ class TestPlanCompressionLevels: "opts,match", [ # No algo: the backend the strategy picks is the one checked - (CompressionOptions(strategy="fast", level=13), "lz4 compression level"), - (CompressionOptions(strategy="fsat"), "Unknown strategy: fsat"), + ( + CompressionOptions(strategy=Strategy.FAST, level=13), + "lz4 compression level", + ), + ( + CompressionOptions(strategy="fsat"), # ty: ignore[invalid-argument-type] - the point of the test + "Unknown strategy: fsat", + ), + (CompressionOptions(algo="zsdt"), "Unknown compression algorithm: zsdt"), ], ) def test_out_of_range_level_is_unrunnable( diff --git a/tests/python/frontend/test_dispatch.py b/tests/python/frontend/test_dispatch.py index 3dca039..9f4ff50 100644 --- a/tests/python/frontend/test_dispatch.py +++ b/tests/python/frontend/test_dispatch.py @@ -45,7 +45,7 @@ class TestInferFormat: ("x.7z", "7z"), ("x.gz", "gz"), ("x.zst", "zst"), - ("x.comp", None), + ("x.COMP", "comp"), ("x.txt", None), ("x", None), ("x.foo.lz4", "lz4"), @@ -222,12 +222,42 @@ def test_single_file_format_refuses_several_inputs(self, tree: Path) -> None: with pytest.raises(ValueError, match="tar.gz"): plan_compress(CompressRequest([tree], format="gz")) + def test_comp_format_is_the_compresso_container( + self, small_file: Path, tmp_path: Path + ) -> None: + """Test that naming comp writes the default container, whatever the output.""" + output = tmp_path / "misleading.gz" + + job = plan_compress(CompressRequest([small_file], output=output, format="COMP")) + + assert isinstance(job, CompressionJob) + assert job.plan.can_compress + assert job.plan.options.format is None + assert job.plan.dest == output + + @pytest.mark.parametrize( + ("fmt", "output"), [("comp", None), (None, Path("out.comp"))], ids=["-F", "-o"] + ) + def test_comp_format_refuses_several_inputs( + self, tree: Path, fmt: str | None, output: Path | None + ) -> None: + """Test that comp on a directory errors, suggesting zip, not tar.comp.""" + with pytest.raises(ValueError, match="such as zip"): + plan_compress(CompressRequest([tree], output=output, format=fmt)) + def test_unknown_format_cannot_run(self, small_file: Path) -> None: - """Test that an unknown format plans a job that refuses to run.""" + """Test that an unknown format plans a job that refuses to run, naming + the archive formats as well as the single-file ones.""" job = plan_compress(CompressRequest([small_file], format="bogus")) assert isinstance(job, CompressionJob) assert not job.plan.can_compress + assert job.plan.reason_if_unavailable is not None + assert job.plan.reason_if_unavailable.startswith("Unknown format: bogus") + assert all( + f" {fmt}," in job.plan.reason_if_unavailable + for fmt in ("gz", "zip", "tar", "tar.gz") + ) class TestPlanExtract: From 0718eaa0d62bad9b71dae966e7b29f0050b2f83f Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Thu, 5 Nov 2026 17:13:27 +0000 Subject: [PATCH 28/31] refactor(cli): pad a run's output in one place - main() now wraps a terminal stdout and stderr so a run gets exactly one blank line above its first visible output and one below its last - Shared between the streams so they pad once between them, and nothing extra where the output already has one - Piped output isn't padded, since tar and gzip don't pad what a script reads - Drops every command's own leading and trailing print(), so a summary, a stats block and an error no longer each decide their own spacing - A verbose run instead gets a blank line after the progress bar, to set it apart from the stats - Drops speed, saved bytes, and the separate ratio and original-size from compress stats, which the plan block and the output line already show - Adds ratio to the archive output summary - Adds unit tests for the new terminal output wrapping --- src/compresso/cli/__init__.py | 73 ++++++++++++++++++++++++++++++- src/compresso/cli/_render/info.py | 4 -- src/compresso/cli/_render/jobs.py | 25 +++++------ src/compresso/cli/_run.py | 11 +++-- src/compresso/cli/benchmark.py | 9 ++-- src/compresso/cli/compress.py | 12 +++-- src/compresso/cli/extract.py | 12 +++-- tests/python/cli/test_app.py | 39 +++++++++++++++-- 8 files changed, 147 insertions(+), 38 deletions(-) diff --git a/src/compresso/cli/__init__.py b/src/compresso/cli/__init__.py index cb8829c..4a8725f 100644 --- a/src/compresso/cli/__init__.py +++ b/src/compresso/cli/__init__.py @@ -2,6 +2,10 @@ from __future__ import annotations +import re +import sys +from typing import Any, TextIO + from . import benchmark, compress, extract, formats, info, verify from . import list as list_ from ._app import app @@ -23,9 +27,76 @@ _module.register() +# Styling and spaces, so Rich's padded blank lines count as blank +_INVISIBLE = re.compile(r"\x1b\[[0-9;]*m| ") + + +class _Padded: + """A terminal stream that gives a run's output one blank line above it. + + main() adds the one below. Output that already starts or ends with a blank + line, as `--help` does, gets no extra. + """ + + # Shared, so stdout and stderr pad once between them + tail: str | None = None # the last two visible characters written + last: _Padded | None = None + + def __init__(self, stream: TextIO) -> None: + """Initialise a padded stream. + + Args: + stream: The underlying stream to write to. + """ + self._stream = stream + + def write(self, data: str) -> int: + """Write to the underlying stream, with a blank line above if needed. + + Args: + data: The string to write. + + Returns: + The number of characters written. + """ + # Typer/Click probes for a binary stream with write(b"") + visible = _INVISIBLE.sub("", data) if isinstance(data, str) else "" + if visible: + if _Padded.tail is None and not visible.startswith("\n"): + self._stream.write("\n") + + _Padded.tail = ((_Padded.tail or "") + visible)[-2:] + _Padded.last = self + + return self._stream.write(data) + + def __getattr__(self, name: str) -> Any: + """Delegate attribute access to the underlying stream. + + Args: + name: The name of the attribute to access. + + Returns: + The value of the requested attribute. + """ + return getattr(self._stream, name) + + def main() -> None: """Entry point for the CLI.""" - app() + # Not when piped, as tar and gzip don't pad what a script reads + if sys.stdout.isatty(): + sys.stdout = _Padded(sys.stdout) + + if sys.stderr.isatty(): + sys.stderr = _Padded(sys.stderr) + + try: + app() + + finally: + if _Padded.last and _Padded.tail != "\n\n": + _Padded.last.write("\n") if __name__ == "__main__": diff --git a/src/compresso/cli/_render/info.py b/src/compresso/cli/_render/info.py index 52d2507..1d3903a 100644 --- a/src/compresso/cli/_render/info.py +++ b/src/compresso/cli/_render/info.py @@ -146,7 +146,6 @@ def archive_info( return data lines: list[str] = [ - "", f"File: {file}", f"Format: {fmt}", f"Entries: {len(entries)}", @@ -159,7 +158,6 @@ def archive_info( lines.append(f"Compression: {ratio:.1f}% of original") print("\n".join(lines)) - print() return data @@ -200,7 +198,6 @@ def codec_info( return data lines: list[str] = [ - "", f"File: {file}", f"Format: {fmt}", f"Size: {format_size(size_bytes=size)}", @@ -219,7 +216,6 @@ def codec_info( lines.append(f"Compression: {size / orig_size * 100:.1f}% of original") print("\n".join(lines)) - print() return data diff --git a/src/compresso/cli/_render/jobs.py b/src/compresso/cli/_render/jobs.py index 3b840e4..6683507 100644 --- a/src/compresso/cli/_render/jobs.py +++ b/src/compresso/cli/_render/jobs.py @@ -55,6 +55,7 @@ def compress_plan(plan: CompressionPlan) -> None: f"Compressing: {plan.src}", f"Output: {plan.dest}", f"Algorithm: {plan.backend_name}", + f"Input size: {format_size(size_bytes=plan.input_size)}", ] if not plan.options.format: @@ -78,22 +79,14 @@ def compress_result(plan: CompressionPlan, written: Path, elapsed: float) -> Non ratio: int | float = ( (compressed_size / plan.input_size) * 100 if plan.input_size > 0 else 0 ) - speed_mbs: int | float = ( - (plan.input_size / (1024 * 1024)) / elapsed if elapsed > 0 else 0 - ) - print() succeed("Compression successful!\n") if written != plan.dest: print(f" Renamed to: {written}") print( - f" Original size: {format_size(size_bytes=plan.input_size)}\n" - f" Compressed size: {format_size(size_bytes=compressed_size)}\n" - f" Ratio: {ratio:.1f}% of original\n" - f" Time: {format_time(seconds=elapsed)}\n" - f" Speed: {speed_mbs:.2f} MB/s\n" - f" Saved: {format_size(size_bytes=plan.input_size - compressed_size)}\n" + f" Output size: {format_size(size_bytes=compressed_size)} ({ratio:.1f}%)\n" + f" Time: {format_time(seconds=elapsed)}\n" ) @@ -119,14 +112,20 @@ def archive_result(plan: ArchivePlan, written: Path, elapsed: float) -> None: written: The path actually written, which a rename may have moved. elapsed: Seconds the job took. """ + compressed_size: int = written.stat().st_size + ratio: int | float = ( + (compressed_size / plan.total_input_size) * 100 + if plan.total_input_size > 0 + else 0 + ) + succeed("Archive created!\n") if written != plan.output: print(f" Renamed to: {written}") print( - f" Input size: {format_size(size_bytes=plan.total_input_size)}\n" - f" Archive size: {format_size(size_bytes=written.stat().st_size)}\n" - f" Time: {format_time(seconds=elapsed)}\n" + f" Archive size: {format_size(size_bytes=written.stat().st_size)} ({ratio:.1f}%)\n" + f" Time: {format_time(seconds=elapsed)}\n", ) diff --git a/src/compresso/cli/_run.py b/src/compresso/cli/_run.py index 385a1cd..37fcc17 100644 --- a/src/compresso/cli/_run.py +++ b/src/compresso/cli/_run.py @@ -104,6 +104,7 @@ def run_job( *, quiet: bool, what: str, + verbose: bool = False, ) -> tuple[float, JobResult]: """Run `job` under a progress bar. @@ -114,6 +115,8 @@ def run_job( total: Expected byte count; small jobs get no bar. quiet: Whether to suppress the bar. what: Operation name for messages, e.g. "Compression". + verbose: Whether a blank line follows the bar, to set it apart from + the stats block. Returns: Seconds the job took, and its result; a `SourceRemovalError` in that @@ -124,11 +127,13 @@ def run_job( """ start_time: float = time.time() - with progress_bar( - label, total, enabled=not quiet and total > BAR_THRESHOLD - ) as on_progress: + bar: bool = not quiet and total > BAR_THRESHOLD + with progress_bar(label, total, enabled=bar) as on_progress: result = job.run(progress=on_progress) + if bar and verbose: + print() + elapsed: float = time.time() - start_time if result.cancelled: diff --git a/src/compresso/cli/benchmark.py b/src/compresso/cli/benchmark.py index 59a9270..9a6afc0 100644 --- a/src/compresso/cli/benchmark.py +++ b/src/compresso/cli/benchmark.py @@ -113,23 +113,22 @@ def benchmark( ) if not results: - fail("No benchmark results generated\n") + fail("No benchmark results generated") benchmark_results(results) if update_cache: print() - succeed("Speed cache updated\n") - print() + succeed("Speed cache updated") except KeyboardInterrupt: cancelled("Benchmark") except FileNotFoundError as e: - fail(f"File not found: {e}\n", EXIT_USAGE) + fail(f"File not found: {e}", EXIT_USAGE) except Exception as e: - fail(f"Benchmark error: {e}\n") + fail(f"Benchmark error: {e}") def register() -> None: diff --git a/src/compresso/cli/compress.py b/src/compresso/cli/compress.py index a247594..b614157 100644 --- a/src/compresso/cli/compress.py +++ b/src/compresso/cli/compress.py @@ -51,7 +51,13 @@ def _compress_one_file( before = plan.src.stat() elapsed, _ = run_job( - job, plan.src, "Compressing", plan.input_size, quiet=quiet, what="Compression" + job, + plan.src, + "Compressing", + plan.input_size, + quiet=quiet, + what="Compression", + verbose=verbose, ) # RENAME may have written somewhere other than `plan.dest`; job.plan is @@ -98,6 +104,7 @@ def _archive_many(job: ArchiveJob, quiet: bool, verbose: bool) -> None: plan.total_input_size, quiet=quiet, what="Archive", + verbose=verbose, ) # RENAME may have written somewhere other than `plan.output`; job.plan is @@ -314,9 +321,6 @@ def run_one(job: CompressionJob | ArchiveJob) -> None: else: _compress_one_file(job, quiet, verbose, remove) - if not (quiet or verbose): - print() - sys.exit(run_all([job], run_one, "Compression")) except KeyboardInterrupt: diff --git a/src/compresso/cli/extract.py b/src/compresso/cli/extract.py index d3e3c50..7149044 100644 --- a/src/compresso/cli/extract.py +++ b/src/compresso/cli/extract.py @@ -83,7 +83,13 @@ def _extract_archive(job: ExtractJob, quiet: bool, verbose: bool, remove: bool) total: int = sum(entry.size for entry in plan.entries) before = plan.archive.stat() elapsed, _ = run_job( - job, plan.archive, "Extracting", total, quiet=quiet, what="Extraction" + job, + plan.archive, + "Extracting", + total, + quiet=quiet, + what="Extraction", + verbose=verbose, ) # Names and includes the filter dropped still count as found @@ -132,6 +138,7 @@ def _decompress_one_file( compressed_size, quiet=quiet, what="Decompression", + verbose=verbose, ) # RENAME may have written somewhere other than `plan.dest`; job.plan is @@ -407,9 +414,6 @@ def run_one(job: ExtractJob | DecompressionJob) -> None: _decompress_one_file(job, quiet, verbose, remove) - if not (quiet or verbose): - print() - try: sys.exit(run_all(plan_extract(request), run_one, "Extraction")) diff --git a/tests/python/cli/test_app.py b/tests/python/cli/test_app.py index ebe6e81..9e31418 100644 --- a/tests/python/cli/test_app.py +++ b/tests/python/cli/test_app.py @@ -3,6 +3,7 @@ from __future__ import annotations import gzip +import io import os import re import subprocess @@ -16,6 +17,7 @@ import compresso.cli.formats as formats_cmd import compresso.cli.info as info_cmd import compresso.cli.list as list_cmd +from compresso import cli from compresso.cli._render import ( EXIT_CANCELLED, EXIT_FAILED, @@ -227,14 +229,14 @@ def test_default_is_one_summary_line( ratio = r"\(\d+%\) " if args[0] == "compress" else "" assert re.fullmatch( - rf"\n{re.escape(str(name))} \S+ \S+ → \S+ \S+ {ratio}in \S+\n\n", output + rf"{re.escape(str(name))} \S+ \S+ → \S+ \S+ {ratio}in \S+\n\n", output ), output def test_inputs_are_one_blank_line_apart( self, payload: Path, tmp_path: Path ) -> None: - """Test that several inputs' lines have one blank line before, between - and after them.""" + """Test that several inputs' lines have one blank line between and + after them.""" inputs = [tmp_path / "a.gz", tmp_path / "b.gz"] for path in inputs: path.write_bytes(gzip.compress(payload.read_bytes())) @@ -242,7 +244,7 @@ def test_inputs_are_one_blank_line_apart( output = invoke("extract", *inputs, "-C", tmp_path / "dest").output assert re.fullmatch( - "".join(rf"\n{re.escape(str(path))} [^\n]+\n" for path in inputs) + "\n", + "".join(rf"{re.escape(str(path))} [^\n]+\n\n" for path in inputs), output, ), output @@ -292,6 +294,35 @@ def test_an_error_lands_after_the_inputs_before_it( assert result.returncode == EXIT_FAILED assert result.stdout.index(str(good)) < result.stdout.index(str(bad)) + @pytest.mark.parametrize( + "argv", + [["formats"], ["--help"], ["info", "missing"]], + ids=["unpadded", "already-padded", "stderr"], + ) + def test_a_terminal_gets_one_blank_line_around_output( + self, monkeypatch: pytest.MonkeyPatch, argv: list[str] + ) -> None: + """Test that main() pads terminal output to exactly one blank line + before and after, whether or not it already had one.""" + + class Terminal(io.StringIO): + def isatty(self) -> bool: + return True + + # One stream for both, so the order they were written in shows + terminal = Terminal() + monkeypatch.setattr(sys, "stdout", terminal) + monkeypatch.setattr(sys, "stderr", terminal) + monkeypatch.setattr(sys, "argv", ["compresso", *argv]) + monkeypatch.setattr(cli._Padded, "tail", None) + monkeypatch.setattr(cli._Padded, "last", None) + + with pytest.raises(SystemExit): + cli.main() + + visible = re.sub(r"\x1b\[[0-9;]*m| ", "", terminal.getvalue()) + assert re.fullmatch(r"\n[^\n].*[^\n]\n\n", visible, re.DOTALL), visible + class TestExitCodes: """Test 0 success, 1 the operation failed, 2 the request could not be made.""" From 96882ba2752ca2fc91869ed46aed117cedfb43c3 Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Thu, 5 Nov 2026 18:04:35 +0000 Subject: [PATCH 29/31] refactor(cli): add a decorator for command error handling - New handles_errors() holds the Ctrl-C exit, the core-error exit and the catch-all that each command repeated - Drops the outer try block from the command bodies - Drops 'HeaderError' and 'BackendError' from the compress and extract clauses, as they are 'Error' subclasses already - Commands that only had a catch-all now name the operation for a core error, rather than calling it unexpected --- src/compresso/cli/_render/__init__.py | 39 ++++++++++++- src/compresso/cli/benchmark.py | 9 +-- src/compresso/cli/compress.py | 81 ++++++++++++--------------- src/compresso/cli/extract.py | 16 +----- src/compresso/cli/formats.py | 18 ++---- src/compresso/cli/info.py | 18 ++---- src/compresso/cli/list.py | 18 ++---- src/compresso/cli/verify.py | 20 +++---- tests/python/cli/test_app.py | 14 +++++ 9 files changed, 117 insertions(+), 116 deletions(-) diff --git a/src/compresso/cli/_render/__init__.py b/src/compresso/cli/_render/__init__.py index 1e6e5c8..3ccf82f 100644 --- a/src/compresso/cli/_render/__init__.py +++ b/src/compresso/cli/_render/__init__.py @@ -6,12 +6,14 @@ from __future__ import annotations +import functools import sys -from collections.abc import Generator +from collections.abc import Callable, Generator from contextlib import contextmanager from pathlib import Path -from typing import NoReturn +from typing import NoReturn, ParamSpec +from ..._core import Error from ...frontend._job import ProgressCallback from .._app import app @@ -130,6 +132,39 @@ def cancelled(what: str) -> NoReturn: sys.exit(EXIT_CANCELLED) +P = ParamSpec("P") + + +def handles_errors(what: str) -> Callable[[Callable[P, None]], Callable[P, None]]: + """Give a command the standard Ctrl-C and error exits. + + Args: + what: Operation name for the messages, e.g. "Compression". + + Returns: + A decorator wrapping the command. + """ + + def decorate(fn: Callable[P, None]) -> Callable[P, None]: + @functools.wraps(fn) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> None: + try: + fn(*args, **kwargs) + + except KeyboardInterrupt: + cancelled(what) + + except Error as e: + fail(f"{what} error: {e}") + + except Exception as e: + fail(f"Unexpected error: {e}") + + return wrapper + + return decorate + + @contextmanager def progress_bar( label: str, total: int, *, enabled: bool diff --git a/src/compresso/cli/benchmark.py b/src/compresso/cli/benchmark.py index 9a6afc0..d394085 100644 --- a/src/compresso/cli/benchmark.py +++ b/src/compresso/cli/benchmark.py @@ -10,7 +10,7 @@ from ..introspect.benchmark import benchmark_file from ..introspect.capabilities import list_capabilities from ._app import app -from ._render import EXIT_USAGE, cancelled, fail, succeed +from ._render import EXIT_USAGE, fail, handles_errors, succeed from ._render.benchmark import benchmark_header, benchmark_results # Built from the core so `--help` lists only what this build compiled in @@ -36,6 +36,7 @@ def _parse_level(value: str) -> int | None: fail(f"Error: Invalid level {value!r}; use an integer or 'auto'", EXIT_USAGE) +@handles_errors("Benchmark") def benchmark( file: Annotated[Path, app.Argument(help="File to benchmark")], algo: Annotated[ @@ -121,15 +122,9 @@ def benchmark( print() succeed("Speed cache updated") - except KeyboardInterrupt: - cancelled("Benchmark") - except FileNotFoundError as e: fail(f"File not found: {e}", EXIT_USAGE) - except Exception as e: - fail(f"Benchmark error: {e}") - def register() -> None: """Attach this command to the app.""" diff --git a/src/compresso/cli/compress.py b/src/compresso/cli/compress.py index b614157..0125feb 100644 --- a/src/compresso/cli/compress.py +++ b/src/compresso/cli/compress.py @@ -7,13 +7,13 @@ from pathlib import Path from typing import Annotated -from .._core import BackendError, Error, HeaderError, SourceRemovalError +from .._core import SourceRemovalError from ..frontend.api import CompressionJob, Strategy from ..frontend.archive_api import ArchiveJob, OverwriteMode from ..frontend.dispatch import CompressRequest, plan_compress from ..introspect.capabilities import list_capabilities from ._app import app -from ._render import EXIT_USAGE, cancelled, fail +from ._render import EXIT_USAGE, fail, handles_errors from ._render.jobs import ( archive_plan, archive_result, @@ -130,6 +130,7 @@ def _archive_many(job: ArchiveJob, quiet: bool, verbose: bool) -> None: raise NotRemoved(*result.error.failures) +@handles_errors("Compression") def compress( inputs: Annotated[ list[Path], app.Argument(help="Files and directories to compress") @@ -284,53 +285,43 @@ def compress( if verbose and quiet: fail("Error: -v contradicts -q", EXIT_USAGE) - try: - if not inputs: - fail("Error: No input paths given", EXIT_USAGE) - - algo_name: str | None = None if algo == "auto" else algo - - # -a names the algorithm that -s would otherwise pick - if algo_name and strategy: - fail("Error: -a contradicts -s", EXIT_USAGE) - - request = CompressRequest( - inputs=inputs, - output=output, - format=format, - algo=algo_name, - strategy=strategy, - level=level, - overwrite=overwrite_mode, - threads=threads, - include=include or (), - exclude=exclude or (), - remove_sources=remove, - dereference=dereference, - ) - - try: - job = plan_compress(request) - - except ValueError as e: - fail(f"Error: {e}", EXIT_USAGE) - - def run_one(job: CompressionJob | ArchiveJob) -> None: - if isinstance(job, ArchiveJob): - _archive_many(job, quiet, verbose) - else: - _compress_one_file(job, quiet, verbose, remove) + if not inputs: + fail("Error: No input paths given", EXIT_USAGE) + + algo_name: str | None = None if algo == "auto" else algo + + # -a names the algorithm that -s would otherwise pick + if algo_name and strategy: + fail("Error: -a contradicts -s", EXIT_USAGE) + + request = CompressRequest( + inputs=inputs, + output=output, + format=format, + algo=algo_name, + strategy=strategy, + level=level, + overwrite=overwrite_mode, + threads=threads, + include=include or (), + exclude=exclude or (), + remove_sources=remove, + dereference=dereference, + ) - sys.exit(run_all([job], run_one, "Compression")) + try: + job = plan_compress(request) - except KeyboardInterrupt: - cancelled("Compression") + except ValueError as e: + fail(f"Error: {e}", EXIT_USAGE) - except (Error, HeaderError, BackendError) as e: - fail(f"Compression error: {e}") + def run_one(job: CompressionJob | ArchiveJob) -> None: + if isinstance(job, ArchiveJob): + _archive_many(job, quiet, verbose) + else: + _compress_one_file(job, quiet, verbose, remove) - except Exception as e: - fail(f"Unexpected error: {e}") + sys.exit(run_all([job], run_one, "Compression")) def register() -> None: diff --git a/src/compresso/cli/extract.py b/src/compresso/cli/extract.py index 7149044..1279bab 100644 --- a/src/compresso/cli/extract.py +++ b/src/compresso/cli/extract.py @@ -7,7 +7,6 @@ from pathlib import Path from typing import Annotated -from .._core import BackendError, Error, HeaderError from ..frontend.api import DecompressionJob from ..frontend.archive_api import ( ExtractJob, @@ -17,7 +16,7 @@ ) from ..frontend.dispatch import ExtractRequest, plan_extract from ._app import app -from ._render import EXIT_USAGE, cancelled, fail, report_warning +from ._render import EXIT_USAGE, fail, handles_errors, report_warning from ._render.jobs import ( decompress_plan, decompress_result, @@ -165,6 +164,7 @@ def _decompress_one_file( remove_source(plan.src, before) +@handles_errors("Extraction") def extract( inputs: Annotated[ list[Path], @@ -414,17 +414,7 @@ def run_one(job: ExtractJob | DecompressionJob) -> None: _decompress_one_file(job, quiet, verbose, remove) - try: - sys.exit(run_all(plan_extract(request), run_one, "Extraction")) - - except KeyboardInterrupt: - cancelled("Extraction") - - except (Error, HeaderError, BackendError) as e: - fail(f"Extraction error: {e}") - - except Exception as e: - fail(f"Unexpected error: {e}") + sys.exit(run_all(plan_extract(request), run_one, "Extraction")) def register() -> None: diff --git a/src/compresso/cli/formats.py b/src/compresso/cli/formats.py index f065349..3a93dc4 100644 --- a/src/compresso/cli/formats.py +++ b/src/compresso/cli/formats.py @@ -6,25 +6,19 @@ from ..frontend._formats import archive_formats from ..introspect.capabilities import list_capabilities from ._app import app -from ._render import cancelled, fail +from ._render import fail, handles_errors from ._render.formats import list_formats +@handles_errors("Format listing") def formats() -> None: """List the available compression algorithms and file formats.""" - try: - caps = list_capabilities() + caps = list_capabilities() - if not caps: - fail("No compression backends available") + if not caps: + fail("No compression backends available") - list_formats(caps, standalone_formats(), archive_formats()) - - except KeyboardInterrupt: - cancelled("Format listing") - - except Exception as e: - fail(f"Error listing formats: {e}") + list_formats(caps, standalone_formats(), archive_formats()) def register() -> None: diff --git a/src/compresso/cli/info.py b/src/compresso/cli/info.py index cbb6e0f..4b257e0 100644 --- a/src/compresso/cli/info.py +++ b/src/compresso/cli/info.py @@ -14,7 +14,7 @@ from ..frontend.dispatch import is_archive_format from ..introspect.file_inspect import inspect as inspect_file from ._app import app -from ._render import cancelled, fail, input_header +from ._render import handles_errors, input_header from ._render.info import archive_info, codec_info, file_info, print_json from ._run import InputFailed, require_files, run_all @@ -64,6 +64,7 @@ def _describer(path: Path) -> Callable[..., dict[str, object]]: ) +@handles_errors("Inspection") def info( inputs: Annotated[ list[Path], app.Argument(help="Files to describe", readable=False) @@ -97,19 +98,12 @@ def run_one(path: Path) -> None: items.append(describe(as_json=output_json)) - try: - code = run_all(inputs, run_one, "Inspection") + code = run_all(inputs, run_one, "Inspection") - if output_json: - print_json(items) + if output_json: + print_json(items) - sys.exit(code) - - except KeyboardInterrupt: - cancelled("Inspection") - - except Exception as e: - fail(f"Unexpected error: {e}") + sys.exit(code) def register() -> None: diff --git a/src/compresso/cli/list.py b/src/compresso/cli/list.py index a470b0e..b6e3c64 100644 --- a/src/compresso/cli/list.py +++ b/src/compresso/cli/list.py @@ -11,7 +11,7 @@ from ..frontend.archive_api import plan_extraction as plan_archive_extraction from ..frontend.dispatch import is_archive_format from ._app import app -from ._render import cancelled, fail, input_header +from ._render import handles_errors, input_header from ._render.info import list_entries, print_json from ._run import InputFailed, require_files, run_all @@ -44,6 +44,7 @@ def _read_entries(path: Path) -> tuple[str, list[ArchiveEntry]]: return fmt, plan.entries +@handles_errors("Listing") def list_archives( inputs: Annotated[ list[Path], app.Argument(help="Archives to list", readable=False) @@ -83,19 +84,12 @@ def run_one(path: Path) -> None: listings.append({"path": str(path), "format": fmt, "entries": entries}) - try: - code = run_all(inputs, run_one, "Listing") + code = run_all(inputs, run_one, "Listing") - if output_json: - print_json(listings) + if output_json: + print_json(listings) - sys.exit(code) - - except KeyboardInterrupt: - cancelled("Listing") - - except Exception as e: - fail(f"Unexpected error: {e}") + sys.exit(code) def register() -> None: diff --git a/src/compresso/cli/verify.py b/src/compresso/cli/verify.py index 99502a1..4979207 100644 --- a/src/compresso/cli/verify.py +++ b/src/compresso/cli/verify.py @@ -10,7 +10,7 @@ from .._core import TrailingDataWarning from ..frontend.api import VerificationJob from ._app import app -from ._render import cancelled, fail, succeed +from ._render import handles_errors, succeed from ._run import InputFailed, require_files, run_all, run_job @@ -33,6 +33,7 @@ def _verify_one(job: VerificationJob, quiet: bool) -> None: succeed(f"OK: {plan.src}") +@handles_errors("Test") def verify( inputs: Annotated[ list[Path], @@ -67,20 +68,13 @@ def verify( """ require_files(inputs) - try: - jobs = (VerificationJob.from_file(src=path, threads=threads) for path in inputs) + jobs = (VerificationJob.from_file(src=path, threads=threads) for path in inputs) - with warnings.catch_warnings(): - warnings.simplefilter("error", TrailingDataWarning) - code = run_all(jobs, lambda job: _verify_one(job, quiet), "Test") + with warnings.catch_warnings(): + warnings.simplefilter("error", TrailingDataWarning) + code = run_all(jobs, lambda job: _verify_one(job, quiet), "Test") - sys.exit(code) - - except KeyboardInterrupt: - cancelled("Test") - - except Exception as e: - fail(f"Unexpected error: {e}") + sys.exit(code) def register() -> None: diff --git a/tests/python/cli/test_app.py b/tests/python/cli/test_app.py index 9e31418..c7e0b9c 100644 --- a/tests/python/cli/test_app.py +++ b/tests/python/cli/test_app.py @@ -18,6 +18,7 @@ import compresso.cli.info as info_cmd import compresso.cli.list as list_cmd from compresso import cli +from compresso._core import Error from compresso.cli._render import ( EXIT_CANCELLED, EXIT_FAILED, @@ -468,6 +469,19 @@ def interrupt(*args: object, **kwargs: object) -> None: assert expected in result.output +def test_core_error_names_the_operation(monkeypatch: pytest.MonkeyPatch) -> None: + """Test that an error from the core is reported as the command's, not unexpected.""" + + def boom() -> None: + raise Error("boom") + + monkeypatch.setattr(formats_cmd, "list_capabilities", boom) + + result = invoke("formats", expect=EXIT_FAILED) + + assert "Format listing error: boom" in result.output + + class TestThreads: """Test that -T reaches each kind of job, and refuses a negative count.""" From 1a052d62d456c5349225b27708375bcaa71902bd Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Fri, 6 Nov 2026 23:15:43 +0000 Subject: [PATCH 30/31] fix(win32): stat through a link, and don't raise writing to a pipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fs_stat_wide() now takes the type, size, mtime, identity and link count from the handle, which follows the link - A handle it can't open is a dangling link, which now fails with an errno rather than returning a row with a zeroed identity - main() now reconfigures stdout with 'backslashreplace', since a pipe takes the locale code page, which can't encode the '→' in a summary line - The entry-over-the-archive case is now skipped on Windows, which won't let libarchive's open file be replaced - The snappy parametrisation test now gets explicit ids, since raw bytes overflows Windows' 32767-character 'PYTEST_CURRENT_TEST' --- src/compresso/cli/__init__.py | 6 ++++++ src/compresso/csrc/fsutil_win32.c | 33 +++++++++++++++++++++++-------- tests/python/cli/test_extract.py | 4 ++++ tests/python/test_core.py | 2 ++ 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/compresso/cli/__init__.py b/src/compresso/cli/__init__.py index 4a8725f..b4857c3 100644 --- a/src/compresso/cli/__init__.py +++ b/src/compresso/cli/__init__.py @@ -2,6 +2,7 @@ from __future__ import annotations +import io import re import sys from typing import Any, TextIO @@ -84,6 +85,11 @@ def __getattr__(self, name: str) -> Any: def main() -> None: """Entry point for the CLI.""" + # A pipe on Windows takes the locale code page, which can't encode markers + # like '→'; escape rather than raise + if isinstance(sys.stdout, io.TextIOWrapper): + sys.stdout.reconfigure(errors="backslashreplace") + # Not when piped, as tar and gzip don't pad what a script reads if sys.stdout.isatty(): sys.stdout = _Padded(sys.stdout) diff --git a/src/compresso/csrc/fsutil_win32.c b/src/compresso/csrc/fsutil_win32.c index 925b818..fff3e43 100644 --- a/src/compresso/csrc/fsutil_win32.c +++ b/src/compresso/csrc/fsutil_win32.c @@ -51,6 +51,8 @@ int fs_widen(const char *utf8, wchar_t *out, int out_count) { return 0; } +static void set_errno_from_win32(int fallback); + static int fs_narrow(const wchar_t *wide, char *out, int out_size) { if (WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wide, -1, out, out_size, NULL, NULL) == 0) { @@ -93,18 +95,33 @@ static int fs_stat_wide(const wchar_t *wpath, fs_stat *out) { else out->type = FS_TYPE_OTHER; - // _wstat64 leaves st_ino 0, so the identity needs a handle + // _wstat64 describes a symlink itself, and leaves st_ino 0; the handle + // follows links and supplies the rest; failing to open means dangling link if (out->type == FS_TYPE_FILE || out->type == FS_TYPE_DIR) { HANDLE h = fs_open_for_metadata(wpath); + if (h == INVALID_HANDLE_VALUE) { + set_errno_from_win32(ENOENT); + return -1; + } BY_HANDLE_FILE_INFORMATION info; - if (h != INVALID_HANDLE_VALUE) { - if (GetFileInformationByHandle(h, &info)) { - out->dev = info.dwVolumeSerialNumber; - out->ino = ((uint64_t)info.nFileIndexHigh << 32) | info.nFileIndexLow; - out->nlink = info.nNumberOfLinks; - } - CloseHandle(h); + BOOL ok = GetFileInformationByHandle(h, &info); + CloseHandle(h); + if (!ok) { + set_errno_from_win32(EIO); + return -1; } + + int is_dir = (info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; + out->type = is_dir ? FS_TYPE_DIR : FS_TYPE_FILE; + out->size = + is_dir ? 0 : ((uint64_t)info.nFileSizeHigh << 32) | info.nFileSizeLow; + // FILETIME counts 100ns intervals since 1601-01-01 + ULONGLONG ticks = ((ULONGLONG)info.ftLastWriteTime.dwHighDateTime << 32) | + info.ftLastWriteTime.dwLowDateTime; + out->mtime = (int64_t)(ticks / 10000000ULL) - 11644473600LL; + out->dev = info.dwVolumeSerialNumber; + out->ino = ((uint64_t)info.nFileIndexHigh << 32) | info.nFileIndexLow; + out->nlink = info.nNumberOfLinks; } return 0; diff --git a/tests/python/cli/test_extract.py b/tests/python/cli/test_extract.py index 428541f..ceb9dcf 100644 --- a/tests/python/cli/test_extract.py +++ b/tests/python/cli/test_extract.py @@ -525,6 +525,10 @@ def test_a_failed_decompression_keeps_the_input(self, tmp_path: Path) -> None: assert source.exists() + @pytest.mark.skipif( + sys.platform == "win32", + reason="Windows can't replace the archive while libarchive has it open", + ) def test_entry_written_over_the_archive_stays(self, tmp_path: Path) -> None: """Test that --rm doesn't delete what replaced the archive at its path.""" archive = tmp_path / "self.tar" diff --git a/tests/python/test_core.py b/tests/python/test_core.py index f251a29..1dd66c0 100644 --- a/tests/python/test_core.py +++ b/tests/python/test_core.py @@ -839,6 +839,8 @@ def test_snappy_chunks_are_walked( SNAPPY_ID + _snappy_chunk(0x00, b"\x81\x80\x04"), # States 65537 SNAPPY_ID + _snappy_chunk(0x01, b"x" * 65537), ], + # The bytes as an ID overflow Windows' 32767-character PYTEST_CURRENT_TEST + ids=["truncated", "trailing", "reserved", "states-65537", "chunk-65537"], ) def test_snappy_states_no_size_unless_fully_described( self, tmp_path: Path, data: bytes From d23280cc2d34949a746e9253d151b9feaea96615 Mon Sep 17 00:00:00 2001 From: Rob Webb Date: Fri, 6 Nov 2026 23:47:43 +0000 Subject: [PATCH 31/31] fix(fsutil): join paths with a platform-specific separator - New 'FS_SEP' names the OS-specific separator beside 'FS_IS_SEP', which already accepted both - fs_join() now uses 'FS_SEP', rather than always inserting '/', which a Windows path mixed with the backslashes --- src/compresso/csrc/fsutil.c | 2 +- src/compresso/csrc/fsutil.h | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/compresso/csrc/fsutil.c b/src/compresso/csrc/fsutil.c index 66ec124..2bf24ec 100644 --- a/src/compresso/csrc/fsutil.c +++ b/src/compresso/csrc/fsutil.c @@ -70,7 +70,7 @@ int fs_join(char *out, size_t out_size, const char *dir, const char *name) { memcpy(out, dir, dir_len); if (sep_len) - out[dir_len] = '/'; + out[dir_len] = FS_SEP; memcpy(out + dir_len + sep_len, name, name_len + 1); return 0; } diff --git a/src/compresso/csrc/fsutil.h b/src/compresso/csrc/fsutil.h index 0d06b1d..cbd7a36 100644 --- a/src/compresso/csrc/fsutil.h +++ b/src/compresso/csrc/fsutil.h @@ -14,6 +14,7 @@ // so prefix comparisons must test both #if defined(_WIN32) || defined(_WIN64) #define FS_IS_SEP(c) ((c) == '/' || (c) == '\\') +#define FS_SEP '\\' #define FS_NULL_DEVICE "NUL" #include @@ -23,6 +24,7 @@ int fs_widen(const char *utf8, wchar_t *out, int out_count); #else #define FS_IS_SEP(c) ((c) == '/') +#define FS_SEP '/' #define FS_NULL_DEVICE "/dev/null" #endif