From 825a8e9efbd5ccd4c845ee6299781daa812aa985 Mon Sep 17 00:00:00 2001 From: Alexander Belikov Date: Tue, 22 Sep 2026 01:16:54 +0200 Subject: [PATCH 1/2] improve foos --- .github/workflows/pre-commit.yml | 2 + .pre-commit-config.yaml | 8 +- CHANGELOG.md | 247 +-- README.md | 65 +- docs/contributing.md | 11 +- docs/examples.md | 146 +- docs/getting_started/installation.md | 29 +- docs/getting_started/quickstart.md | 65 +- docs/index.md | 18 +- docs/reference/core.md | 84 +- docs/reference/index.md | 20 +- pyproject.toml | 20 +- suthing/__init__.py | 65 +- suthing/compare.py | 246 ++- suthing/decorate.py | 192 -- suthing/environ.py | 62 + suthing/file_handle.py | 501 +++-- suthing/fs.py | 233 +++ suthing/hashing.py | 139 ++ suthing/iterx.py | 37 + suthing/jsonable.py | 93 + suthing/jsonl.py | 180 ++ suthing/log.py | 73 + suthing/profiling.py | 169 ++ suthing/py.typed | 0 suthing/text.py | 55 + suthing/timer.py | 146 +- test/conftest.py | 15 +- test/data/{example.jsonld => example.jsonl} | 0 .../{example.jsonld.gz => example.jsonl.gz} | Bin test/test_compare.py | 91 + test/test_decorate.py | 67 - test/test_file_handle.py | 241 ++- test/test_hashing.py | 68 + test/test_helpers.py | 169 ++ test/test_jsonl.py | 55 + test/test_profile.py | 35 - test/test_profiling.py | 72 + test/test_timer.py | 50 +- uv.lock | 1606 ++++++++++------- 40 files changed, 3717 insertions(+), 1658 deletions(-) delete mode 100644 suthing/decorate.py create mode 100644 suthing/environ.py create mode 100644 suthing/fs.py create mode 100644 suthing/hashing.py create mode 100644 suthing/iterx.py create mode 100644 suthing/jsonable.py create mode 100644 suthing/jsonl.py create mode 100644 suthing/log.py create mode 100644 suthing/profiling.py create mode 100644 suthing/py.typed create mode 100644 suthing/text.py rename test/data/{example.jsonld => example.jsonl} (100%) rename test/data/{example.jsonld.gz => example.jsonl.gz} (100%) create mode 100644 test/test_compare.py delete mode 100644 test/test_decorate.py create mode 100644 test/test_hashing.py create mode 100644 test/test_helpers.py create mode 100644 test/test_jsonl.py delete mode 100644 test/test_profile.py create mode 100644 test/test_profiling.py diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 4dba338..237e766 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -18,3 +18,5 @@ jobs: run: uv sync --group dev - name: Run pre-commit run: uv run pre-commit run --all-files + - name: Run tests + run: uv run pytest test diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 31175da..2b6474c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ fail_fast: false repos: # Python linting and formatting - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.10 + rev: v0.16.8 hooks: - id: ruff-check args: [--fix, --ignore, E722] @@ -21,7 +21,7 @@ repos: # YAML formatting - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks - rev: v2.15.0 + rev: v2.16.0 hooks: - id: pretty-format-yaml args: [--autofix, --indent, '4', --preserve-quotes] @@ -36,12 +36,12 @@ repos: # TOML formatting and sorting - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks - rev: v2.15.0 + rev: v2.16.0 hooks: - id: pretty-format-toml args: [--autofix] - repo: https://github.com/pappasam/toml-sort - rev: v0.24.3 + rev: v0.25.0 hooks: - id: toml-sort args: [-ia] diff --git a/CHANGELOG.md b/CHANGELOG.md index f3d9acd..4771655 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,209 +5,70 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -# Changelog - Connection Configuration Library - -## Version 0.4.0 (2025-05-01) - -### Breaking Changes - -1. **Connection Type Access Changed** - - Before: `connection_kind.config_class()` (method call) - - After: `connection_kind.config_class` (property access) - - Migration: - ```python - # Before - config_class = ConnectionKind.ARANGO.config_class() - - # After - config_class = ConnectionKind.ARANGO.config_class - ``` - -2. **Connection Configuration Creation** - - Before: Direct class instantiation with `hosts` parameter - - After: Use `url` parameter or factory with more flexible options - - Migration: - ```python - # Before - config = ArangoConnectionConfig(hosts="http://localhost:8529", cred_name="root", cred_pass="password") - - # After - config = ArangoConnectionConfig(url="http://localhost:8529", username="root", password="password") - # Or using factory - config = ConfigFactory.create_config({ - "db_type": "arango", - "url": "http://localhost:8529", - "username": "root", - "password": "password" - }) - ``` - -3. **Renamed Credential Parameters** - - Before: `cred_name`, `cred_pass` - - After: `username`, `password` (with backward compatibility) - - Migration: - ```python - # Before - config = Neo4jConnectionConfig(cred_name="neo4j", cred_pass="password") - - # After (preferred) - config = Neo4jConnectionConfig(username="neo4j", password="password") - # Old params still work but are deprecated - config = Neo4jConnectionConfig(cred_name="neo4j", cred_pass="password") # Still works - ``` - - - Before: `ip_addr` - - After: `hostname` - - - Before: WSGIConfig.host - - After: WSGIConfig.listen_addr - - - Migration: - ```python - # Before - config = Neo4jConnectionConfig(cred_name="neo4j", cred_pass="password") - - # After (preferred) - config = Neo4jConnectionConfig(username="neo4j", password="password") - # Old params still work but are deprecated - config = Neo4jConnectionConfig(cred_name="neo4j", cred_pass="password") # Still works - ``` - - -4. **URL Parsing Changes** - - Before: Custom parsing with string operations - - After: Standard library `urlparse` for robust URL handling - - Migration: - ```python - # Before - URL components were extracted with custom string operations - # After - Just use the url parameter and components are automatically extracted - config = WSGIConfig(url="http://localhost:5000/api/v1") - print(config.protocol) # "http" - print(config.ip_addr) # "localhost" - print(config.port) # "5000" - print(config.path) # "/api/v1" - ``` - -5. **Changed `hosts` Parameter to `url`** - - Before: Primary URL parameter was named `hosts` - - After: Primary URL parameter is named `url` - - Migration: - ```python - # Before - config = ProtoConnectionConfig(hosts="http://localhost:8080") - - # After - config = ProtoConnectionConfig(url="http://localhost:8080") - ``` - -### New Features - -1. **Direct URL-based Configuration** - ```python - # Create config with just a URL - config = ConfigFactory.create_config(url="neo4j://neo4j:password@localhost:7687/mydb") - ``` - -2. **Improved Factory with Type Inference** - ```python - # Factory now tries to infer connection type from URL - config = ConfigFactory.create_config(url="http://localhost:8529/_db/mydb") # Will infer ArangoDB if "arango" is in URL - ``` - -3. **Expanded File Handling** - ```python - # Load configuration from JSON or YAML - config = ConfigFactory.create_config(path="config.json") - config = ConfigFactory.create_config(path="config.yaml") - - # Save configuration - from suthing import FileHandle - FileHandle.save(config.__dict__, "config.json") - ``` - -4. **First-Class Support for Connection Base Class** - ```python - # Use the factory with automatic type detection - from suthing import ConnectionConfig - config_dict = {"db_type": "neo4j", "url": "neo4j://localhost:7687"} - config = ConnectionConfig.from_dict(config_dict) - ``` - -### Other Improvements - -1. **Better Type Annotations** - ```python - from typing import Dict, Optional - - # Types are now properly annotated - def process_config(config: ConnectionConfig) -> Dict[str, Optional[str]]: - return { - "type": config.connection_type.value if config.connection_type else None, - "url": config.url, - "database": getattr(config, "database", None) - } - ``` - -2. **Enhanced Error Handling** - ```python - # More specific error messages - try: - config = ConnectionConfig.from_dict({"db_type": "unknown"}) - except ValueError as e: - print(e) # "Connection type 'unknown' not supported. Should be one of: ['arango', 'neo4j', 'wsgi']" - ``` - - -## Migration Guide - -### Step 2: Update URL Parameters -```python -# Before -config = Neo4jConnectionConfig(hosts="http://localhost:7687", cred_name="neo4j", cred_pass="pass") - -# After -config = Neo4jConnectionConfig(url="http://localhost:7687", username="neo4j", password="pass") -``` - -### Step 3: Update Factory Usage -```python -# Before -config = ConfigFactory.create_config(dict_like={"db_type": "arango", "hosts": "http://localhost:8529"}) - -# After -config = ConfigFactory.create_config(dict_like={"db_type": "arango", "url": "http://localhost:8529"}) -# Or more directly -config = ConfigFactory.create_config(url="http://localhost:8529") -``` - -### Step 4: Update Connection Type Access -```python -# Before -config_class = db_type.config_class() - -# After -config_class = db_type.config_class -``` +## [Unreleased] +### Added + +- `suthing.jsonl`: `iter_jsonl` (streaming, strict or skip-and-warn), `read_jsonl` returning `(rows, errors)` with line numbers, and `write_jsonl` (append and atomic modes). All accept compressed files. +- `suthing.hashing`: `canonical_json`, `stable_hash`, `text_hash`, `bytes_hash`, `file_hash`, `tree_hash`. `stable_hash(obj)` is byte-identical to `sha256(json.dumps(obj, sort_keys=True, separators=(",", ":")))`, so existing stored hashes stay valid. +- `suthing.fs`: `atomic_write` / `atomic_open` (same-directory temp file + `os.replace`; keeps permissions; optional `fsync`), `open_compressed`, `expand_path`. +- `.bz2` and `.xz` compression, plus `.zst` via the new `zstd` extra. +- `FileHandle.load_resource(package, name)` for data shipped inside a package, and `FileHandle.iter` for streaming JSONL, text lines and CSV chunks. +- TSV support, `.jsonl` / `.ndjson` extensions, and a `.env` writer. +- `Timer`: live `elapsed` inside the block, `elapsed_ms`, `format(digits)`, an optional `label` and `log` callback, and decorator use. +- `diff()` returning a `Difference` (path, expected, actual, reason) for every mismatch, with `rel_tol` / `abs_tol`, `ignore_order` and `max_diffs`. +- `Profiler` / `profiled`: opt-in profiling through a context variable, so decorated functions no longer need a `_profiler` argument; `Profiler.summary()` gives count, total, mean, median and max per key. +- `to_jsonable`, `batched`, `slugify`, `env_flag`, `load_env`, `setup_logging`, `utc_now_iso`, `format_duration`. +- `py.typed` marker, Python 3.13 classifier, and a pytest step in CI. + +### Changed + +- **Breaking:** `FileHandle.load(path, *, how=None, **kwargs)` takes one path. Package data moves to `load_resource(package, name)`, and the `fpath=` keyword is gone. Unexpected keyword arguments raise `TypeError` (only CSV/TSV forward them to pandas). +- **Breaking:** an unknown extension raises `ValueError` instead of falling back to text. `how=` now overrides the extension; before, it was ignored whenever the extension was recognised. +- **Breaking:** `.jsonld` is read and written as JSON (JSON-LD). JSON Lines use `.jsonl` / `.ndjson`, and `FileType.JSONLD` is renamed `FileType.JSONL`. +- **Breaking:** `equals` compares lengths. It used `zip` and reported `[1, 2]` and `[1]` as equal. It now also compares sets as sets, treats `nan` as equal to `nan`, and no longer logs at ERROR level. +- **Breaking:** loading a dotenv file returns a `dict` and leaves `os.environ` alone; use `load_env` to export it. +- **Breaking:** YAML loads with `safe_load`. Dumps keep key order and write non-ASCII text as-is. Enums, tuples, paths, `Decimal` and similar values are written as plain YAML instead of `!!python/...` tags. +- JSON dumps write non-ASCII text as-is and convert enums, dates, UUIDs, paths and numpy values through `to_jsonable`. +- `FileHandle.dump` writes atomically by default, returns the written path, and takes `mkdir=`. It raises `TypeError` on a value the format cannot hold instead of writing an empty or `repr`'d file. Text into a `.gz` file works; it used to raise `TypeError`. +- pandas is imported only when a CSV/TSV file is read or written. + +### Removed + +- **Breaking:** `secureit`, `timeit`, `Report`, `Return`, `SProfiler`, `profile` and the `suthing.decorate` module. Use `Profiler` / `profiled` for profiling and `Timer` for timing. +- Unused runtime dependencies `dataclass-wizard` and `strenum`. `mkdocs-gen-files` moves to the `docs` dependency group. + +## [0.5.1] - 2026-02-01 + +### Changed + +- Require Python ≥3.11 (drop 3.10). +- Drop the upper pin on `dataclass-wizard`. +- Add `ty` to the `dev` dependency group for type checking. + +## [0.5.0] - 2025-11-09 + +### Removed + +- **Breaking:** remove the `suthing.connection` package (`ConnectionConfig`, `ConfigFactory`, and DB/WSGI connection configs). Connection configuration no longer ships with this library. +- Dedicated pytest GitHub Actions workflow; checks run through pre-commit. + +## [0.4.0] - 2025-05-01 + +Historical release that reshaped the connection-configuration API (URL/`username`/`password` parameters, factory changes). That API was removed entirely in [0.5.0](#050---2025-11-09). ## [0.3.0] - 2025-01-15 -- published on pypi +- published on pypi ## [0.2.4] - 2023-09-01 + - FileHandle - added support for reading .env files and pushing them to environment - ## [0.2.3] - 2023-08-30 + - switched to python 3.10 - FileHandle - added support for reading txt files by default - - if a single argument to FileHandle.load() is not named it is interpreted as filepath - + - if a single argument to FileHandle.load() is not named it is interpreted as filepath diff --git a/README.md b/README.md index 7b8554a..28edc51 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # SUThing suthing logo -SUThing /ˈsu.θɪŋ/ or /ˈsʌ.θɪŋ/ (Some Useful Things) is a collection of useful classes and decorators. +SUThing /ˈsu.θɪŋ/ or /ˈsʌ.θɪŋ/ (Some Useful Things) is a collection of small, dependable utilities. -A Python utility package providing tools for file handling, timing, profiling, and data comparison. +A Python utility package providing tools for file handling, timing, profiling, data comparison and hashing. ![Python](https://img.shields.io/badge/python-%3E=3.11-blue?logo=python) [![PyPI version](https://badge.fury.io/py/suthing.svg)](https://badge.fury.io/py/suthing) @@ -12,10 +12,11 @@ A Python utility package providing tools for file handling, timing, profiling, a ## Features -- **File Handling**: one-line file reading/wriing with file format infererence from provided extension (YAML, JSON, CSV, pickle, gz etc.) -- **Performance Measurement**: Simple timer utilities and profiling decorators -- **Data Comparison**: Deep comparison of nested data structures -- **Error Handling**: Decorators for secure function execution and error tracking +- **File Handling**: one-line reading and writing with the format inferred from the extension (YAML, JSON, JSON Lines, CSV/TSV, text, dotenv, pickle), transparent `.gz`/`.bz2`/`.xz`/`.zst` compression, atomic writes and streaming reads +- **Timing and Profiling**: a `Timer` context manager/decorator and opt-in function profiling with per-key statistics +- **Data Comparison**: deep `diff` of nested structures that reports the path of every difference, with numeric tolerance and order-insensitive matching +- **Hashing**: stable hashes of JSON-like values, text, files and directory trees +- **Small Helpers**: `batched`, `slugify`, `to_jsonable`, `env_flag`, `setup_logging`, `utc_now_iso` ## Documentation Full documentation is available at: [growgraph.github.io/suthing](https://growgraph.github.io/suthing) @@ -33,12 +34,17 @@ pip install suthing ```python from suthing import FileHandle, FileType -# Read YAML file -data = FileHandle.load(fpath="config.yaml") +# format inferred from the extension +data = FileHandle.load("config.yaml") -# Write compressed JSON -# file type inferred from extension +# compressed by suffix, written atomically FileHandle.dump(data, "output.json.gz") + +# data shipped inside a package +defaults = FileHandle.load_resource("mypkg.data", "defaults.yaml") + +# explicit format for an unrecognised extension +secret = FileHandle.load("token.secret", how=FileType.TXT) ``` ### Timing Code @@ -55,38 +61,45 @@ print(f"Execution took {t.elapsed_str}") ### Profiling Functions ```python -from suthing import profile, SProfiler +from suthing import Profiler, profiled -profiler = SProfiler() -@profile(_argnames="input_size") -def my_function(input_size): - # Function code - pass +@profiled(key_args="input_size") +def my_function(input_size): ... + -# Run with profiler -my_function(input_size=100, _profiler=profiler) +with Profiler() as prof: + my_function(100) -# View results -stats = profiler.view_stats() +stats = prof.summary() # {"my_function(input_size=100)": ProfileStats(...)} ``` ### Deep Comparison ```python -from suthing import equals +from suthing import diff, equals -# Compare nested structures -result = equals(complex_dict1, complex_dict2) +equals(expected, actual) # bool +for d in diff(expected, actual): # where they differ + print(d) # $.users[1].name: values differ (expected='Bob', actual='Rob') ``` ## Requirements -- Python 3.10+ -- pandas +- Python 3.11+ +- pandas (imported only for CSV/TSV) - PyYAML - python-dotenv +- optional: `zstandard` for `.zst` files (`pip install "suthing[zstd]"`) + +## Development + +```bash +uv sync --group dev +uv run pytest test +uv run ty check suthing test +``` ## Contributing -Contributions are welcome! Please feel free to submit a Pull Request. \ No newline at end of file +Contributions are welcome! Please feel free to submit a Pull Request. See [Contributing](docs/contributing.md) for the full workflow. diff --git a/docs/contributing.md b/docs/contributing.md index c2ac05c..b9634c9 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -8,11 +8,11 @@ We welcome contributions to Suthing! This document provides guidelines and instr 2. Clone your fork locally 3. Install the development dependencies: ```bash - uv sync --dev + uv sync --group dev ``` 4. Install pre-commit hooks: ```bash - pre-commit install + uv run pre-commit install ``` ## Development Workflow @@ -22,9 +22,10 @@ We welcome contributions to Suthing! This document provides guidelines and instr git checkout -b feature/your-feature-name ``` -2. Make your changes and ensure tests pass: +2. Make your changes and ensure tests and type checks pass: ```bash - pytest test + uv run pytest test + uv run ty check suthing test ``` 3. Commit your changes with a descriptive message: @@ -82,4 +83,4 @@ When reporting issues, please include: ## License -By contributing to Suthing, you agree that your contributions will be licensed under the project's MIT License. \ No newline at end of file +By contributing to Suthing, you agree that your contributions will be licensed under the project's MIT License. diff --git a/docs/examples.md b/docs/examples.md index 92fe341..aa73727 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -1,105 +1,123 @@ # Examples -This page contains practical examples of using Suthing in real-world scenarios. +Practical recipes for the utilities in Suthing. -## Configuration Management - -Load and manage configuration files with automatic format detection: +## Configuration Files ```python -from suthing import FileHandle +from suthing import FileHandle, load_env -# Load configuration from different formats config = FileHandle.load("config.yaml") -env_vars = FileHandle.load(".env") -df = FileHandle.load("secrets.csv") +settings = FileHandle.load("settings.env") # a dict; os.environ is untouched +load_env("settings.env") # explicitly push into os.environ -# Save configuration with compression FileHandle.dump(config, "config.json.gz") ``` -## Performance Optimization +## Large Record Files + +Stream JSON Lines instead of loading them whole, and collect bad lines instead of failing: + +```python +from suthing import iter_jsonl, read_jsonl, write_jsonl + +for record in iter_jsonl("events.jsonl.gz"): + handle(record) + +rows, errors = read_jsonl("labels.jsonl", require_object=True) +for error in errors: + print(error) # "line 17: not valid JSON (...)" + +write_jsonl(results, "out/results.jsonl", mkdir=True) +write_jsonl(more_results, "out/results.jsonl", append=True) +``` + +CSV files can be streamed in `DataFrame` chunks: + +```python +from suthing import FileHandle + +for chunk in FileHandle.iter("big.csv.gz", chunksize=50_000): + process(chunk) +``` + +## Safe Checkpoints + +A reader never sees a half-written file, and a crash mid-write leaves the previous checkpoint intact: + +```python +from suthing import FileHandle, atomic_write + +FileHandle.dump(state, "checkpoints/state.json") # atomic by default +atomic_write("checkpoints/READY", "ok", durable=True) # fsync before the rename +``` -Identify bottlenecks in your code: +## Content Hashes and Cache Keys ```python -from suthing import Timer, profile, SProfiler +from suthing import file_hash, stable_hash, text_hash, tree_hash + +key = stable_hash( + {"model": "m", "params": {"k": 3}}, length=16 +) # key order does not matter +text_hash(prompt, length=12) +file_hash("data/corpus.tar") +tree_hash("data/corpus", pattern="*.json") +``` + +`stable_hash(obj)` equals `sha256(json.dumps(obj, sort_keys=True, separators=(",", ":")))`, so it can replace that expression without changing hashes already stored. + +## Performance + +```python +from suthing import Profiler, Timer, profiled -# Time a specific operation with Timer() as t: result = process_large_dataset() -print(f"Dataset processing took {t.elapsed_str}") +print(f"Dataset processing took {t.elapsed_str} ({t.elapsed_ms} ms)") -# Profile a function with different inputs -profiler = SProfiler() -@profile(_argnames=["size", "batch_size"]) +@profiled(key_args=["size", "batch_size"]) def process_data(size, batch_size): for i in range(0, size, batch_size): process_batch(i, batch_size) -# Run with different parameters -process_data(size=1000, batch_size=100, _profiler=profiler) -process_data(size=1000, batch_size=50, _profiler=profiler) -# Compare performance -stats = profiler.view_stats() +with Profiler() as prof: + process_data(size=1000, batch_size=100) + process_data(size=1000, batch_size=50) + +for key, stats in prof.summary().items(): + print(key, stats.count, stats.mean) ``` -## Data Validation +## Test Assertions -Compare and validate complex data structures: +`diff` says where two structures differ, which makes failing assertions readable: ```python -from suthing import equals - -# Compare configuration files -config1 = FileHandle.load("config1.yaml") -config2 = FileHandle.load("config2.yaml") -if not equals(config1, config2): - print("Configurations differ!") - -# Compare nested data structures -expected = { - "users": [ - {"id": 1, "name": "Alice"}, - {"id": 2, "name": "Bob"} - ], - "settings": {"theme": "dark", "notifications": True} -} - -actual = { - "users": [ - {"id": 1, "name": "Alice"}, - {"id": 2, "name": "Bob"} - ], - "settings": {"theme": "dark", "notifications": True} -} - -if equals(expected, actual): - print("Data matches expected structure") -``` +from suthing import diff -## Error Handling +differences = diff(expected, actual, rel_tol=1e-9, ignore_order=True) +assert not differences, "\n".join(map(str, differences)) +``` -Use decorators for secure function execution: +## Small Helpers ```python -from suthing import secure_execution +from suthing import batched, env_flag, setup_logging, slugify, to_jsonable, utc_now_iso -@secure_execution -def process_sensitive_data(data): - # Your code here - return result +for batch in batched(rows, 500): + db.insert_many(batch) -# The function will handle errors gracefully -result = process_sensitive_data(data) +slugify(" Person / Company ") # "Person-Company" +env_flag("MYAPP_DEBUG") # "1"/"true"/"yes"/"on" → True; typos raise +setup_logging("INFO") # or setup_logging(config="logging.conf") +to_jsonable({"when": datetime.now(), "score": numpy.float32(0.5)}) +utc_now_iso() # "2026-01-02T10:15:00+00:00" ``` ## More Examples -For more detailed examples and use cases, check out: - - - [API Reference](reference/index.md) - Complete API documentation - [Quick Start](getting_started/quickstart.md) - Basic usage examples diff --git a/docs/getting_started/installation.md b/docs/getting_started/installation.md index 3966187..b368128 100644 --- a/docs/getting_started/installation.md +++ b/docs/getting_started/installation.md @@ -2,8 +2,8 @@ ## Prerequisites -- Python 3.10 or higher -- pip (Python package installer) +- Python 3.11 or higher +- [uv](https://docs.astral.sh/uv/) (for development) or pip (for end-user install) ## Installation Steps @@ -17,6 +17,7 @@ pip install suthing ```python import suthing + print(suthing.__version__) ``` @@ -30,6 +31,12 @@ Suthing requires the following packages: These will be automatically installed when you install Suthing using pip. +Reading and writing `.zst` (Zstandard) files needs one optional package: + +```bash +pip install "suthing[zstd]" +``` + ## Development Installation If you want to contribute to Suthing or run the tests: @@ -40,21 +47,27 @@ git clone https://github.com/growgraph/suthing.git cd suthing ``` -2. Install in development mode: +2. Install development dependencies (uses `[dependency-groups]`, not optional extras): ```bash -pip install -e ".[dev]" +uv sync --group dev ``` 3. Install pre-commit hooks: ```bash -pre-commit install +uv run pre-commit install +``` + +4. Run tests and type checks: +```bash +uv run pytest test +uv run ty check suthing test ``` ## Troubleshooting If you encounter any issues during installation: -1. Make sure you have Python 3.10 or higher installed -2. Try upgrading pip: `pip install --upgrade pip` +1. Make sure you have Python 3.11 or higher installed +2. For pip installs, try upgrading pip: `pip install --upgrade pip` 3. Check if all dependencies are properly installed -4. If problems persist, please open an issue on GitHub \ No newline at end of file +4. If problems persist, please open an issue on GitHub diff --git a/docs/getting_started/quickstart.md b/docs/getting_started/quickstart.md index e1e5931..cc2c9a7 100644 --- a/docs/getting_started/quickstart.md +++ b/docs/getting_started/quickstart.md @@ -1,71 +1,80 @@ # Quick Start -This guide will help you get started with Suthing quickly. We'll cover the main features with simple examples. +This guide covers the main features with short examples. ## File Handling -Suthing makes file operations simple with automatic format detection: +The format comes from the file extension, and a compression suffix (`.gz`, `.bz2`, `.xz`, `.zst`) is handled transparently: ```python from suthing import FileHandle -# Read a YAML file data = FileHandle.load("config.yaml") +FileHandle.dump(data, "out/config.json.gz", mkdir=True) # atomic by default -# Write to a compressed JSON file -FileHandle.dump(data, "output.json.gz") +# data files shipped inside a package +defaults = FileHandle.load_resource("mypkg.data", "defaults.yaml") ``` -## Performance Measurement +An unknown extension is an error; pass `how=` to choose the format explicitly: -Time your code execution easily: +```python +from suthing import FileType + +notes = FileHandle.load("notes.secret", how=FileType.TXT) +``` + +## Timing ```python from suthing import Timer with Timer() as t: - # Your code here result = some_expensive_operation() print(f"Operation took {t.elapsed_str}") + +# or log on exit +import logging + +with Timer("ingest", log=logging.getLogger(__name__).info): + ingest() ``` ## Profiling Functions -Profile your functions to understand their performance: +Decorated functions are timed only while a `Profiler` is active: ```python -from suthing import profile, SProfiler +from suthing import Profiler, profiled + -profiler = SProfiler() +@profiled(key_args="input_size") +def process_data(input_size): ... -@profile(_argnames="input_size") -def process_data(input_size): - # Process data based on input size - return result -# Run with profiler -result = process_data(input_size=100, _profiler=profiler) +with Profiler() as prof: + process_data(100) + process_data(1000) -# View profiling results -stats = profiler.view_stats() +prof.summary() # {"process_data(input_size=100)": ProfileStats(count=1, ...), ...} ``` ## Data Comparison -Compare complex data structures: +`diff` lists every difference with its location; `equals` is its boolean form: ```python -from suthing import equals +from suthing import diff, equals -# Compare nested dictionaries -dict1 = {"a": {"b": 1, "c": [1, 2, 3]}} -dict2 = {"a": {"b": 1, "c": [1, 2, 3]}} -result = equals(dict1, dict2) # True -``` +expected = {"a": {"b": 1, "c": [1, 2, 3]}} +actual = {"a": {"b": 1, "c": [1, 2]}} +equals(expected, actual) # False +for d in diff(expected, actual): + print(d) # $.a.c[2]: missing item (expected=3, actual=) +``` ## Next Steps - - Check out more [Examples](../examples.md) -- Read the complete [API Reference](../reference/index.md) \ No newline at end of file +- Read the complete [API Reference](../reference/index.md) diff --git a/docs/index.md b/docs/index.md index 5a76a9b..1721172 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,8 +1,8 @@ # Welcome to Suthing -SUThing /ˈsu.θɪŋ/ or /ˈsʌ.θɪŋ/ (Some Useful Things) is a collection of useful classes and decorators. +SUThing /ˈsu.θɪŋ/ or /ˈsʌ.θɪŋ/ (Some Useful Things) is a collection of small, dependable utilities. -A Python utility package providing tools for file handling, timing, profiling, and data comparison. +A Python utility package providing tools for file handling, timing, profiling, data comparison and hashing. ![Python](https://img.shields.io/badge/python-%3E=3.11-blue?logo=python) [![PyPI version](https://badge.fury.io/py/suthing.svg)](https://badge.fury.io/py/suthing) @@ -12,10 +12,11 @@ A Python utility package providing tools for file handling, timing, profiling, a ## Features -- **File Handling**: One-line file reading/writing with file format inference from provided extension (YAML, JSON, CSV, pickle, gz etc.) -- **Performance Measurement**: Simple timer utilities and profiling decorators -- **Data Comparison**: Deep comparison of nested data structures -- **Error Handling**: Decorators for secure function execution and error tracking +- **File Handling**: one-line reading and writing with the format inferred from the extension (YAML, JSON, JSON Lines, CSV/TSV, text, dotenv, pickle), transparent `.gz`/`.bz2`/`.xz`/`.zst` compression, atomic writes and streaming reads +- **Timing and Profiling**: a `Timer` context manager/decorator and opt-in function profiling with per-key statistics +- **Data Comparison**: deep `diff` of nested structures that reports the path of every difference, with numeric tolerance and order-insensitive matching +- **Hashing**: stable hashes of JSON-like values, text, files and directory trees +- **Small Helpers**: `batched`, `slugify`, `to_jsonable`, `env_flag`, `setup_logging`, `utc_now_iso` ## Quick Start @@ -45,10 +46,11 @@ Explore the documentation to learn more about Suthing's features: ## Requirements -- Python 3.10+ -- pandas +- Python 3.11+ +- pandas (imported only for CSV/TSV) - PyYAML - python-dotenv +- optional: `zstandard` for `.zst` files (`pip install "suthing[zstd]"`) ## Contributing diff --git a/docs/reference/core.md b/docs/reference/core.md index 0055e01..f19ba88 100644 --- a/docs/reference/core.md +++ b/docs/reference/core.md @@ -1,35 +1,87 @@ -# Core Module +# Core Modules -The core module provides essential utilities for file handling, timing, and data comparison. +Everything below is importable from the top-level `suthing` package. ## File Handling -::: suthing.file_handle.FileHandle +::: suthing.file_handle options: show_root_toc_entry: false - show_source: true - heading_level: 2 + heading_level: 3 -## Timer +## JSON Lines -::: suthing.timer.Timer +::: suthing.jsonl options: show_root_toc_entry: false - show_source: true - heading_level: 2 + heading_level: 3 -## Profiler +## File System -::: suthing.decorate.SProfiler +::: suthing.fs options: show_root_toc_entry: false - show_source: true - heading_level: 2 + heading_level: 3 + +## Timing + +::: suthing.timer + options: + show_root_toc_entry: false + heading_level: 3 + +## Profiling + +::: suthing.profiling + options: + show_root_toc_entry: false + heading_level: 3 ## Comparison -::: suthing.compare.equals +::: suthing.compare + options: + show_root_toc_entry: false + heading_level: 3 + +## Hashing + +::: suthing.hashing + options: + show_root_toc_entry: false + heading_level: 3 + +## JSON Conversion + +::: suthing.jsonable + options: + show_root_toc_entry: false + heading_level: 3 + +## Iteration + +::: suthing.iterx + options: + show_root_toc_entry: false + heading_level: 3 + +## Text + +::: suthing.text + options: + show_root_toc_entry: false + heading_level: 3 + +## Environment + +::: suthing.environ + options: + show_root_toc_entry: false + heading_level: 3 + +## Logging + +::: suthing.log options: show_root_toc_entry: false - show_source: true - heading_level: 2 \ No newline at end of file + heading_level: 3 diff --git a/docs/reference/index.md b/docs/reference/index.md index d36d270..b7c34f2 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -1,16 +1,24 @@ # API Reference -Welcome to the Suthing API Reference documentation. This section contains detailed documentation for all modules and classes. +Every public name is importable from `suthing` directly. ## Core Modules -- [File Handling](core.md#file-handling) - Utilities for file operations -- [Timer](core.md#timer) - Performance measurement tools -- [Profiler](core.md#profiler) - Function profiling utilities -- [Comparison](core.md#comparison) - Deep data structure comparison +- [File Handling](core.md#file-handling) - `FileHandle`, `FileType`: format-aware load/dump with compression +- [JSON Lines](core.md#json-lines) - `iter_jsonl`, `read_jsonl`, `write_jsonl` +- [File System](core.md#file-system) - `atomic_write`, `atomic_open`, `open_compressed`, `expand_path` +- [Timing](core.md#timing) - `Timer`, `format_duration`, `utc_now_iso` +- [Profiling](core.md#profiling) - `Profiler`, `profiled` +- [Comparison](core.md#comparison) - `diff`, `equals` +- [Hashing](core.md#hashing) - `stable_hash`, `canonical_json`, `text_hash`, `file_hash`, `tree_hash` +- [JSON Conversion](core.md#json-conversion) - `to_jsonable` +- [Iteration](core.md#iteration) - `batched` +- [Text](core.md#text) - `slugify` +- [Environment](core.md#environment) - `env_flag`, `load_env` +- [Logging](core.md#logging) - `setup_logging` ## Quick Links - [Getting Started](../getting_started/quickstart.md) - Basic usage examples - [Examples](../examples.md) - Practical use cases -- [Contributing](../contributing.md) - How to contribute \ No newline at end of file +- [Contributing](../contributing.md) - How to contribute diff --git a/pyproject.toml b/pyproject.toml index c985119..1b3933c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,26 +10,24 @@ dev = [ "ty>=0.0.1a25" ] docs = [ + "mkdocs-gen-files>=0.5.0", "mkdocs-material", "mkdocs>=1.6.1", "mkdocstrings[python]>=0.29.1" ] [project] -authors = [{email = "alexander@growgraph.dev", name = "Alexander Belikov"}] classifiers = [ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython" ] dependencies = [ - "dataclass-wizard>=0.33.1", - "mkdocs-gen-files>=0.5.0", "pandas>=2.3.3", "python-dotenv>=1.0.0,<2", - "pyyaml>=3.10", - "strenum>=0.4.15" + "pyyaml>=6.0" ] description = "Small Useful Things : python package for convenient primitives" keywords = [ @@ -40,7 +38,14 @@ license = "MIT" name = "suthing" readme = "README.md" requires-python = ">=3.11" -version = "0.5.1" +version = "0.6.0" + +[[project.authors]] +email = "alexander@growgraph.dev" +name = "Alexander Belikov" + +[project.optional-dependencies] +zstd = ["zstandard>=0.22"] [project.urls] Homepage = "https://github.com/growgraph/suthing" @@ -50,6 +55,3 @@ include = ["suthing"] [tool.hatch.build.targets.wheel] include = ["suthing"] - -[tool.mypy] -ignore_missing_imports = true diff --git a/suthing/__init__.py b/suthing/__init__.py index 59ee8c5..f5eb4d0 100644 --- a/suthing/__init__.py +++ b/suthing/__init__.py @@ -1,21 +1,62 @@ +"""Small useful things: format-aware file I/O, timing, profiling, deep diffs, +hashing and other helpers shared across projects.""" + from importlib.metadata import version -from .compare import equals -from .decorate import Report, Return, SProfiler, profile, secureit, timeit -from .file_handle import FileHandle -from .timer import Timer +from .compare import MISSING, Difference, diff, equals +from .environ import env_flag, load_env +from .file_handle import FileHandle, FileType +from .fs import atomic_open, atomic_write, expand_path, open_compressed +from .hashing import ( + bytes_hash, + canonical_json, + file_hash, + stable_hash, + text_hash, + tree_hash, +) +from .iterx import batched +from .jsonable import to_jsonable +from .jsonl import JsonlError, iter_jsonl, read_jsonl, write_jsonl +from .log import setup_logging +from .profiling import Profiler, ProfileStats, active_profiler, profiled +from .text import slugify +from .timer import Timer, format_duration, utc_now_iso __version__ = version(__name__) - __all__ = [ + "MISSING", + "Difference", + "FileHandle", + "FileType", + "JsonlError", + "ProfileStats", + "Profiler", "Timer", + "active_profiler", + "atomic_open", + "atomic_write", + "batched", + "bytes_hash", + "canonical_json", + "diff", + "env_flag", "equals", - "Report", - "Return", - "SProfiler", - "profile", - "timeit", - "secureit", - "FileHandle", + "expand_path", + "file_hash", + "format_duration", + "iter_jsonl", + "load_env", + "open_compressed", + "profiled", + "read_jsonl", + "setup_logging", + "slugify", + "stable_hash", + "text_hash", + "to_jsonable", + "tree_hash", + "utc_now_iso", + "write_jsonl", ] diff --git a/suthing/compare.py b/suthing/compare.py index 1498e78..fa8761b 100644 --- a/suthing/compare.py +++ b/suthing/compare.py @@ -1,36 +1,232 @@ -"""Utilities for comparing complex nested data structures.""" +"""Deep comparison of nested data, with the location of every difference.""" -import logging -from collections.abc import Iterable -from typing import Any +from __future__ import annotations -logger = logging.getLogger(__name__) +import math +from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Set as AbstractSet +from numbers import Real +from typing import Any, NamedTuple -excluded_types = (str, dict) +class _Missing: + def __repr__(self) -> str: + return "" -def equals(a: Any, b: Any) -> bool: - """Deep comparison of two objects. - Recursively compares nested dictionaries and iterables. - Strings and dicts are treated as atomic values. +#: Stands in for the absent side of a :class:`Difference`. +MISSING: Any = _Missing() - Args: - a: First object to compare - b: Second object to compare - Returns: - True if objects are equal, False otherwise +class Difference(NamedTuple): + """One place where two structures differ. + + Attributes: + path: Where, e.g. ``$.users[1].name``. + expected: The value on the expected side, or :data:`MISSING`. + actual: The value on the actual side, or :data:`MISSING`. + reason: Short description of the mismatch. """ - if isinstance(a, dict) and isinstance(b, dict): - if a.keys() != b.keys(): - logger.error(f"a: {a.keys()} ; b: {b.keys()}") + + path: str + expected: Any + actual: Any + reason: str + + def __str__(self) -> str: + return ( + f"{self.path}: {self.reason}" + f" (expected={self.expected!r}, actual={self.actual!r})" + ) + + +class _Enough(Exception): + pass + + +def _child(path: str, key: Any) -> str: + if isinstance(key, str) and key.isidentifier(): + return f"{path}.{key}" + return f"{path}[{key!r}]" + + +def _is_container(x: Any) -> bool: + return isinstance(x, Iterable) and not isinstance(x, (str, bytes, bytearray)) + + +class _Differ: + def __init__( + self, + rel_tol: float, + abs_tol: float, + ignore_order: bool, + max_diffs: int | None, + ) -> None: + self.rel_tol = rel_tol + self.abs_tol = abs_tol + self.ignore_order = ignore_order + self.max_diffs = max_diffs + self.found: list[Difference] = [] + + def add(self, path: str, expected: Any, actual: Any, reason: str) -> None: + self.found.append(Difference(path, expected, actual, reason)) + if self.max_diffs is not None and len(self.found) >= self.max_diffs: + raise _Enough + + def same(self, a: Any, b: Any) -> bool: + sub = _Differ(self.rel_tol, self.abs_tol, self.ignore_order, 1) + try: + sub.walk(a, b, "$") + except _Enough: return False + return not sub.found + + def walk(self, e: Any, a: Any, path: str) -> None: + if isinstance(e, Mapping) and isinstance(a, Mapping): + self.mappings(e, a, path) + elif isinstance(e, AbstractSet) and isinstance(a, AbstractSet): + self.sets(e, a, path) + elif _is_container(e) and _is_container(a): + if not isinstance(e, Mapping) and not isinstance(a, Mapping): + self.sequences(e, a, path) + else: + self.add(path, e, a, f"type {type(e).__name__} != {type(a).__name__}") else: - return all([equals(a[k], b[k]) for k in a.keys()]) - elif (isinstance(a, Iterable) and not isinstance(a, excluded_types)) and ( - isinstance(b, Iterable) and not isinstance(b, excluded_types) - ): - return all([equals(ea, eb) for ea, eb in zip(a, b)]) - else: - return a == b + self.scalars(e, a, path) + + def mappings(self, e: Mapping, a: Mapping, path: str) -> None: + for k, v in e.items(): + if k in a: + self.walk(v, a[k], _child(path, k)) + else: + self.add(_child(path, k), v, MISSING, "missing key") + for k, v in a.items(): + if k not in e: + self.add(_child(path, k), MISSING, v, "unexpected key") + + def sets(self, e: AbstractSet, a: AbstractSet, path: str) -> None: + for v in e - a: + self.add(path, v, MISSING, "missing element") + for v in a - e: + self.add(path, MISSING, v, "unexpected element") + + def sequences(self, e: Iterable, a: Iterable, path: str) -> None: + es = e if isinstance(e, Sequence) else list(e) + as_ = a if isinstance(a, Sequence) else list(a) + if self.ignore_order: + self.unordered(es, as_, path) + return + for i in range(min(len(es), len(as_))): + self.walk(es[i], as_[i], f"{path}[{i}]") + for i in range(len(as_), len(es)): + self.add(f"{path}[{i}]", es[i], MISSING, "missing item") + for i in range(len(es), len(as_)): + self.add(f"{path}[{i}]", MISSING, as_[i], "unexpected item") + + def unordered(self, es: Sequence, as_: Sequence, path: str) -> None: + unmatched = list(range(len(as_))) + for i, ev in enumerate(es): + for j in unmatched: + if self.same(ev, as_[j]): + unmatched.remove(j) + break + else: + self.add(f"{path}[{i}]", ev, MISSING, "no matching item") + for j in unmatched: + self.add(f"{path}[{j}]", MISSING, as_[j], "unexpected item") + + def scalars(self, e: Any, a: Any, path: str) -> None: + if _is_number(e) and _is_number(a): + if not self.numbers_match(e, a): + self.add(path, e, a, "values differ") + return + try: + equal = bool(e == a) + except (TypeError, ValueError) as exc: # e.g. an ambiguous array truth value + self.add(path, e, a, f"cannot compare ({type(exc).__name__})") + return + if not equal: + reason = ( + "values differ" + if type(e) is type(a) + else f"type {type(e).__name__} != {type(a).__name__}" + ) + self.add(path, e, a, reason) + + def numbers_match(self, e: Any, a: Any) -> bool: + fe, fa = float(e), float(a) + if math.isnan(fe) or math.isnan(fa): + return math.isnan(fe) and math.isnan(fa) + if self.rel_tol == 0 and self.abs_tol == 0: + return e == a + return math.isclose(fe, fa, rel_tol=self.rel_tol, abs_tol=self.abs_tol) + + +def _is_number(x: Any) -> bool: + return isinstance(x, Real) and not isinstance(x, bool) + + +def diff( + expected: Any, + actual: Any, + *, + rel_tol: float = 0.0, + abs_tol: float = 0.0, + ignore_order: bool = False, + max_diffs: int | None = None, +) -> list[Difference]: + """List every difference between two nested structures. + + Mappings are compared key by key; sets as sets; other non-string iterables + (lists, tuples, generators, arrays) item by item, so a length mismatch is + reported as missing or unexpected items. Everything else is compared with + ``==``. Numbers of different types compare by value, ``nan`` equals + ``nan``, and ``bool`` is not treated as a number. + + Args: + expected: Reference value. + actual: Value under test. + rel_tol: Relative tolerance for numbers (:func:`math.isclose`). + abs_tol: Absolute tolerance for numbers. + ignore_order: Match sequence items regardless of position (as a + multiset). Quadratic in sequence length. + max_diffs: Stop after this many differences. + + Returns: + The differences, empty when the structures match. + + Example: + ```python + for d in diff({"a": [1, 2]}, {"a": [1]}): + print(d) + # $.a[1]: missing item (expected=2, actual=) + ``` + """ + differ = _Differ(rel_tol, abs_tol, ignore_order, max_diffs) + try: + differ.walk(expected, actual, "$") + except _Enough: + pass + return differ.found + + +def equals( + a: Any, + b: Any, + *, + rel_tol: float = 0.0, + abs_tol: float = 0.0, + ignore_order: bool = False, +) -> bool: + """Whether two nested structures match; see :func:`diff` for the rules. + + Stops at the first difference. + """ + return not diff( + a, + b, + rel_tol=rel_tol, + abs_tol=abs_tol, + ignore_order=ignore_order, + max_diffs=1, + ) diff --git a/suthing/decorate.py b/suthing/decorate.py deleted file mode 100644 index e929f6d..0000000 --- a/suthing/decorate.py +++ /dev/null @@ -1,192 +0,0 @@ -from __future__ import annotations - -import dataclasses -import functools -import hashlib -from collections import defaultdict -from copy import deepcopy -from typing import Any - -from suthing.timer import Timer - - -class SProfiler: - def __init__(self): - self._accumulator: defaultdict[str, list] = defaultdict(list) - - def add_metric(self, hkey, metric_key=None, value=0): - self._accumulator[hkey] += [value] - - def view_stats(self): - return deepcopy(self._accumulator) - - -@dataclasses.dataclass -class SimpleReturn: - ret: Any - elapsed: float | None = None - success: bool | None = None - exception: Exception | None = None - - -class Report: - def __init__(self, *args, **kwargs): - self.hkey: str = kwargs.pop("hkey", None) - self.elapsed: float = kwargs.pop("elapsed", None) - self.success: bool = kwargs.pop("success", None) - self.exception: Exception = kwargs.pop("exception", None) - - def __repr__(self): - s = "" - for k, v in self.__dict__.items(): - s += f"{k} : {v.__repr__()} \n" - return s - - -class Return(Report): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.ret: Any = kwargs.pop("ret", None) - self.reports: list[Report] = kwargs.pop("reports", []) - - def __repr__(self): - s = "" - for k, v in self.__dict__.items(): - s += f"{k} : {v.__repr__()} \n" - return s - - def update(self, rets: list[Report]): - for r in rets: - self.reports += [Report(**r.__dict__)] - - -def hash_args(*args, **kwargs): - m = hashlib.sha256() - for arg in args: - m.update(f"{arg}".encode("utf-8")) - for key in sorted(kwargs.keys()): - m.update(f"{kwargs[key]}".encode("utf-8")) - return m.hexdigest() - - -def derive_hid(arg_names, *args, **kwargs): - if arg_names is not None: - if kwargs: - if isinstance(arg_names, list): - extra = ",".join( - [f"{arg_names}={kwargs.get(a, None)}" for a in arg_names] - ) - elif isinstance(arg_names, str): - option_a = kwargs.get(arg_names, None) - if option_a is None and args: - option_a = args[0] - extra = f"{arg_names}={option_a}" - else: - raise TypeError( - "arg_names should be str or list of str," - f" {type(arg_names)} provided" - ) - else: - extra = f"{args[0]}" - else: - extra = hash_args(*args, **kwargs) - if len(extra) > 20: - extra = extra[:8] - extra = f"({extra})" - return extra - - -def simple_timeit(foo): - @functools.wraps(foo) - def wrapper(*args, **kwargs): - with Timer() as timer: - r = foo(*args, **kwargs) - if isinstance(r, Return): - r.elapsed = timer.elapsed - else: - r = SimpleReturn(ret=r, elapsed=timer.elapsed) - return r - - return wrapper - - -def simple_secureit(foo): - @functools.wraps(foo) - def wrapper(*args, **kwargs): - try: - r = foo(*args, **kwargs) - if isinstance(r, Return): - r.success = True - else: - r = SimpleReturn(ret=r, success=True) - except Exception as e: - r = SimpleReturn(ret=None, success=False, exception=e) - return r - - return wrapper - - -def timeit(foo, arg_name=None): - @functools.wraps(foo) - def wrapper(*args, **kwargs): - with Timer() as timer: - r = foo(*args, **kwargs) - - extra_str = derive_hid(arg_name, *args, **kwargs) - hkey = foo.__name__ + f"<{extra_str}>" - if isinstance(r, Return): - r.elapsed = timer.elapsed - if r.hkey is None: - r.hkey = hkey - else: - r = Return(ret=r, hkey=hkey, elapsed=timer.elapsed) - return r - - return wrapper - - -def secureit(foo, arg_name=None): - @functools.wraps(foo) - def wrapper(*args, **kwargs): - extra_str = derive_hid(arg_name, *args, **kwargs) - hkey = foo.__name__ + f"<{extra_str}>" - try: - r = foo(*args, **kwargs) - if isinstance(r, Return): - r.success = True - if r.hkey is None: - r.hkey = hkey - else: - r = Return(ret=r, hkey=hkey, success=True) - except Exception as e: - r = Return(ret=None, hkey=hkey, success=False, exception=e) - return r - - return wrapper - - -def profile(_foo=None, _argnames=None): - def wrapper(foo): - @functools.wraps(foo) - def decorate_with_timing(*args, **kwargs): - _profiler = kwargs.get("_profiler", None) - if _argnames is not None and not isinstance(_argnames, str): - raise TypeError( - f"_arg_name type should be str, got {type(_argnames)} instead" - ) - if _profiler is not None: - with Timer() as timer: - r = foo(*args, **kwargs) - extra_str = derive_hid(_argnames, *args, **kwargs) - hkey = foo.__name__ + f"{extra_str}" - _profiler.add_metric(hkey=hkey, value=timer.elapsed) - else: - r = foo(*args, **kwargs) - return r - - return decorate_with_timing - - if _foo is None: - return wrapper - else: - return wrapper(_foo) diff --git a/suthing/environ.py b/suthing/environ.py new file mode 100644 index 0000000..2d2ecd4 --- /dev/null +++ b/suthing/environ.py @@ -0,0 +1,62 @@ +"""Environment variables: boolean flags and ``.env`` files.""" + +from __future__ import annotations + +import os +import pathlib + +from suthing.fs import PathLike + +_TRUE = frozenset({"1", "true", "t", "yes", "y", "on"}) +_FALSE = frozenset({"0", "false", "f", "no", "n", "off"}) + + +def env_flag(name: str, default: bool = False) -> bool: + """Read environment variable *name* as a boolean. + + ``1/true/t/yes/y/on`` are true and ``0/false/f/no/n/off`` are false, + ignoring case and surrounding whitespace. Unset or empty gives *default*. + + Args: + name: Variable name. + default: Value when the variable is unset or empty. + + Returns: + The flag value. + + Raises: + ValueError: For any other value, so a typo does not silently read as + *default*. + """ + raw = os.environ.get(name, "").strip().lower() + if not raw: + return default + if raw in _TRUE: + return True + if raw in _FALSE: + return False + raise ValueError(f"environment variable {name}={raw!r} is not a boolean") + + +def load_env(path: PathLike, *, override: bool = False) -> dict[str, str]: + """Load a dotenv file into :data:`os.environ`. + + To read a dotenv file *without* touching the environment, use + ``FileHandle.load(path)``, which returns its values as a dict. + + Args: + path: Dotenv file. + override: Replace variables that are already set. By default the + existing environment wins. + + Returns: + The variables the file defines (with values ``None`` in the file + dropped). + """ + from dotenv import dotenv_values, load_dotenv + + p = pathlib.Path(path).expanduser() + if not p.is_file(): + raise FileNotFoundError(str(p)) + load_dotenv(p, override=override) + return {k: v for k, v in dotenv_values(p).items() if v is not None} diff --git a/suthing/file_handle.py b/suthing/file_handle.py index ad88616..d5e9b1d 100644 --- a/suthing/file_handle.py +++ b/suthing/file_handle.py @@ -1,253 +1,354 @@ -"""Utilities for handling different file types and formats. +"""Read and write files whose format is inferred from the file name. -This module provides a flexible interface for reading and writing various file formats -including YAML, JSON, CSV, pickle and others. Supports both regular and gzipped files. +Supported formats are listed in :class:`FileType`; the extension → format map +is :data:`EXTENSIONS`. Any of them can be compressed with ``.gz``, ``.bz2``, +``.xz`` or ``.zst`` (see :mod:`suthing.fs`). """ -import gzip +from __future__ import annotations + import io import json -import logging import pathlib import pickle -import pkgutil +from collections.abc import Iterator, Mapping from enum import Enum +from importlib import resources +from typing import IO, Any -import pandas as pd import yaml -from dotenv import load_dotenv - -logger = logging.getLogger(__name__) +from yaml.representer import RepresenterError - -def suffixes(fp: str | pathlib.Path): - """Extract file suffixes from a path. - - Args: - fp: File path as string or Path object - - Returns: - List of suffix strings - """ - if isinstance(fp, str): - fp = pathlib.Path(fp) - suffixes = fp.suffixes - if not suffixes and "." in fp.stem: - return [fp.stem] - return fp.suffixes +from suthing.fs import ( + PathLike, + atomic_open, + open_compressed, + split_compression, + wrap_compressed, +) +from suthing.jsonable import to_jsonable +from suthing.jsonl import iter_jsonl_stream, write_jsonl_stream class FileType(str, Enum): - """Supported file types for reading and writing.""" + """Formats :class:`FileHandle` reads and writes.""" YAML = "yaml" JSON = "json" - JSONLD = "jsonld" + JSONL = "jsonl" PICKLE = "pkl" CSV = "csv" + TSV = "tsv" TXT = "txt" ENV = "env" +#: File extension → format. ``.jsonld`` is JSON-LD, i.e. a JSON document. +EXTENSIONS: dict[str, FileType] = { + ".yaml": FileType.YAML, + ".yml": FileType.YAML, + ".json": FileType.JSON, + ".jsonld": FileType.JSON, + ".jsonl": FileType.JSONL, + ".ndjson": FileType.JSONL, + ".pkl": FileType.PICKLE, + ".pickle": FileType.PICKLE, + ".csv": FileType.CSV, + ".tsv": FileType.TSV, + ".txt": FileType.TXT, + ".md": FileType.TXT, + ".env": FileType.ENV, +} + + +def detect_format(path: PathLike) -> tuple[FileType | None, str | None]: + """Infer the format and compression of *path* from its name. + + Args: + path: File path or name. + + Returns: + ``(format, compression)``; ``format`` is ``None`` for an unknown + extension and ``compression`` is ``None`` for an uncompressed file. + """ + fmt, compression = split_compression(path) + return EXTENSIONS.get(fmt), compression + + +def _resolve(path: PathLike, how: FileType | str | None) -> tuple[FileType, str | None]: + detected, compression = detect_format(path) + if how is not None: + return FileType(how), compression + if detected is None: + raise ValueError( + f"cannot infer the format of {str(path)!r} from its extension;" + f" pass how=FileType.. Known extensions: {', '.join(EXTENSIONS)}" + ) + return detected, compression + + +class _Dumper(yaml.SafeDumper): + """Safe YAML dumper that writes tuples as lists and coerces other types.""" + + +def _represent_other(dumper: yaml.SafeDumper, data: Any) -> yaml.Node: + converted = to_jsonable(data) + if type(converted) is type(data): + raise RepresenterError( + f"cannot represent an object of type {type(data).__name__}" + ) + return dumper.represent_data(converted) + + +_Dumper.add_representer(tuple, _Dumper.represent_list) +_Dumper.add_multi_representer(object, _represent_other) + + +def _no_kwargs(how: FileType, kwargs: Mapping[str, Any]) -> None: + if kwargs: + raise TypeError( + f"unexpected keyword arguments for {how.name}: {', '.join(kwargs)}" + ) + + +def _read(stream: IO[bytes], how: FileType, **kwargs: Any) -> Any: + if how in (FileType.CSV, FileType.TSV): + import pandas as pd + + if how is FileType.TSV: + kwargs.setdefault("sep", "\t") + return pd.read_csv(stream, **kwargs) + _no_kwargs(how, kwargs) + if how is FileType.PICKLE: + return pickle.load(stream) + if how is FileType.YAML: + return yaml.safe_load(stream) + if how is FileType.JSON: + return json.load(stream) + if how is FileType.JSONL: + return list(iter_jsonl_stream(stream)) + if how is FileType.TXT: + return stream.read().decode("utf-8") + if how is FileType.ENV: + from dotenv import dotenv_values + + text = io.StringIO(stream.read().decode("utf-8")) + return {k: v for k, v in dotenv_values(stream=text).items() if v is not None} + raise AssertionError(f"unhandled format {how}") # pragma: no cover + + +def _env_value(value: Any) -> str: + text = str(value) + if text and all(c.isalnum() or c in "._-/:@+,=" for c in text): + return text + escaped = text.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n") + return f'"{escaped}"' + + +def _encode(item: Any, how: FileType, **kwargs: Any) -> bytes: + if how in (FileType.CSV, FileType.TSV): + import pandas as pd + + if not isinstance(item, (pd.DataFrame, pd.Series)): + raise TypeError( + f"{how.name} output needs a pandas DataFrame or Series," + f" got {type(item).__name__}" + ) + if how is FileType.TSV: + kwargs.setdefault("sep", "\t") + return item.to_csv(**kwargs).encode("utf-8") + _no_kwargs(how, kwargs) + if how is FileType.PICKLE: + return pickle.dumps(item, pickle.HIGHEST_PROTOCOL) + if how is FileType.YAML: + text = yaml.dump(item, Dumper=_Dumper, sort_keys=False, allow_unicode=True) + return text.encode("utf-8") + if how is FileType.JSON: + text = json.dumps(item, indent=2, ensure_ascii=False, default=to_jsonable) + return (text + "\n").encode("utf-8") + if how is FileType.TXT: + if not isinstance(item, str): + raise TypeError(f"TXT output needs a str, got {type(item).__name__}") + return item.encode("utf-8") + if how is FileType.ENV: + if not isinstance(item, Mapping): + raise TypeError(f"ENV output needs a mapping, got {type(item).__name__}") + lines = [f"{k}={_env_value(v)}\n" for k, v in item.items()] + return "".join(lines).encode("utf-8") + raise AssertionError(f"unhandled format {how}") # pragma: no cover + + class FileHandle: - """Main class for handling file operations across different formats.""" + """Format-aware file I/O: one call to read or write a whole file. + + The format comes from the extension (:data:`EXTENSIONS`) unless ``how`` is + given, and a compression suffix is handled transparently. An unknown + extension is an error rather than a guess. + + Formats read to / write from: + + - YAML, JSON (``.jsonld`` too): any YAML/JSON value. YAML loads with + ``safe_load``; values json cannot serialise go through + :func:`suthing.to_jsonable`. + - JSONL: a list of values, one per line. + - CSV, TSV: a pandas ``DataFrame`` (extra keyword arguments go to + ``read_csv`` / ``to_csv``). + - TXT: a ``str``. + - ENV: a ``dict[str, str]``; loading does *not* touch ``os.environ`` + (use :func:`suthing.load_env` for that). + - PICKLE: any picklable object. Only unpickle files you trust. + + Example: + >>> data = FileHandle.load("config.yaml") + >>> FileHandle.dump(data, "out/config.json.gz", mkdir=True) + """ @classmethod - def _find_mode(cls, lemma: str): - """Determine file type from file extension. + def load( + cls, path: PathLike, *, how: FileType | str | None = None, **kwargs: Any + ) -> Any: + """Read a file from disk. Args: - lemma: File extension string + path: File to read; ``~`` is expanded. + how: Format override; by default inferred from the extension. + **kwargs: Passed to ``pandas.read_csv`` for CSV/TSV. Any other + format rejects extra arguments. Returns: - FileType enum value corresponding to the extension + The parsed contents (see the class docstring for types). + + Raises: + ValueError: If the format cannot be inferred. + TypeError: For keyword arguments the format does not take. """ - if lemma in [".yml", ".yaml"]: - return FileType.YAML - elif lemma == ".json": - return FileType.JSON - elif lemma == ".jsonld": - return FileType.JSONLD - elif lemma in [".pkl", ".pickle"]: - return FileType.PICKLE - elif lemma in [".csv"]: - return FileType.CSV - elif lemma in [".env"]: - return FileType.ENV - else: - return FileType.TXT + fmt, _ = _resolve(path, how) + with open_compressed(path, "rb") as stream: + return _read(stream, fmt, **kwargs) @classmethod - def _dump_pointer(cls, item, p, how: FileType, bytes_: bool = True) -> None: - """Write data to file pointer in specified format. + def load_resource( + cls, + package: str, + name: str, + *, + how: FileType | str | None = None, + **kwargs: Any, + ) -> Any: + """Read a data file shipped inside an importable package. Args: - item: Data to write - p: File pointer - how: FileType indicating format to write in - bytes_: Whether to write in bytes mode + package: Dotted package name, e.g. ``"mypkg.data"``. + name: File name relative to the package; may contain ``/``. + how: Format override; by default inferred from the extension. + **kwargs: As for :meth:`load`. + + Returns: + The parsed contents. """ - if how == FileType.PICKLE: - pickle.dump(item, p, pickle.HIGHEST_PROTOCOL) - elif how == FileType.YAML: - yc = yaml.dump(item) - if bytes_: - yc = yc.encode("utf-8") # type: ignore - p.write(yc) - elif how == FileType.JSON: - jc = json.dumps(item, indent=2) + "\n" - if bytes_: - jc = jc.encode("utf-8") # type: ignore - p.write(jc) - elif how == FileType.JSONLD: - for subitem in item: - jc = json.dumps(subitem) + "\n" - if bytes_: - jc = jc.encode("utf-8") # type: ignore - p.write(jc) - elif how == FileType.CSV and ( - isinstance(item, pd.DataFrame) or isinstance(item, pd.Series) - ): - r = item.to_csv() - if bytes_: - r = r.encode("utf-8") # type: ignore - p.write(r) - elif how == FileType.TXT: - p.write(str(item)) + fmt, compression = _resolve(name, how) + data = resources.files(package).joinpath(name).read_bytes() + with io.BytesIO(data) as raw: + if compression is None: + return _read(raw, fmt, **kwargs) + with wrap_compressed(raw, compression, "rb") as stream: + return _read(stream, fmt, **kwargs) @classmethod - def _open_pointer(cls, p: io.BytesIO | gzip.GzipFile, how: FileType, **kwargs): - """Read data from file pointer in specified format. + def iter( + cls, + path: PathLike, + *, + how: FileType | str | None = None, + chunksize: int = 10_000, + **kwargs: Any, + ) -> Iterator[Any]: + """Stream a file instead of loading it whole. Args: - p: File pointer - how: FileType indicating format to read - **kwargs: Additional arguments passed to readers + path: File to read. + how: Format override; by default inferred from the extension. + chunksize: Rows per ``DataFrame`` chunk for CSV/TSV. + **kwargs: Passed to ``pandas.read_csv`` for CSV/TSV, and to + :func:`suthing.jsonl.iter_jsonl_stream` (``strict``, + ``require_object``) for JSONL. - Returns: - Data read from file in appropriate format + Yields: + JSONL: one value per line. CSV/TSV: ``DataFrame`` chunks. + TXT: lines without their trailing newline. Raises: - ValueError: If trying to read gzipped env files + ValueError: For a format that cannot be streamed. """ - if how == FileType.PICKLE: - r = pickle.load(p) - elif how == FileType.YAML: - r = yaml.load(p, Loader=yaml.FullLoader) - elif how == FileType.JSON: - r = json.load(p) - elif how == FileType.JSONLD: - r = [json.loads(s.decode()) for s in p.readlines()] - elif how == FileType.CSV: - r = pd.read_csv(p, **kwargs) # type: ignore[arg-type] - elif how == FileType.TXT: - r = p.read().decode() - elif how == FileType.ENV: - if isinstance(p, io.BytesIO): - config = io.StringIO(p.getvalue().decode("UTF-8")) - r = load_dotenv(stream=config) + fmt, _ = _resolve(path, how) + if fmt not in (FileType.JSONL, FileType.CSV, FileType.TSV, FileType.TXT): + raise ValueError(f"{fmt.name} files cannot be streamed; use load()") + with open_compressed(path, "rb") as stream: + if fmt is FileType.JSONL: + yield from iter_jsonl_stream(stream, **kwargs) + elif fmt is FileType.TXT: + _no_kwargs(fmt, kwargs) + for raw in stream: + yield raw.decode("utf-8").rstrip("\r\n") else: - raise ValueError("Will not read gzipped env files") - else: - r = dict() - return r + import pandas as pd + + if fmt is FileType.TSV: + kwargs.setdefault("sep", "\t") + with pd.read_csv(stream, chunksize=chunksize, **kwargs) as reader: + yield from reader @classmethod - def load( + def dump( cls, - ppath: str | pathlib.Path | None = None, - pname: str | None = None, - how: FileType = FileType.YAML, - **kwargs, - ): - """ - - :param ppath: - :param pname: - :param how: - :param kwargs: - :return: - """ - - compression = kwargs.pop("compression", None) - fpath: str | pathlib.Path | None = kwargs.pop("fpath", None) - - # assume loading from a package - if pname is not None: - lemmas = suffixes(pname) - if lemmas[-1] == ".gz": - compression = "gz" - how_ = cls._find_mode(lemmas[-2]) - else: - how_ = cls._find_mode(lemmas[-1]) - if how_: - how = how_ - if ppath is not None and isinstance(ppath, str): - bytes_ = pkgutil.get_data(ppath, pname) - else: - raise ValueError( - "package name provided, package path (as a string) needed" - ) + item: Any, + path: PathLike, + *, + how: FileType | str | None = None, + atomic: bool = True, + mkdir: bool = False, + **kwargs: Any, + ) -> pathlib.Path: + """Write *item* to a file, compressing by suffix. - # interpret as filesystem load - else: - if fpath is None: - if ppath is not None: - fpath = ppath - else: - raise ValueError("either fpath or ppath should be provided") - fpath = pathlib.Path(fpath).expanduser().as_posix() - lemmas = suffixes(fpath) - if lemmas[-1] == ".gz": - compression = "gz" - how_ = cls._find_mode(lemmas[-2]) - else: - how_ = cls._find_mode(lemmas[-1]) - if how_: - how = how_ - with open(fpath, "rb") as fp: - bytes_ = fp.read() - - if bytes_ is None: - raise ValueError("None received as Bytes") - - if compression == "gz": - with gzip.GzipFile(fileobj=io.BytesIO(bytes_), mode="r") as p: - r = cls._open_pointer(p, how, **kwargs) - else: - with io.BytesIO(bytes_) as p: - r = cls._open_pointer(p, how, **kwargs) - return r + Args: + item: Value to write (see the class docstring for accepted types). + path: Destination; ``~`` is expanded. + how: Format override; by default inferred from the extension. + atomic: Replace *path* in one step, so readers never see a + partial file (see :func:`suthing.fs.atomic_open`). + mkdir: Create missing parent directories first. + **kwargs: Passed to ``DataFrame.to_csv`` for CSV/TSV. Any other + format rejects extra arguments. - @classmethod - def dump(cls, item, path: str | pathlib.Path, how: FileType = FileType.YAML): - """ + Returns: + The path written. - :param item: - :param path: if path ends with ".gz" the output will be gzip compressed - :param how: - :return: + Raises: + ValueError: If the format cannot be inferred. + TypeError: If *item* does not fit the format, or for keyword + arguments the format does not take. """ - - lemmas = suffixes(path) - path = pathlib.Path(path).expanduser().as_posix() - if lemmas[-1] == ".gz": - compression = "gz" - how_ = cls._find_mode(lemmas[-2]) + fmt, compression = _resolve(path, how) + dest = pathlib.Path(path).expanduser() + if mkdir: + dest.parent.mkdir(parents=True, exist_ok=True) + if fmt is FileType.JSONL: + _no_kwargs(fmt, kwargs) + write = lambda stream: write_jsonl_stream(item, stream) else: - compression = None - how_ = cls._find_mode(lemmas[-1]) - if how_: - how = how_ - if how == FileType.PICKLE: - mode = "wb" - else: - mode = "w" - if compression == "gz": - if not path.endswith(".gz"): - path += ".gz" - with gzip.GzipFile(path, mode=mode) as p: - cls._dump_pointer(item, p, how) - else: - with open(path, mode=mode) as p: - cls._dump_pointer(item, p, how, bytes_=False) + # Encode first: a type error must not leave a truncated file behind. + payload = _encode(item, fmt, **kwargs) + write = lambda stream: stream.write(payload) + + if not atomic: + with open_compressed(dest, "wb") as stream: + write(stream) + return dest + with atomic_open(dest) as raw: + if compression is None: + write(raw) + else: + with wrap_compressed(raw, compression, "wb") as stream: + write(stream) + return dest diff --git a/suthing/fs.py b/suthing/fs.py new file mode 100644 index 0000000..f3532a9 --- /dev/null +++ b/suthing/fs.py @@ -0,0 +1,233 @@ +"""File-system primitives: path expansion, transparent compression, atomic writes. + +Compression is chosen from the file name: ``.gz``, ``.bz2`` and ``.xz`` use the +standard library; ``.zst`` needs the optional ``zstandard`` package +(``pip install suthing[zstd]``). +""" + +from __future__ import annotations + +import bz2 +import contextlib +import gzip +import io +import lzma +import os +import pathlib +import tempfile +from collections.abc import Callable, Iterator +from importlib import import_module +from typing import IO, Any, cast + +PathLike = str | os.PathLike[str] + +Opener = Callable[[Any, str], IO[bytes]] + + +def _gzip_open(fileobj: Any, mode: str) -> IO[bytes]: + return cast(IO[bytes], gzip.GzipFile(fileobj=fileobj, mode=mode)) + + +def _bz2_open(fileobj: Any, mode: str) -> IO[bytes]: + # BZ2File types its mode as a literal; open_compressed has validated it. + return cast(IO[bytes], bz2.BZ2File(fileobj, cast(Any, mode))) + + +def _xz_open(fileobj: Any, mode: str) -> IO[bytes]: + return cast(IO[bytes], lzma.LZMAFile(fileobj, mode)) + + +def _zstd_open(fileobj: Any, mode: str) -> IO[bytes]: + try: + zstandard = import_module("zstandard") + except ImportError as e: # pragma: no cover - depends on the environment + raise ImportError( + ".zst files need the optional 'zstandard' package:" + " pip install 'suthing[zstd]'" + ) from e + stream = zstandard.open(fileobj, mode) + if mode == "rb": + # The zstandard reader has no readline(); buffering it restores line + # iteration, which the JSONL and TXT readers rely on. + return cast(IO[bytes], io.BufferedReader(stream)) + return stream + + +#: Compression suffix → function opening a binary stream over a file object. +COMPRESSORS: dict[str, Opener] = { + ".gz": _gzip_open, + ".bz2": _bz2_open, + ".xz": _xz_open, + ".zst": _zstd_open, +} + + +def expand_path(path: PathLike) -> pathlib.Path: + """Expand ``~`` and make *path* absolute, resolving symlinks. + + Args: + path: Path as a string or path-like object. + + Returns: + The resolved absolute path. + """ + return pathlib.Path(path).expanduser().resolve() + + +def split_compression(path: PathLike) -> tuple[str, str | None]: + """Split a file name into its format suffix and compression suffix. + + The format suffix is the last extension once any compression suffix is + removed. A dotfile such as ``.env`` counts as its own suffix. + + Args: + path: File path or name. + + Returns: + ``(format_suffix, compression_suffix)``, lower-cased; ``format_suffix`` + is ``""`` when there is none and ``compression_suffix`` is ``None`` for + an uncompressed name. + + Example: + >>> split_compression("data/rows.jsonl.gz") + ('.jsonl', '.gz') + >>> split_compression("config/.env") + ('.env', None) + """ + name = pathlib.PurePath(path).name.lower() + compression = None + for suffix in COMPRESSORS: + if name.endswith(suffix) and len(name) > len(suffix): + compression = suffix + name = name[: -len(suffix)] + break + fmt = pathlib.PurePath(name).suffix + if not fmt and name.startswith(".") and name.count(".") == 1: + fmt = name + return fmt, compression + + +@contextlib.contextmanager +def open_compressed(path: PathLike, mode: str = "rb") -> Iterator[IO[bytes]]: + """Open *path* as a binary stream, decompressing or compressing by suffix. + + Args: + path: File to open; ``~`` is expanded. + mode: One of ``"rb"``, ``"wb"``, ``"ab"``. + + Yields: + A binary file object. + + Raises: + ValueError: If *mode* is not a binary read/write/append mode. + """ + if mode not in ("rb", "wb", "ab"): + raise ValueError(f"mode must be 'rb', 'wb' or 'ab', got {mode!r}") + p = pathlib.Path(path).expanduser() + _, compression = split_compression(p) + with open(p, mode) as raw: + if compression is None: + yield raw + else: + with wrap_compressed(raw, compression, mode) as stream: + yield stream + + +def wrap_compressed(fileobj: IO[bytes], compression: str, mode: str) -> IO[bytes]: + """Wrap an open binary file object in a (de)compressing stream. + + Args: + fileobj: Underlying binary file object. + compression: A key of :data:`COMPRESSORS`, e.g. ``".gz"``. + mode: ``"rb"``, ``"wb"`` or ``"ab"``. + + Returns: + A binary stream over *fileobj*; close it before closing *fileobj* so + compressed trailers get flushed. + """ + return COMPRESSORS[compression](fileobj, mode) + + +def _default_file_mode() -> int: + # os.umask is the only portable way to read the umask: set it and restore it. + umask = os.umask(0) + os.umask(umask) + return 0o666 & ~umask + + +@contextlib.contextmanager +def atomic_open( + path: PathLike, *, mkdir: bool = False, durable: bool = False +) -> Iterator[IO[bytes]]: + """Open a binary stream whose contents replace *path* only on success. + + Data goes to a temporary file in the same directory, which is moved over + *path* with :func:`os.replace` when the block exits without an exception. + Readers therefore see either the old file or the complete new one, and + concurrent writers never interleave (the last one to finish wins). On an + exception the temporary file is removed and *path* is left untouched. + + The new file keeps the permissions of the file it replaces, or gets the + usual ``0o666 & ~umask`` if *path* did not exist. + + Args: + path: Destination file; ``~`` is expanded. + mkdir: Create missing parent directories first. + durable: ``fsync`` the data before the rename, so the new contents + survive a crash, not just a concurrent reader. + + Yields: + A binary file object to write to. + """ + dest = pathlib.Path(path).expanduser() + if mkdir: + dest.parent.mkdir(parents=True, exist_ok=True) + try: + mode = dest.stat().st_mode & 0o7777 + except FileNotFoundError: + mode = _default_file_mode() + fd, tmp_name = tempfile.mkstemp( + dir=dest.parent, prefix=f".{dest.name}.", suffix=".tmp" + ) + tmp = pathlib.Path(tmp_name) + try: + with os.fdopen(fd, "wb") as f: + yield f + f.flush() + if durable: + os.fsync(f.fileno()) + os.chmod(tmp, mode) + os.replace(tmp, dest) + except BaseException: + tmp.unlink(missing_ok=True) + raise + + +def atomic_write( + path: PathLike, + data: bytes | str, + *, + encoding: str = "utf-8", + mkdir: bool = False, + durable: bool = False, +) -> pathlib.Path: + """Write *data* to *path* atomically (see :func:`atomic_open`). + + The data is written as-is: a ``.gz`` name is not compressed. Use + :meth:`suthing.FileHandle.dump` for format- and compression-aware writes. + + Args: + path: Destination file. + data: Bytes, or text encoded with *encoding*. + encoding: Encoding for text data. + mkdir: Create missing parent directories first. + durable: ``fsync`` before the rename. + + Returns: + The destination path. + """ + payload = data.encode(encoding) if isinstance(data, str) else data + dest = pathlib.Path(path).expanduser() + with atomic_open(dest, mkdir=mkdir, durable=durable) as f: + f.write(payload) + return dest diff --git a/suthing/hashing.py b/suthing/hashing.py new file mode 100644 index 0000000..26179ce --- /dev/null +++ b/suthing/hashing.py @@ -0,0 +1,139 @@ +"""Stable content hashes: of JSON-like values, text, bytes, files and directory trees. + +Every function takes an ``algorithm`` name understood by :func:`hashlib.new` +(``"sha256"`` by default) and returns a hex digest. +""" + +from __future__ import annotations + +import hashlib +import json +import pathlib +from collections.abc import Callable +from typing import Any + +from suthing.fs import PathLike + +_CHUNK = 1 << 20 + + +def _truncate(digest: str, length: int | None) -> str: + if length is None: + return digest + if length < 1: + raise ValueError(f"length must be positive, got {length}") + return digest[:length] + + +def canonical_json(obj: Any, *, default: Callable[[Any], Any] | None = None) -> str: + """Render *obj* as canonical JSON: sorted keys, no whitespace, ASCII-escaped. + + Equal values give identical text regardless of dict insertion order, which + makes the result suitable for hashing and for use as a cache key. + + Args: + obj: JSON-serialisable value. + default: Hook for values json cannot serialise (e.g. + :func:`suthing.to_jsonable`). Without it such values raise + ``TypeError``, which keeps an accidental ``repr`` out of a hash. + + Returns: + The canonical JSON text. + """ + return json.dumps(obj, sort_keys=True, separators=(",", ":"), default=default) + + +def bytes_hash( + data: bytes, *, length: int | None = None, algorithm: str = "sha256" +) -> str: + """Hex digest of *data*, optionally truncated to *length* characters.""" + return _truncate(hashlib.new(algorithm, data).hexdigest(), length) + + +def text_hash( + text: str, + *, + length: int | None = None, + algorithm: str = "sha256", + encoding: str = "utf-8", +) -> str: + """Hex digest of *text* encoded with *encoding*, optionally truncated.""" + return bytes_hash(text.encode(encoding), length=length, algorithm=algorithm) + + +def stable_hash( + obj: Any, + *, + length: int | None = None, + algorithm: str = "sha256", + default: Callable[[Any], Any] | None = None, +) -> str: + """Hex digest of :func:`canonical_json` of *obj*. + + Equal to ``hashlib.sha256(json.dumps(obj, sort_keys=True, + separators=(",", ":")).encode("utf-8")).hexdigest()``, so it can replace + that expression without changing any stored hash. + + Args: + obj: JSON-serialisable value. + length: Keep only the first *length* hex characters. + algorithm: :mod:`hashlib` algorithm name. + default: Passed to :func:`canonical_json`. + + Returns: + The hex digest. + """ + return text_hash( + canonical_json(obj, default=default), length=length, algorithm=algorithm + ) + + +def file_hash( + path: PathLike, *, length: int | None = None, algorithm: str = "sha256" +) -> str: + """Hex digest of a file's raw bytes, read in chunks.""" + h = hashlib.new(algorithm) + with open(pathlib.Path(path).expanduser(), "rb") as f: + while chunk := f.read(_CHUNK): + h.update(chunk) + return _truncate(h.hexdigest(), length) + + +def tree_hash( + root: PathLike, + *, + pattern: str = "*", + length: int | None = None, + algorithm: str = "sha256", +) -> str: + """Hex digest of every file under *root* and its path relative to *root*. + + Files are visited in sorted order of their POSIX relative paths, so the + result does not depend on the file system's listing order. Renaming or + moving a file changes the hash; so does changing its contents. Directories + and symlinks to directories contribute only through the files they hold. + + Args: + root: Directory to hash. + pattern: Glob matched recursively against file names (``rglob``). + length: Keep only the first *length* hex characters. + algorithm: :mod:`hashlib` algorithm name. + + Returns: + The hex digest. + + Raises: + NotADirectoryError: If *root* is not a directory. + """ + base = pathlib.Path(root).expanduser() + if not base.is_dir(): + raise NotADirectoryError(str(base)) + files = sorted( + (p.relative_to(base).as_posix(), p) for p in base.rglob(pattern) if p.is_file() + ) + h = hashlib.new(algorithm) + for rel, p in files: + h.update(rel.encode("utf-8")) + h.update(b"\0") + h.update(bytes.fromhex(file_hash(p, algorithm=algorithm))) + return _truncate(h.hexdigest(), length) diff --git a/suthing/iterx.py b/suthing/iterx.py new file mode 100644 index 0000000..4f37b7e --- /dev/null +++ b/suthing/iterx.py @@ -0,0 +1,37 @@ +"""Iteration helpers.""" + +from __future__ import annotations + +from collections.abc import Iterable, Iterator +from itertools import islice +from typing import TypeVar + +T = TypeVar("T") + + +def batched(iterable: Iterable[T], n: int) -> Iterator[list[T]]: + """Split *iterable* into consecutive lists of *n* items; the last may be shorter. + + Works on any iterable, including generators, and consumes it lazily. + Unlike :func:`itertools.batched` (Python 3.12+) it yields lists, which + callers can index, extend or pass straight to bulk-insert APIs. + + Args: + iterable: Items to split. + n: Batch size. + + Yields: + Lists of at most *n* items. + + Raises: + ValueError: If *n* is less than 1. + + Example: + >>> list(batched(range(5), 2)) + [[0, 1], [2, 3], [4]] + """ + if n < 1: + raise ValueError(f"batch size must be at least 1, got {n}") + it = iter(iterable) + while batch := list(islice(it, n)): + yield batch diff --git a/suthing/jsonable.py b/suthing/jsonable.py new file mode 100644 index 0000000..1e993f9 --- /dev/null +++ b/suthing/jsonable.py @@ -0,0 +1,93 @@ +"""Coerce Python values into what :func:`json.dumps` accepts.""" + +from __future__ import annotations + +import dataclasses +import math +import pathlib +import sys +from collections.abc import Mapping +from datetime import date, datetime, time, timedelta +from decimal import Decimal +from enum import Enum +from typing import Any +from uuid import UUID + + +def to_jsonable(obj: Any, *, nan: Any = None) -> Any: + """Recursively convert *obj* into JSON-serialisable built-in types. + + Conversions: + + - ``dict``/mapping → ``dict`` with ``str`` keys; ``list``, ``tuple``, + ``set`` and ``frozenset`` → ``list`` (sets are sorted when they can be, + so the output is deterministic) + - ``Enum`` → its ``value``; subclasses of ``str``/``int``/``float`` → the + plain built-in + - ``datetime``/``date``/``time`` → ISO 8601 text; ``timedelta`` → seconds + - ``Decimal`` → ``float``; ``UUID`` and paths → ``str`` + - dataclass instances → ``dict`` of their fields + - numpy scalars and arrays → Python scalars and nested lists (numpy is + only consulted when it is already imported) + - non-finite floats (``nan``, ``inf``) → *nan*, since JSON has no such value + + Can also be passed as ``default=`` to :func:`json.dumps`, where it is called + only for values json cannot serialise itself. + + Args: + obj: Value to convert. + nan: Replacement for non-finite floats. + + Returns: + A structure of ``dict``, ``list``, ``str``, ``int``, ``float``, + ``bool`` and ``None``. + + Raises: + TypeError: For a value with no known conversion. + """ + if obj is None or isinstance(obj, bool): + return obj + if isinstance(obj, Enum): + return to_jsonable(obj.value, nan=nan) + if isinstance(obj, str): + return str(obj) + if isinstance(obj, int): + return int(obj) + if isinstance(obj, float): + return float(obj) if math.isfinite(obj) else nan + if isinstance(obj, Mapping): + return {_key(k): to_jsonable(v, nan=nan) for k, v in obj.items()} + if isinstance(obj, (list, tuple)): + return [to_jsonable(v, nan=nan) for v in obj] + if isinstance(obj, (set, frozenset)): + items = [to_jsonable(v, nan=nan) for v in obj] + try: + return sorted(items) + except TypeError: + return items + if isinstance(obj, (datetime, date, time)): + return obj.isoformat() + if isinstance(obj, timedelta): + return obj.total_seconds() + if isinstance(obj, Decimal): + return to_jsonable(float(obj), nan=nan) + if isinstance(obj, (UUID, pathlib.PurePath)): + return str(obj) + if dataclasses.is_dataclass(obj) and not isinstance(obj, type): + return { + f.name: to_jsonable(getattr(obj, f.name), nan=nan) + for f in dataclasses.fields(obj) + } + np = sys.modules.get("numpy") + if np is not None: + if isinstance(obj, np.ndarray): + return to_jsonable(obj.tolist(), nan=nan) + if isinstance(obj, np.generic): + return to_jsonable(obj.item(), nan=nan) + raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable") + + +def _key(key: Any) -> str: + if isinstance(key, Enum): + key = key.value + return key if isinstance(key, str) else str(key) diff --git a/suthing/jsonl.py b/suthing/jsonl.py new file mode 100644 index 0000000..5c14775 --- /dev/null +++ b/suthing/jsonl.py @@ -0,0 +1,180 @@ +"""JSON Lines (``.jsonl`` / ``.ndjson``): one JSON value per line. + +All functions accept compressed files (``.jsonl.gz`` etc., see :mod:`suthing.fs`). +Blank lines are skipped on read. +""" + +from __future__ import annotations + +import json +import logging +import pathlib +from collections.abc import Iterable, Iterator +from typing import IO, Any, NamedTuple + +from suthing.fs import ( + PathLike, + atomic_open, + open_compressed, + split_compression, + wrap_compressed, +) +from suthing.jsonable import to_jsonable + +logger = logging.getLogger(__name__) + + +class JsonlError(NamedTuple): + """A line that could not be read. + + Attributes: + line: 1-based line number. + message: What was wrong with it. + """ + + line: int + message: str + + def __str__(self) -> str: + return f"line {self.line}: {self.message}" + + +def _parse( + stream: IO[bytes], *, require_object: bool +) -> Iterator[tuple[int, Any, str | None]]: + """Yield ``(line_no, value, error)`` for each non-blank line.""" + for line_no, raw in enumerate(stream, start=1): + text = raw.strip() + if not text: + continue + try: + value = json.loads(text) + except json.JSONDecodeError as e: + yield line_no, None, f"not valid JSON ({e.msg})" + continue + if require_object and not isinstance(value, dict): + yield line_no, None, f"expected a JSON object, got {type(value).__name__}" + continue + yield line_no, value, None + + +def iter_jsonl_stream( + stream: IO[bytes], *, strict: bool = True, require_object: bool = False +) -> Iterator[Any]: + """Parse JSON Lines from an open binary stream; see :func:`iter_jsonl`.""" + for line_no, value, error in _parse(stream, require_object=require_object): + if error is None: + yield value + elif strict: + raise ValueError(f"line {line_no}: {error}") + else: + logger.warning("skipping line %d: %s", line_no, error) + + +def iter_jsonl( + path: PathLike, *, strict: bool = True, require_object: bool = False +) -> Iterator[Any]: + """Stream the values of a JSON Lines file without loading it whole. + + Args: + path: File to read; may be compressed. + strict: Raise on a bad line. When ``False``, bad lines are logged at + WARNING level and skipped. + require_object: Treat any value that is not a JSON object as a bad line. + + Yields: + One parsed value per non-blank line. + + Raises: + ValueError: On a bad line when *strict* is set; the message carries the + line number. + """ + with open_compressed(path, "rb") as stream: + yield from iter_jsonl_stream( + stream, strict=strict, require_object=require_object + ) + + +def read_jsonl( + path: PathLike, *, require_object: bool = False +) -> tuple[list[Any], list[JsonlError]]: + """Read a whole JSON Lines file, collecting bad lines instead of raising. + + Args: + path: File to read; may be compressed. + require_object: Treat any value that is not a JSON object as a bad line. + + Returns: + ``(rows, errors)``: the values that parsed, in file order, and one + :class:`JsonlError` per line that did not. + """ + rows: list[Any] = [] + errors: list[JsonlError] = [] + with open_compressed(path, "rb") as stream: + for line_no, value, error in _parse(stream, require_object=require_object): + if error is None: + rows.append(value) + else: + errors.append(JsonlError(line_no, error)) + return rows, errors + + +def encode_jsonl_row(row: Any) -> bytes: + """Encode one value as a UTF-8 JSON line, newline included. + + Non-ASCII text is written as-is and values json cannot serialise go through + :func:`suthing.to_jsonable`. + """ + return (json.dumps(row, ensure_ascii=False, default=to_jsonable) + "\n").encode( + "utf-8" + ) + + +def write_jsonl_stream(rows: Iterable[Any], stream: IO[bytes]) -> int: + """Write *rows* to an open binary stream; returns the number written.""" + if isinstance(rows, (str, bytes, dict)): + raise TypeError( + f"JSON Lines needs an iterable of rows, got {type(rows).__name__}" + ) + count = 0 + for row in rows: + stream.write(encode_jsonl_row(row)) + count += 1 + return count + + +def write_jsonl( + rows: Iterable[Any], + path: PathLike, + *, + append: bool = False, + atomic: bool = True, + mkdir: bool = False, +) -> int: + """Write *rows* as JSON Lines, compressing by suffix. + + Args: + rows: Values to write, one per line; any iterable, consumed once. + path: Destination file. + append: Add to the end of an existing file instead of replacing it. + A compressed file gains a new compressed member, which every + reader of that format accepts. + atomic: Replace *path* in one step (see :func:`suthing.fs.atomic_open`). + Ignored when appending. + mkdir: Create missing parent directories first. + + Returns: + The number of rows written. + """ + dest = pathlib.Path(path).expanduser() + if mkdir: + dest.parent.mkdir(parents=True, exist_ok=True) + if append or not atomic: + with open_compressed(dest, "ab" if append else "wb") as stream: + return write_jsonl_stream(rows, stream) + _, compression = split_compression(dest) + with atomic_open(dest) as raw: + if compression is None: + return write_jsonl_stream(rows, raw) + with wrap_compressed(raw, compression, "wb") as stream: + return write_jsonl_stream(rows, stream) diff --git a/suthing/log.py b/suthing/log.py new file mode 100644 index 0000000..a37cf09 --- /dev/null +++ b/suthing/log.py @@ -0,0 +1,73 @@ +"""One-call logging configuration for scripts and CLIs.""" + +from __future__ import annotations + +import logging +import logging.config +import pathlib +import sys +from typing import TextIO + +from suthing.fs import PathLike + +DEFAULT_FORMAT = "%(asctime)s %(levelname)s %(name)s: %(message)s" + + +def setup_logging( + level: int | str = "INFO", + *, + config: PathLike | None = None, + fmt: str = DEFAULT_FORMAT, + stream: TextIO | None = None, + force: bool = False, +) -> None: + """Configure the root logger, from a config file or with sensible defaults. + + With *config*, the file decides everything: ``.conf``/``.ini`` files go to + :func:`logging.config.fileConfig` and ``.yaml``/``.yml``/``.json`` files + to :func:`logging.config.dictConfig`. Loggers that already exist are kept + enabled in both cases. Without *config*, :func:`logging.basicConfig` is + called with *level*, *fmt* and *stream*. + + Args: + level: Root level, as a number or a name such as ``"DEBUG"``. + config: Optional logging config file. + fmt: Format string when no *config* is given. + stream: Output stream when no *config* is given (default ``stderr``). + force: Replace handlers already attached to the root logger. + + Raises: + FileNotFoundError: If *config* does not exist. + ValueError: If *config* has an unsupported extension. + """ + if config is None: + logging.basicConfig( + level=level, format=fmt, stream=stream or sys.stderr, force=force + ) + return + + path = pathlib.Path(config).expanduser() + if not path.is_file(): + raise FileNotFoundError(str(path)) + suffix = path.suffix.lower() + if force: + root = logging.getLogger() + for handler in root.handlers[:]: + root.removeHandler(handler) + handler.close() + if suffix in (".conf", ".ini", ".cfg"): + logging.config.fileConfig(path, disable_existing_loggers=False) + elif suffix in (".yaml", ".yml", ".json"): + from suthing.file_handle import FileHandle + + spec = FileHandle.load(path) + if not isinstance(spec, dict): + raise ValueError(f"{path} does not hold a logging config mapping") + spec.setdefault("version", 1) + spec.setdefault("disable_existing_loggers", False) + logging.config.dictConfig(spec) + else: + raise ValueError( + f"unsupported logging config {path.name!r}:" + " expected .conf/.ini/.cfg or .yaml/.yml/.json" + ) diff --git a/suthing/profiling.py b/suthing/profiling.py new file mode 100644 index 0000000..ce6546d --- /dev/null +++ b/suthing/profiling.py @@ -0,0 +1,169 @@ +"""Opt-in function profiling. + +Decorate functions with :func:`profiled`; they are timed only while a +:class:`Profiler` is active, and cost one context-variable lookup otherwise:: + + @profiled(key_args="batch_size") + def ingest(rows, batch_size=100): ... + + with Profiler() as prof: + ingest(rows, batch_size=50) + ingest(rows, batch_size=500) + prof.summary() # {"ingest(batch_size=50)": ProfileStats(...), ...} + +The active profiler lives in a :class:`contextvars.ContextVar`, so it follows +``async`` tasks. Threads start with no active profiler unless they run in a +copied context (``contextvars.copy_context().run``). +""" + +from __future__ import annotations + +import functools +import inspect +import statistics +from collections import defaultdict +from collections.abc import Callable, Sequence +from contextvars import ContextVar, Token +from time import perf_counter +from typing import Any, NamedTuple, ParamSpec, Self, TypeVar, overload + +P = ParamSpec("P") +R = TypeVar("R") + +_active: ContextVar[Profiler | None] = ContextVar("suthing_profiler", default=None) + + +class ProfileStats(NamedTuple): + """Timing statistics for one profiling key, in seconds.""" + + count: int + total: float + mean: float + p50: float + max: float + + +class Profiler: + """Collects timings recorded by :func:`profiled` functions while active.""" + + def __init__(self) -> None: + self._samples: defaultdict[str, list[float]] = defaultdict(list) + self._tokens: list[Token[Profiler | None]] = [] + + def __enter__(self) -> Self: + self._tokens.append(_active.set(self)) + return self + + def __exit__(self, *exc: object) -> None: + _active.reset(self._tokens.pop()) + + def record(self, key: str, seconds: float) -> None: + """Add one timing under *key*.""" + self._samples[key].append(seconds) + + def samples(self) -> dict[str, list[float]]: + """A copy of every recorded timing, by key.""" + return {k: list(v) for k, v in self._samples.items()} + + def summary(self) -> dict[str, ProfileStats]: + """Count, total, mean, median and max timing per key.""" + return { + k: ProfileStats( + count=len(v), + total=sum(v), + mean=statistics.fmean(v), + p50=statistics.median(v), + max=max(v), + ) + for k, v in self._samples.items() + } + + def reset(self) -> None: + """Drop all recorded timings.""" + self._samples.clear() + + +def active_profiler() -> Profiler | None: + """The profiler recording in the current context, if any.""" + return _active.get() + + +def _key_builder( + func: Callable[..., Any], name: str, key_args: Sequence[str] +) -> Callable[..., str]: + if not key_args: + return lambda *args, **kwargs: name + signature = inspect.signature(func) + unknown = [a for a in key_args if a not in signature.parameters] + if unknown: + raise ValueError(f"{name} has no parameter(s) {', '.join(unknown)}") + + def build(*args: Any, **kwargs: Any) -> str: + try: + bound = signature.bind(*args, **kwargs) + except TypeError: + return name + bound.apply_defaults() + parts = ",".join(f"{a}={bound.arguments.get(a)!r}" for a in key_args) + return f"{name}({parts})" + + return build + + +@overload +def profiled(func: Callable[P, R], /) -> Callable[P, R]: ... + + +@overload +def profiled( + *, key_args: str | Sequence[str] = (), name: str | None = None +) -> Callable[[Callable[P, R]], Callable[P, R]]: ... + + +def profiled( + func: Callable[P, R] | None = None, + /, + *, + key_args: str | Sequence[str] = (), + name: str | None = None, +) -> Callable[P, R] | Callable[[Callable[P, R]], Callable[P, R]]: + """Time calls to a function whenever a :class:`Profiler` is active. + + Usable bare (``@profiled``) or with options (``@profiled(key_args="x")``). + + Args: + func: Function to wrap (bare use). + key_args: Parameter name(s) whose values become part of the key, so + calls with different arguments are reported separately + (``"load(size=100)"``). Defaults are filled in. + name: Key prefix; defaults to the function's qualified name. + + Returns: + The wrapped function, or a decorator. + + Raises: + ValueError: At decoration time, if a *key_args* name is not a + parameter of the function. + """ + args_tuple = (key_args,) if isinstance(key_args, str) else tuple(key_args) + + def decorate(f: Callable[P, R]) -> Callable[P, R]: + label = name or getattr(f, "__qualname__", None) or repr(f) + key_of = _key_builder(f, label, args_tuple) + + @functools.wraps(f) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: + prof = _active.get() + if prof is None: + return f(*args, **kwargs) + start = perf_counter() + try: + return f(*args, **kwargs) + finally: + prof.record(key_of(*args, **kwargs), perf_counter() - start) + + return wrapper + + if func is not None: + return decorate(func) + return decorate diff --git a/suthing/py.typed b/suthing/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/suthing/text.py b/suthing/text.py new file mode 100644 index 0000000..061f3b9 --- /dev/null +++ b/suthing/text.py @@ -0,0 +1,55 @@ +"""Text helpers.""" + +from __future__ import annotations + +import re +import unicodedata + +_UNSAFE = re.compile(r"[^A-Za-z0-9._-]+") + + +def slugify( + text: str, + *, + sep: str = "-", + fallback: str = "item", + lower: bool = False, + ascii_fold: bool = False, + max_length: int | None = None, +) -> str: + """Turn *text* into a token safe for file names and URL path segments. + + Every run of characters outside ``A-Za-z0-9._-`` becomes one *sep*, and + *sep* is stripped from both ends. + + Args: + text: Input text. + sep: Replacement for each run of unsafe characters. + fallback: Returned when nothing safe is left. + lower: Lower-case the result. + ascii_fold: Replace accented letters by their base letter (``"é"`` → + ``"e"``) instead of treating them as unsafe. + max_length: Cut the result to at most this many characters, then strip + *sep* again. + + Returns: + The slug, or *fallback*. + + Example: + >>> slugify(" Person / Company ") + 'Person-Company' + >>> slugify("Café Menü", sep="_", lower=True, ascii_fold=True) + 'cafe_menu' + """ + if ascii_fold: + text = ( + unicodedata.normalize("NFKD", text) + .encode("ascii", "ignore") + .decode("ascii") + ) + slug = _UNSAFE.sub(sep, text.strip()).strip(sep) + if lower: + slug = slug.lower() + if max_length is not None: + slug = slug[:max_length].strip(sep) + return slug or fallback diff --git a/suthing/timer.py b/suthing/timer.py index 9dc47ca..1578434 100644 --- a/suthing/timer.py +++ b/suthing/timer.py @@ -1,49 +1,125 @@ -"""Simple timer utility for code performance measurement.""" +"""Wall-clock timing and timestamps.""" -from timeit import default_timer +from __future__ import annotations -seconds_per_minute = 60 +import logging +from collections.abc import Callable +from contextlib import ContextDecorator +from datetime import UTC, datetime +from time import perf_counter +from typing import Self +logger = logging.getLogger(__name__) -class Timer: - """Context manager for timing code execution.""" +SECONDS_PER_MINUTE = 60 - def __init__(self): - """Initialize timer with default settings.""" - self.timer = default_timer - self.mins = 0 - self.secs = 0 - self.elapsed = 0 - def __enter__(self): - """Start timing on context enter. +def format_duration(seconds: float, digits: int = 2) -> str: + """Format a duration as ``"1 min 30.5 sec"`` (minutes only when non-zero). - Returns: - self: Timer instance + Args: + seconds: Duration in seconds. + digits: Decimal places for the seconds part. + + Returns: + The formatted duration. + """ + mins = int(seconds // SECONDS_PER_MINUTE) + secs = round(seconds - mins * SECONDS_PER_MINUTE, digits) + text = f"{secs} sec" + return f"{mins} min {text}" if mins > 0 else text + + +def utc_now_iso(timespec: str = "seconds") -> str: + """Current UTC time as ISO 8601 text, e.g. ``"2026-09-22T10:15:00+00:00"``. + + Args: + timespec: Precision, as for :meth:`datetime.datetime.isoformat` + (``"seconds"``, ``"milliseconds"``, ``"microseconds"``, ...). + + Returns: + The timestamp, with an explicit ``+00:00`` offset. + """ + return datetime.now(UTC).isoformat(timespec=timespec) + + +class Timer(ContextDecorator): + """Measure the wall-clock time of a block or a function. + + Uses :func:`time.perf_counter`. ``elapsed`` is live inside the block and + frozen once it exits. On exit the timer reports itself to *log* if given, + otherwise logs at DEBUG level when it has a *label*. + + As a decorator (``@Timer("load")``) the same instance is re-entered on + every call, so it reports the most recent call; it is not safe for + recursive or concurrent calls of the decorated function. + + Example: + >>> with Timer() as t: + ... do_work() + >>> print(t.elapsed_str) + >>> with Timer("ingest", log=print): + ... ingest() # prints "ingest: 1.23 sec" on exit + """ + + def __init__( + self, label: str | None = None, log: Callable[[str], object] | None = None + ) -> None: + """Create a timer. + + Args: + label: Name used when reporting on exit. + log: Called with ``"