diff --git a/.github/workflows/build_workflow.yml b/.github/workflows/build_workflow.yml index d0aa857a..bd158784 100644 --- a/.github/workflows/build_workflow.yml +++ b/.github/workflows/build_workflow.yml @@ -19,10 +19,10 @@ jobs: - name: Checkout Code Repository uses: actions/checkout@v3 - - name: Set up Python 3.13 + - name: Set up Python 3.14 uses: actions/setup-python@v4 with: - python-version: "3.13" + python-version: "3.14" # Run all pre-commit hooks on all the files. # Getting only staged files can be tricky in case a new PR is opened @@ -36,7 +36,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.11", "3.12", "3.13"] + python-version: ["3.11", "3.12", "3.13", "3.14"] defaults: run: shell: bash -l {0} @@ -72,11 +72,6 @@ jobs: conda list # Ensure we have the right Python version python --version - # Fix pip issues for Python 3.12+ - if [[ "${{ matrix.python-version }}" == "3.12" ]] || [[ "${{ matrix.python-version }}" == "3.13" ]]; then - python -m ensurepip --upgrade || true - python -m pip install --upgrade --force-reinstall pip setuptools wheel - fi - name: Install `zstash` Package run: | @@ -121,7 +116,7 @@ jobs: environment-file: conda/dev.yml channel-priority: flexible # Changed from strict to flexible auto-update-conda: true - python-version: "3.13" # Use stable Python version for docs + python-version: "3.14" # Use stable Python version for docs # sphinx-multiversion allows for version docs. - name: Build Sphinx Docs diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..1a2173e9 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,131 @@ +# Agent Guidelines for zstash + +This file provides guidance for AI coding agents working on the +[zstash](https://github.com/E3SM-Project/zstash) repository. + +## Project overview + +**zstash** is an HPSS long-term archiving tool for E3SM (Energy Exascale Earth +System Model). It bundles files into standard tar archives for efficient storage -- typically, but not always, on the NERSC HPSS tape system. Globus can be used to transfer data between machines. A SQLite index database (`index.db`) tracks every archived file with its path, size, mtime, +md5 checksum, tar name, and offset. + +Key commands: `zstash create`, `zstash update`, `zstash check`, `zstash extract`, `zstash ls`, `zstash version`. + +Documentation: + +## Key Design Decisions + +- Files are **never split** across tar archives. +- Tar files are created locally first, then transferred. +- The index database (`index.db`) is the single source of truth for what is + archived and where. +- `zstash update` always creates new tar files; it never appends to existing + ones (by design, to avoid corruption risk). +- Checksums (md5) are computed on-the-fly during both create and extract. + +## Repository layout + +``` +zstash/ # Main Python package + main.py # CLI entry point + create.py # zstash create + update.py # zstash update + check.py # zstash check + extract.py # zstash extract + ls.py # zstash ls + hpss.py / hpss_utils.py # HPSS (hsi) transfer + globus.py / globus_utils.py # Globus transfer + settings.py # Global settings / config + utils.py # Shared utilities + parallel.py # Parallel worker support +tests/ + unit/ # Pure-function tests (pytest) + integration/ + python_tests/ + group_by_command/ # unittest, one file per command + group_by_workflow/ # unittest, end-to-end workflows + bash_tests/ # Bash scripts; some require HPSS/Globus access +docs/source/ # Sphinx documentation (reStructuredText) +conda/dev.yml # Development conda environment +setup.cfg # pre-commit (e.g., flake8, isort, mypy) configuration +``` + +## Tech stack + +- **Python** ≥ 3.11, < 3.14 +- **SQLite** (`sqlite3` stdlib) for the index database +- **globus-sdk** ≥ 3.15, < 4.0 for Globus transfers +- **six** ≥ 1.16 for compatibility helpers +- Build / packaging: `setup.py` + `setup.cfg` + +## Code style + +- **Formatter**: `black` (v25.1.0) — run automatically via pre-commit. +- **Import order**: `isort` (v6.0.1) with `multi_line_output=3`. +- **Linter**: `flake8` (v7.3.0), max line length **119**, config in `setup.cfg`. +- **Type checker**: `mypy` (v1.18.2), `check_untyped_defs = True`. +- Comments should explain *why*, not *what*. +- Use Python type annotations in virtually all cases. +- Error handling: for `None` checks on values that must be non-`None`, use `TypeError` with descriptive messages. + +All checks are enforced via `pre-commit` — **every commit must pass +`pre-commit run --all-files`**. + +## Setting up a development environment + +```bash +# First, activate conda. +# Then: +rm -rf build +conda clean --all --y +conda env create -f conda/dev.yml -n env-name +conda activate env-name +pre-commit run --all-files +python -m pip install . +``` + +Individual `pre-commit` hooks: `trailing-whitespace`, `end-of-file-fixer`, +`check-yaml`, `black`, `isort`, `flake8`, `mypy`. Run with `pre-commit run ` + +## Testing + +The Python tests can be run as follows: +```bash +# Unit tests (fast, no HPSS/Globus required) +pytest tests/unit/test_*.py + +# Integration tests (machine-independent) +python -m unittest tests/integration/python_tests/group_by_command/test_*.py +python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py +``` + +Some integration tests are skipped automatically when `hsi`/HPSS is not +available (i.e., off of Perlmutter). Bash tests under `tests/integration/bash_tests/` may require +a specific machine and/or Globus credentials. + +When to add tests: +- Adding new features or internal functions +- A bug is found + +When to modify tests: +- Modifying features or internal functions. + +When NOT to modify tests: +- Making non-functional code changes. For example, changes to support new versions of Python should not change behavior and thus tests should not be changed (even if implementation changes are required). + +## Git Workflow + +- `main` must always be deployable. +- All changes are made through **feature branches**. +- Rebase onto `main` (never merge) to resolve conflicts. +- For small changes, use "Squash and merge", to reduce the number of extraneous commits. +- For big changes, use "Create a merge commit", after squashing commits into distinct chunks of work. +- Open a PR early for discussion; merge only after CI passes and PR is approved. +- Never commit directly to `main`. +- Changes should be as minimal as possible to achieve the objective. Avoid over-engineering. + +## Adding Dependencies + +New dependencies go in `conda/dev.yml`. Note the change in PR description and discuss with the team before adding new packages. + +They must be added to `conda/dev.yml` (not just as `import` statements), and tool version changes must be kept in sync between `conda/dev.yml` and `.pre-commit-config.yaml`. diff --git a/conda/dev.yml b/conda/dev.yml index 6ec8e1b9..85f64d32 100644 --- a/conda/dev.yml +++ b/conda/dev.yml @@ -5,7 +5,8 @@ dependencies: # Base # ================= - pip - - python >=3.11,<3.14 + - python >=3.11,<3.15 + - setuptools - sqlite - six >=1.16.0 - globus-sdk >=3.15.0,<4.0 diff --git a/setup.cfg b/setup.cfg index 4e6c6cb4..5a75b3ef 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,7 +46,7 @@ exclude = venv [mypy] -python_version = 3.13 +python_version = 3.14 check_untyped_defs = True ignore_missing_imports = True warn_unused_ignores = True diff --git a/setup.py b/setup.py index 292653e0..67df2e8e 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,6 @@ author_email="forsyth2@llnl.gov, golaz1@llnl.gov, shaheen2@llnl.gov", description="Long term HPSS archiving software for E3SM", packages=find_packages(include=["zstash", "zstash.*"]), - python_requires=">=3.11,<3.14", + python_requires=">=3.11,<3.15", entry_points={"console_scripts": ["zstash=zstash.main:main"]}, ) diff --git a/zstash/extract.py b/zstash/extract.py index 64977aef..06faf40c 100644 --- a/zstash/extract.py +++ b/zstash/extract.py @@ -659,7 +659,12 @@ def extractFiles( # noqa: C901 logger.debug("Valid md5: {} {}".format(md5, fname)) elif extract_this_file: - tar.extract(tarinfo) + if sys.version_info >= (3, 12): + tar.extract( + tarinfo, filter="tar" + ) # "data" is too restrictive, "fully_trusted" is too permissive. + else: + tar.extract(tarinfo) # Note: tar.extract() will not restore time stamps of symbolic # links. Could not find a Python-way to restore it either, so # relying here on 'touch'. This is not the prettiest solution. diff --git a/zstash/main.py b/zstash/main.py index 6201834d..7d011d23 100755 --- a/zstash/main.py +++ b/zstash/main.py @@ -2,6 +2,7 @@ from __future__ import absolute_import, print_function import argparse +import multiprocessing import os import os.path import sys @@ -27,6 +28,12 @@ def handler(signal_received, frame): # ----------------------------------------------------------------------------- def main(): + # Force the use of 'fork' for multiprocessing to ensure consistent behavior + # across Python versions (Python 3.14+ changed the default to 'spawn'). + # 'fork' is only available on POSIX systems (Linux, macOS). + if sys.platform != "win32": + multiprocessing.set_start_method("fork", force=True) + # Run the handler() function when SIGINT is received signal(SIGINT, handler)