diff --git a/.github/workflows/build_workflow.yml b/.github/workflows/build_workflow.yml
index d0aa857a..bd158784 100644
--- a/.github/workflows/build_workflow.yml
+++ b/.github/workflows/build_workflow.yml
@@ -19,10 +19,10 @@ jobs:
- name: Checkout Code Repository
uses: actions/checkout@v3
- - name: Set up Python 3.13
+ - name: Set up Python 3.14
uses: actions/setup-python@v4
with:
- python-version: "3.13"
+ python-version: "3.14"
# Run all pre-commit hooks on all the files.
# Getting only staged files can be tricky in case a new PR is opened
@@ -36,7 +36,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
- python-version: ["3.11", "3.12", "3.13"]
+ python-version: ["3.11", "3.12", "3.13", "3.14"]
defaults:
run:
shell: bash -l {0}
@@ -72,11 +72,6 @@ jobs:
conda list
# Ensure we have the right Python version
python --version
- # Fix pip issues for Python 3.12+
- if [[ "${{ matrix.python-version }}" == "3.12" ]] || [[ "${{ matrix.python-version }}" == "3.13" ]]; then
- python -m ensurepip --upgrade || true
- python -m pip install --upgrade --force-reinstall pip setuptools wheel
- fi
- name: Install `zstash` Package
run: |
@@ -121,7 +116,7 @@ jobs:
environment-file: conda/dev.yml
channel-priority: flexible # Changed from strict to flexible
auto-update-conda: true
- python-version: "3.13" # Use stable Python version for docs
+ python-version: "3.14" # Use stable Python version for docs
# sphinx-multiversion allows for version docs.
- name: Build Sphinx Docs
diff --git a/AGENTS.md b/AGENTS.md
new file mode 100644
index 00000000..1a2173e9
--- /dev/null
+++ b/AGENTS.md
@@ -0,0 +1,131 @@
+# Agent Guidelines for zstash
+
+This file provides guidance for AI coding agents working on the
+[zstash](https://github.com/E3SM-Project/zstash) repository.
+
+## Project overview
+
+**zstash** is an HPSS long-term archiving tool for E3SM (Energy Exascale Earth
+System Model). It bundles files into standard tar archives for efficient storage -- typically, but not always, on the NERSC HPSS tape system. Globus can be used to transfer data between machines. A SQLite index database (`index.db`) tracks every archived file with its path, size, mtime,
+md5 checksum, tar name, and offset.
+
+Key commands: `zstash create`, `zstash update`, `zstash check`, `zstash extract`, `zstash ls`, `zstash version`.
+
+Documentation:
+
+## Key Design Decisions
+
+- Files are **never split** across tar archives.
+- Tar files are created locally first, then transferred.
+- The index database (`index.db`) is the single source of truth for what is
+ archived and where.
+- `zstash update` always creates new tar files; it never appends to existing
+ ones (by design, to avoid corruption risk).
+- Checksums (md5) are computed on-the-fly during both create and extract.
+
+## Repository layout
+
+```
+zstash/ # Main Python package
+ main.py # CLI entry point
+ create.py # zstash create
+ update.py # zstash update
+ check.py # zstash check
+ extract.py # zstash extract
+ ls.py # zstash ls
+ hpss.py / hpss_utils.py # HPSS (hsi) transfer
+ globus.py / globus_utils.py # Globus transfer
+ settings.py # Global settings / config
+ utils.py # Shared utilities
+ parallel.py # Parallel worker support
+tests/
+ unit/ # Pure-function tests (pytest)
+ integration/
+ python_tests/
+ group_by_command/ # unittest, one file per command
+ group_by_workflow/ # unittest, end-to-end workflows
+ bash_tests/ # Bash scripts; some require HPSS/Globus access
+docs/source/ # Sphinx documentation (reStructuredText)
+conda/dev.yml # Development conda environment
+setup.cfg # pre-commit (e.g., flake8, isort, mypy) configuration
+```
+
+## Tech stack
+
+- **Python** ≥ 3.11, < 3.14
+- **SQLite** (`sqlite3` stdlib) for the index database
+- **globus-sdk** ≥ 3.15, < 4.0 for Globus transfers
+- **six** ≥ 1.16 for compatibility helpers
+- Build / packaging: `setup.py` + `setup.cfg`
+
+## Code style
+
+- **Formatter**: `black` (v25.1.0) — run automatically via pre-commit.
+- **Import order**: `isort` (v6.0.1) with `multi_line_output=3`.
+- **Linter**: `flake8` (v7.3.0), max line length **119**, config in `setup.cfg`.
+- **Type checker**: `mypy` (v1.18.2), `check_untyped_defs = True`.
+- Comments should explain *why*, not *what*.
+- Use Python type annotations in virtually all cases.
+- Error handling: for `None` checks on values that must be non-`None`, use `TypeError` with descriptive messages.
+
+All checks are enforced via `pre-commit` — **every commit must pass
+`pre-commit run --all-files`**.
+
+## Setting up a development environment
+
+```bash
+# First, activate conda.
+# Then:
+rm -rf build
+conda clean --all --y
+conda env create -f conda/dev.yml -n env-name
+conda activate env-name
+pre-commit run --all-files
+python -m pip install .
+```
+
+Individual `pre-commit` hooks: `trailing-whitespace`, `end-of-file-fixer`,
+`check-yaml`, `black`, `isort`, `flake8`, `mypy`. Run with `pre-commit run `
+
+## Testing
+
+The Python tests can be run as follows:
+```bash
+# Unit tests (fast, no HPSS/Globus required)
+pytest tests/unit/test_*.py
+
+# Integration tests (machine-independent)
+python -m unittest tests/integration/python_tests/group_by_command/test_*.py
+python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py
+```
+
+Some integration tests are skipped automatically when `hsi`/HPSS is not
+available (i.e., off of Perlmutter). Bash tests under `tests/integration/bash_tests/` may require
+a specific machine and/or Globus credentials.
+
+When to add tests:
+- Adding new features or internal functions
+- A bug is found
+
+When to modify tests:
+- Modifying features or internal functions.
+
+When NOT to modify tests:
+- Making non-functional code changes. For example, changes to support new versions of Python should not change behavior and thus tests should not be changed (even if implementation changes are required).
+
+## Git Workflow
+
+- `main` must always be deployable.
+- All changes are made through **feature branches**.
+- Rebase onto `main` (never merge) to resolve conflicts.
+- For small changes, use "Squash and merge", to reduce the number of extraneous commits.
+- For big changes, use "Create a merge commit", after squashing commits into distinct chunks of work.
+- Open a PR early for discussion; merge only after CI passes and PR is approved.
+- Never commit directly to `main`.
+- Changes should be as minimal as possible to achieve the objective. Avoid over-engineering.
+
+## Adding Dependencies
+
+New dependencies go in `conda/dev.yml`. Note the change in PR description and discuss with the team before adding new packages.
+
+They must be added to `conda/dev.yml` (not just as `import` statements), and tool version changes must be kept in sync between `conda/dev.yml` and `.pre-commit-config.yaml`.
diff --git a/conda/dev.yml b/conda/dev.yml
index 6ec8e1b9..85f64d32 100644
--- a/conda/dev.yml
+++ b/conda/dev.yml
@@ -5,7 +5,8 @@ dependencies:
# Base
# =================
- pip
- - python >=3.11,<3.14
+ - python >=3.11,<3.15
+ - setuptools
- sqlite
- six >=1.16.0
- globus-sdk >=3.15.0,<4.0
diff --git a/setup.cfg b/setup.cfg
index 4e6c6cb4..5a75b3ef 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -46,7 +46,7 @@ exclude =
venv
[mypy]
-python_version = 3.13
+python_version = 3.14
check_untyped_defs = True
ignore_missing_imports = True
warn_unused_ignores = True
diff --git a/setup.py b/setup.py
index 292653e0..67df2e8e 100644
--- a/setup.py
+++ b/setup.py
@@ -7,6 +7,6 @@
author_email="forsyth2@llnl.gov, golaz1@llnl.gov, shaheen2@llnl.gov",
description="Long term HPSS archiving software for E3SM",
packages=find_packages(include=["zstash", "zstash.*"]),
- python_requires=">=3.11,<3.14",
+ python_requires=">=3.11,<3.15",
entry_points={"console_scripts": ["zstash=zstash.main:main"]},
)
diff --git a/zstash/extract.py b/zstash/extract.py
index 64977aef..06faf40c 100644
--- a/zstash/extract.py
+++ b/zstash/extract.py
@@ -659,7 +659,12 @@ def extractFiles( # noqa: C901
logger.debug("Valid md5: {} {}".format(md5, fname))
elif extract_this_file:
- tar.extract(tarinfo)
+ if sys.version_info >= (3, 12):
+ tar.extract(
+ tarinfo, filter="tar"
+ ) # "data" is too restrictive, "fully_trusted" is too permissive.
+ else:
+ tar.extract(tarinfo)
# Note: tar.extract() will not restore time stamps of symbolic
# links. Could not find a Python-way to restore it either, so
# relying here on 'touch'. This is not the prettiest solution.
diff --git a/zstash/main.py b/zstash/main.py
index 6201834d..7d011d23 100755
--- a/zstash/main.py
+++ b/zstash/main.py
@@ -2,6 +2,7 @@
from __future__ import absolute_import, print_function
import argparse
+import multiprocessing
import os
import os.path
import sys
@@ -27,6 +28,12 @@ def handler(signal_received, frame):
# -----------------------------------------------------------------------------
def main():
+ # Force the use of 'fork' for multiprocessing to ensure consistent behavior
+ # across Python versions (Python 3.14+ changed the default to 'spawn').
+ # 'fork' is only available on POSIX systems (Linux, macOS).
+ if sys.platform != "win32":
+ multiprocessing.set_start_method("fork", force=True)
+
# Run the handler() function when SIGINT is received
signal(SIGINT, handler)