From e76fb1b2504de3878e75e608aa887304e9da9357 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 22 Jun 2026 20:42:13 +0000 Subject: [PATCH 01/10] Reorganize and expand documentation guides --- docs/source/dev_guide/index.rst | 7 ++ docs/source/dev_guide/tar_tracking_modes.dot | 45 ++++++++ docs/source/dev_guide/tar_tracking_modes.rst | 107 +++++++++++++++++++ docs/source/dev_guide/testing.rst | 93 ++++++++++++++++ docs/source/index.rst | 19 ++-- docs/source/usage.rst | 9 +- docs/source/user_guide/configuration.rst | 57 ++++++++++ docs/source/user_guide/globus.rst | 92 ++++++++++++++++ docs/source/user_guide/index.rst | 18 ++++ 9 files changed, 434 insertions(+), 13 deletions(-) create mode 100644 docs/source/dev_guide/tar_tracking_modes.dot create mode 100644 docs/source/dev_guide/tar_tracking_modes.rst create mode 100644 docs/source/dev_guide/testing.rst create mode 100644 docs/source/user_guide/configuration.rst create mode 100644 docs/source/user_guide/globus.rst create mode 100644 docs/source/user_guide/index.rst diff --git a/docs/source/dev_guide/index.rst b/docs/source/dev_guide/index.rst index eb02549c..93148bfe 100644 --- a/docs/source/dev_guide/index.rst +++ b/docs/source/dev_guide/index.rst @@ -2,10 +2,17 @@ Developer Guide ############### +Use this guide for repository conventions, testing, release work, and +implementation details that are mainly useful to contributors. + .. toctree:: :maxdepth: 2 + ../contributing + ../design project-standards + tar_tracking_modes + testing ci release_testing release diff --git a/docs/source/dev_guide/tar_tracking_modes.dot b/docs/source/dev_guide/tar_tracking_modes.dot new file mode 100644 index 00000000..29b4118b --- /dev/null +++ b/docs/source/dev_guide/tar_tracking_modes.dot @@ -0,0 +1,45 @@ +digraph tar_tracking_modes { + rankdir=LR; + node [shape=box]; + + source [label="source files"]; + local_tar [label="local tar in cache"]; + index_db [label="index.db"]; + files_table [label="files table"]; + tars_table [label="tars table"]; + + source -> local_tar [label="construct_tars"]; + local_tar -> files_table [label="record members"]; + local_tar -> tars_table [label="record tar metadata"]; + files_table -> index_db; + tars_table -> index_db; + + subgraph cluster_none { + label="--hpss=none"; + local_archive [label="local archive kept in cache"]; + } + + subgraph cluster_hpss { + label="HPSS path"; + hsi_put [label="hsi put"]; + remote_hpss [label="remote HPSS archive"]; + } + + subgraph cluster_globus { + label="globus:// path"; + batch [label="TransferBatch / TransferData"]; + task [label="Globus task"]; + remote_globus [label="remote archive via Globus"]; + } + + local_tar -> local_archive [label="keep local"]; + local_tar -> hsi_put [label="close then transfer"]; + hsi_put -> remote_hpss; + + local_tar -> batch [label="add tar to batch"]; + batch -> task [label="submit"]; + task -> remote_globus [label="success"]; + + index_db -> remote_hpss [label="upload last"]; + index_db -> remote_globus [label="transfer last"]; +} diff --git a/docs/source/dev_guide/tar_tracking_modes.rst b/docs/source/dev_guide/tar_tracking_modes.rst new file mode 100644 index 00000000..bf51c01f --- /dev/null +++ b/docs/source/dev_guide/tar_tracking_modes.rst @@ -0,0 +1,107 @@ +################## +Tar Tracking Modes +################## + +This page explains how zstash creates, tracks, transfers, and cleans up tar +files in the three supported ``--hpss`` modes: + +* ``none`` +* a direct HPSS path +* a Globus URL + +Common lifecycle +================ + +Regardless of mode, ``zstash create`` and ``zstash update`` follow the same +high-level pattern: + +1. Walk the source tree and decide which files belong in the next tar. +2. Create the tar locally in the cache directory. +3. Record file-level metadata in ``files`` and, when enabled, tar-level + metadata in ``tars``. +4. Transfer the tar if the archive uses HPSS or Globus. +5. Transfer ``index.db`` after the tar work is complete. + +The archive database is the source of truth for both file membership and tar +metadata. + +Mode: ``--hpss=none`` +===================== + +In local-only mode, zstash still builds the archive exactly the same way, but +there is no remote transfer step. + +* tar files are created under the local cache, usually ``zstash/`` +* each tar is left in place after it is closed +* zstash removes write permission from completed tar files so they are less + likely to be changed accidentally +* ``index.db`` remains local alongside the tar files + +This is the simplest mode because the local cache is the archive. + +Mode: direct HPSS path +====================== + +When ``--hpss`` points to an HPSS directory, zstash uses ``hsi`` for transfers. + +* each tar is still created locally first +* after a tar is closed, zstash uploads it to the target HPSS directory +* if ``--keep`` is not set, the local tar can be deleted after a successful + transfer +* ``index.db`` is uploaded after the tar work completes, but it is not tracked + for deletion like tar files are + +This mode is the most direct path when the current machine already has HPSS +access. + +Mode: Globus URL +================ + +When ``--hpss`` is a ``globus://`` URL, zstash still creates each tar locally +first, but transfer tracking becomes more explicit. + +* each completed tar is added to a transfer batch +* the batch is associated with Globus task state instead of an immediate + ``hsi put`` +* local tar files are kept until Globus reports success for the corresponding + transfer task +* ``globus_finalize()`` submits any remaining pending transfer data, waits for + completion, and then removes successfully transferred tar files when + ``--keep`` is not set +* ``index.db`` is transferred after the tar workflow, just as in the direct + HPSS mode + +This mode is what allows zstash to archive to HPSS from machines that do not +have direct HPSS access. + +Summary table +============= + +.. list-table:: + :header-rows: 1 + + * - ``--hpss`` value + - Transfer mechanism + - When local tar can be removed + - Where the archive lives + * - ``none`` + - no transfer + - never removed automatically + - local cache only + * - HPSS path + - ``hsi`` + - immediately after successful transfer, unless ``--keep`` + - local cache plus remote HPSS directory + * - Globus URL + - Globus transfer task + - after Globus success and finalization, unless ``--keep`` + - local cache plus remote Globus-backed destination + +Graphviz source +=============== + +The following ``.dot`` file summarizes the relationships between local tar +creation, database updates, and transfer handling in each mode: + +.. literalinclude:: tar_tracking_modes.dot + :language: dot diff --git a/docs/source/dev_guide/testing.rst b/docs/source/dev_guide/testing.rst new file mode 100644 index 00000000..4ed81701 --- /dev/null +++ b/docs/source/dev_guide/testing.rst @@ -0,0 +1,93 @@ +####### +Testing +####### + +This page summarizes the test process described in ``tests/README.md``. + +Test layout +=========== + +The repository separates machine-independent tests from machine-specific +integration tests: + +* ``tests/unit/`` contains pytest-based tests for pure functions +* ``tests/integration/python_tests/group_by_command/`` contains unittest-based + command-oriented integration tests +* ``tests/integration/python_tests/group_by_workflow/`` contains unittest-based + end-to-end workflow tests +* ``tests/integration/bash_tests/run_from_any/`` contains bash-driven tests + that can be run from any machine +* ``tests/integration/bash_tests/run_from_perlmutter/`` contains tests that + need Perlmutter or direct HPSS access +* ``tests/integration/bash_tests/run_from_chrysalis/`` contains tests that need + Chrysalis and Globus-related setup +* ``tests/utils/`` contains shared test helpers + +Recommended baseline workflow +============================= + +For a normal development change, start with the machine-independent checks from +the repository root:: + + rm -rf build + conda clean --all --y + conda env create -f conda/dev.yml -n zstash_dev_test + conda activate zstash_dev_test + pre-commit run --all-files + python -m pip install . + pytest tests/unit/test_*.py + python -m unittest tests/integration/python_tests/group_by_command/test_*.py + python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py + +Some integration tests are skipped automatically on systems that do not have +``hsi`` or HPSS access. + +Bash-based integration tests +============================ + +The bash tests are grouped by the machine or environment they require. + +Run from any machine +-------------------- + +The ``run_from_any`` directory contains bash tests that can be exercised +without a specific facility, although Globus authentication may still be part +of the workflow. The README specifically calls out reviewing the instructions in +``globus_auth.bash`` before running the related scripts. + +Run from Perlmutter +------------------- + +The ``run_from_perlmutter`` directory contains tests that depend on direct HPSS +access and Perlmutter-specific paths. + +Run from Chrysalis +------------------ + +The ``run_from_chrysalis`` directory contains tests that depend on Chrysalis, +Globus setup, and in some cases explicit cleanup of previous authentication +state before rerunning. + +Choosing the right scope +======================== + +Use the smallest test scope that matches the change: + +* unit tests for isolated logic changes +* Python integration tests for command behavior +* bash and machine-specific tests for HPSS- or Globus-specific workflows +* release testing for end-to-end release validation; see + :doc:`release_testing` + +GitHub Actions +============== + +GitHub Actions runs the machine-independent test suite in +``.github/workflows/build_workflow.yml``: + +* ``pytest tests/unit/test_*.py`` +* ``python -m unittest tests/integration/python_tests/group_by_command/test_*.py`` +* ``python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py`` + +That workflow is the baseline CI safety net, while the machine-specific bash +tests remain primarily manual. diff --git a/docs/source/index.rst b/docs/source/index.rst index f01ddb47..ec08556d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -44,18 +44,17 @@ For documentation not included in the version selector (<= ``v1.0.1``): * `v1.0.1 `_ * `v1.0.0 `_ +The documentation is organized into two major sections: + +* :doc:`User Guide ` for installation, day-to-day usage, + Globus setup, and archive management +* :doc:`Developer Guide ` for contributing, testing, release + work, and internal implementation details + .. toctree:: :maxdepth: 2 - :caption: Contents: + :caption: Guides: self - getting_started - tutorial - usage - best_practices - design - database - support + user_guide/index dev_guide/index - contributing - diff --git a/docs/source/usage.rst b/docs/source/usage.rst index b2d4223a..5d36e999 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -40,6 +40,7 @@ where Then zstash will use `Globus `_ to store a new zstash archive on a Globus endpoint. Names ``alcf`` and ``nersc`` are recognized as referring to the ALCF HPSS and NERSC HPSS endpoints, e.g. ``globus://nersc/~/my_archive``. + See :doc:`user_guide/globus` and :doc:`user_guide/configuration` for first-time Globus setup details. * ```` specifies the path to the local directory that should be archived. Additional optional arguments: @@ -55,7 +56,7 @@ Additional optional arguments: The default is 256 GB. Zstash will create tar files that are smaller than MAXSIZE except when individual input files exceed MAXSIZE (as individual files are never split up between different tar files). -* ``--non-blocking`` Zstash will submit a Globus transfer and immediately create a subsequent tarball. That is, Zstash will not wait until the transfer completes to start creating a subsequent tarball. On machines where it takes more time to create a tarball than transfer it, each Globus transfer will have one file. On machines where it takes less time to create a tarball than transfer it, the first transfer will have one file, but the number of tarballs in subsequent transfers will grow finding dynamically the most optimal number of tarballs per transfer. NOTE: zstash is currently always non-blocking. +* ``--non-blocking`` Zstash will submit a Globus transfer and immediately create a subsequent tarball. That is, Zstash will not wait until the transfer completes to start creating a subsequent tarball. On machines where it takes more time to create a tarball than transfer it, each Globus transfer will have one file. On machines where it takes less time to create a tarball than transfer it, the first transfer will have one file, but the number of tarballs in subsequent transfers will grow finding dynamically the most optimal number of tarballs per transfer. * ``--error-on-duplicate-tar`` FOR ADVANCED USERS ONLY: Raise an error if a tar file with the same name already exists in the database. If this flag is set, zstash will exit if it sees a duplicate tar. If it is not set, zstash's behavior will depend on whether or not the --overwrite-duplicate-tar flag is set. * ``--overwrite-duplicate-tars`` FOR ADVANCED USERS ONLY: If a duplicate tar is encountered, overwrite the existing database record with the new one (i.e., it will assume the latest tar is the correct one). If this flag is not set, zstash will permit multiple entries for the same tar in its database. * ``-v`` increases output verbosity. @@ -128,6 +129,9 @@ If you want to store zstash archive on these two remote HPSS file systems, you c .. note:: Always activate Globus endpoints via the Globus web interface before running ``zstash``. +For a more complete first-time Globus workflow, including ``~/.zstash.ini``, +see :doc:`user_guide/globus` and :doc:`user_guide/configuration`. + Check ===== @@ -242,7 +246,7 @@ where * ``--keep`` to keep a copy of the tar files on the local file system after they have been extracted from the archive. Normally, they are deleted after successful transfer. -* ``--non-blocking`` Zstash will submit a Globus transfer and immediately create a subsequent tarball. That is, Zstash will not wait until the transfer completes to start creating a subsequent tarball. On machines where it takes more time to create a tarball than transfer it, each Globus transfer will have one file. On machines where it takes less time to create a tarball than transfer it, the first transfer will have one file, but the number of tarballs in subsequent transfers will grow finding dynamically the most optimal number of tarballs per transfer. NOTE: zstash is currently always non-blocking. +* ``--non-blocking`` Zstash will submit a Globus transfer and immediately create a subsequent tarball. That is, Zstash will not wait until the transfer completes to start creating a subsequent tarball. On machines where it takes more time to create a tarball than transfer it, each Globus transfer will have one file. On machines where it takes less time to create a tarball than transfer it, the first transfer will have one file, but the number of tarballs in subsequent transfers will grow finding dynamically the most optimal number of tarballs per transfer. * ``--error-on-duplicate-tar`` FOR ADVANCED USERS ONLY: Raise an error if a tar file with the same name already exists in the database. If this flag is set, zstash will exit if it sees a duplicate tar. If it is not set, zstash's behavior will depend on whether or not the --overwrite-duplicate-tar flag is set. * ``--overwrite-duplicate-tars`` FOR ADVANCED USERS ONLY: If a duplicate tar is encountered, overwrite the existing database record with the new one (i.e., it will assume the latest tar is the correct one). If this flag is not set, zstash will permit multiple entries for the same tar in its database. * ``-v`` increases output verbosity. @@ -530,4 +534,3 @@ Starting with version 0.3, you can check the version of zstash from the command $ zstash version v0.3.0 - diff --git a/docs/source/user_guide/configuration.rst b/docs/source/user_guide/configuration.rst new file mode 100644 index 00000000..1e7a8832 --- /dev/null +++ b/docs/source/user_guide/configuration.rst @@ -0,0 +1,57 @@ +################## +Configuration File +################## + +zstash may create and read a configuration file at ``~/.zstash.ini`` when you +use Globus-backed archives. + +What ``.zstash.ini`` stores +=========================== + +The main setting currently used by zstash is the local Globus endpoint UUID. +The file uses an INI format such as: + +.. code-block:: ini + + [local] + globus_endpoint_uuid = 6bdc7956-fc0f-4ad2-989c-7aa5ee643a79 + +If the file does not exist, zstash creates it the first time it needs Globus +configuration. + +When you may need to edit it +============================ + +Most users do not need to edit ``~/.zstash.ini``. Manual updates are useful +when: + +* zstash cannot infer the local endpoint UUID from the machine hostname +* you want to override the default local endpoint for a machine +* the machine's Globus collection changed and the old UUID is still recorded +* you are testing on a new system before hostname-based auto-detection has been + added to zstash + +If the UUID is wrong or blank, Globus transfers may fail before they start. + +Finding the right UUID +====================== + +Use the UUID for the local collection that should serve as the source or +destination of the transfer on the machine where you are running ``zstash``. +You can usually find that UUID from the Globus web interface for the +collection. + +After updating ``~/.zstash.ini``, rerun the zstash command that needs Globus. + +Related state files +=================== + +Two other files can affect Globus behavior: + +* ``~/.zstash_globus_tokens.json`` stores refresh tokens from previous Globus + logins +* ``~/.globus-native-apps.cfg`` may be left over from older Globus-based + workflows + +If zstash reports that a stored refresh token is invalid, deleting +``~/.zstash_globus_tokens.json`` is the usual way to force a fresh login. diff --git a/docs/source/user_guide/globus.rst b/docs/source/user_guide/globus.rst new file mode 100644 index 00000000..9b18e8e7 --- /dev/null +++ b/docs/source/user_guide/globus.rst @@ -0,0 +1,92 @@ +############ +Globus Guide +############ + +This page is intended for users who want to run ``zstash`` on a machine that +does not have direct HPSS access, or who prefer to move zstash archives through +the `Globus `_ transfer service. + +When to use Globus +================== + +Use a Globus destination when ``--hpss`` is set to a URL of the form:: + + globus:/// + +Examples include: + +* ``globus://nersc/~/my_archive`` +* ``globus://alcf/~/my_archive`` +* ``globus://9cd89cfd-6d04-11e5-ba46-22000b92c6ec/~/my_archive`` + +The names ``nersc`` and ``alcf`` are built-in shortcuts for the NERSC HPSS and +ALCF HPSS Globus endpoints. + +First-time setup +================ + +For a first Globus-based archive, the safest approach is: + +1. Identify the local Globus collection for the machine where you will run + ``zstash``. +2. Identify the destination collection and destination path. +3. Activate both collections in the Globus web interface before you run + ``zstash``. +4. Start with a small archive so you can confirm that authentication, endpoint + activation, and path selection are correct. + +If you are creating a new archive, a minimal first test looks like:: + + zstash create --hpss=globus://nersc/~/test/my_archive . + +After the transfer completes, verify it with ``zstash check`` or retrieve a +small file with ``zstash extract``. + +Authentication flow +=================== + +The first time zstash needs Globus credentials, it will print an authorization +URL and ask you to paste back the returned code. After a successful login, +zstash stores refresh-token state in ``~/.zstash_globus_tokens.json`` so future +commands usually do not need another interactive login. + +zstash also checks ``~/.zstash.ini`` for the local endpoint UUID. See +:doc:`configuration` for details on when that file needs to be created or +edited manually. + +Choosing endpoint paths +======================= + +The destination portion of ``globus:///`` should name the +remote directory that will hold the zstash archive contents: + +* ``index.db`` +* one or more tar files such as ``000000.tar``, ``000001.tar``, and so on + +As with HPSS paths, use a destination directory that is dedicated to one +zstash archive. + +Operational notes +================= + +* ``zstash create`` and ``zstash update`` create tar files locally first, then + transfer them through Globus. +* ``zstash check``, ``zstash extract``, and ``zstash ls`` still rely on the + archive's ``index.db`` to locate files and tars. +* ``--non-blocking`` applies only to Globus transfers. Use it when you want + zstash to continue building later tar files before the current transfer has + finished. + +Troubleshooting +=============== + +If a Globus workflow fails unexpectedly: + +* Re-activate the source and destination collections in the Globus web + interface. +* Confirm that ``~/.zstash.ini`` points to the correct local endpoint UUID. +* If zstash reports token problems after you switch endpoints or machines, + remove ``~/.zstash_globus_tokens.json`` and retry so zstash can request a new + login. +* If the destination path does not exist, create it first and rerun the + command. diff --git a/docs/source/user_guide/index.rst b/docs/source/user_guide/index.rst new file mode 100644 index 00000000..940fe5b4 --- /dev/null +++ b/docs/source/user_guide/index.rst @@ -0,0 +1,18 @@ +########## +User Guide +########## + +Use this guide for installing zstash, creating and maintaining archives, and +working with HPSS or Globus in day-to-day workflows. + +.. toctree:: + :maxdepth: 2 + + ../getting_started + globus + configuration + ../tutorial + ../usage + ../best_practices + ../database + ../support From 8f314225f8431776d9f6c57f10a30de6a22f9298 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 22 Jun 2026 20:57:44 +0000 Subject: [PATCH 02/10] Restore guide page visibility from docs index --- docs/source/index.rst | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/source/index.rst b/docs/source/index.rst index ec08556d..ed799e14 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -51,6 +51,38 @@ The documentation is organized into two major sections: * :doc:`Developer Guide ` for contributing, testing, release work, and internal implementation details +User Guide pages +================ + +The user-facing documentation is organized under :doc:`user_guide/index` and +includes: + +* :doc:`getting_started` for installation and first-time setup +* :doc:`user_guide/globus` for Globus account setup and transfer workflows +* :doc:`user_guide/configuration` for ``.zstash.ini`` configuration details +* :doc:`tutorial` for a full archive creation and extraction walkthrough +* :doc:`usage` for command-line usage details +* :doc:`best_practices` for archive management recommendations +* :doc:`database` for the archive index database layout +* :doc:`support` for where to ask questions or report issues + +Developer Guide pages +===================== + +The contributor and maintainer documentation is organized under +:doc:`dev_guide/index` and includes: + +* :doc:`contributing` for development environment setup and contribution + workflow +* :doc:`design` for the high-level architecture and implementation overview +* :doc:`dev_guide/project-standards` for coding standards and conventions +* :doc:`dev_guide/tar_tracking_modes` for tar tracking behavior in each storage + mode +* :doc:`dev_guide/testing` for the test layout and execution guidance +* :doc:`dev_guide/ci` for continuous integration details +* :doc:`dev_guide/release_testing` for release validation steps +* :doc:`dev_guide/release` for the release process + .. toctree:: :maxdepth: 2 :caption: Guides: From db0166cb1fdc636089d2ee8adc10b1ee7c2f885a Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 22 Jun 2026 15:36:25 -0700 Subject: [PATCH 03/10] Organize docs files --- .../code_snippets}/tar_tracking_modes.dot | 0 .../figures}/Globus_Screenshot_1.png | Bin .../figures}/Globus_Screenshot_2.png | Bin .../figures}/Globus_Screenshot_3.png | Bin .../figures}/Globus_Screenshot_4.png | Bin .../{ => figures}/docs-version-selector.png | Bin .../figures}/git-flow.svg | 0 .../figures}/github_release.png | Bin .../figures}/pre-commit-flow.svg | 0 .../{ => figures}/pre-commit-passing.png | Bin .../contributing_to_docs.rst} | 2 +- docs/source/dev_guide/index.rst | 7 ++--- ...ct-standards.rst => project_standards.rst} | 6 ++-- docs/source/dev_guide/release.rst | 3 ++ docs/source/dev_guide/tar_tracking_modes.rst | 2 +- docs/source/index.rst | 26 +++++++++--------- .../archived_documentation/index.rst | 10 +++++++ .../archived_documentation}/tutorial.rst | 0 .../{ => user_guide}/best_practices.rst | 8 +++--- docs/source/{ => user_guide}/database.rst | 0 docs/source/{ => user_guide}/design.rst | 0 .../{ => user_guide}/getting_started.rst | 2 +- .../user_guide/{ => globus}/configuration.rst | 0 docs/source/user_guide/globus/index.rst | 9 ++++++ .../{globus.rst => globus/intro.rst} | 6 ++-- docs/source/user_guide/index.rst | 14 +++++----- docs/source/{ => user_guide}/support.rst | 0 docs/source/{ => user_guide}/usage.rst | 0 28 files changed, 58 insertions(+), 37 deletions(-) rename docs/source/{dev_guide => _static/code_snippets}/tar_tracking_modes.dot (100%) rename docs/source/{globus => _static/figures}/Globus_Screenshot_1.png (100%) rename docs/source/{globus => _static/figures}/Globus_Screenshot_2.png (100%) rename docs/source/{globus => _static/figures}/Globus_Screenshot_3.png (100%) rename docs/source/{globus => _static/figures}/Globus_Screenshot_4.png (100%) rename docs/source/_static/{ => figures}/docs-version-selector.png (100%) rename docs/source/{dev_guide => _static/figures}/git-flow.svg (100%) rename docs/source/{dev_guide => _static/figures}/github_release.png (100%) rename docs/source/{dev_guide => _static/figures}/pre-commit-flow.svg (100%) rename docs/source/_static/{ => figures}/pre-commit-passing.png (100%) rename docs/source/{contributing.rst => dev_guide/contributing_to_docs.rst} (98%) rename docs/source/dev_guide/{project-standards.rst => project_standards.rst} (97%) create mode 100644 docs/source/user_guide/archived_documentation/index.rst rename docs/source/{ => user_guide/archived_documentation}/tutorial.rst (100%) rename docs/source/{ => user_guide}/best_practices.rst (97%) rename docs/source/{ => user_guide}/database.rst (100%) rename docs/source/{ => user_guide}/design.rst (100%) rename docs/source/{ => user_guide}/getting_started.rst (99%) rename docs/source/user_guide/{ => globus}/configuration.rst (100%) create mode 100644 docs/source/user_guide/globus/index.rst rename docs/source/user_guide/{globus.rst => globus/intro.rst} (98%) rename docs/source/{ => user_guide}/support.rst (100%) rename docs/source/{ => user_guide}/usage.rst (100%) diff --git a/docs/source/dev_guide/tar_tracking_modes.dot b/docs/source/_static/code_snippets/tar_tracking_modes.dot similarity index 100% rename from docs/source/dev_guide/tar_tracking_modes.dot rename to docs/source/_static/code_snippets/tar_tracking_modes.dot diff --git a/docs/source/globus/Globus_Screenshot_1.png b/docs/source/_static/figures/Globus_Screenshot_1.png similarity index 100% rename from docs/source/globus/Globus_Screenshot_1.png rename to docs/source/_static/figures/Globus_Screenshot_1.png diff --git a/docs/source/globus/Globus_Screenshot_2.png b/docs/source/_static/figures/Globus_Screenshot_2.png similarity index 100% rename from docs/source/globus/Globus_Screenshot_2.png rename to docs/source/_static/figures/Globus_Screenshot_2.png diff --git a/docs/source/globus/Globus_Screenshot_3.png b/docs/source/_static/figures/Globus_Screenshot_3.png similarity index 100% rename from docs/source/globus/Globus_Screenshot_3.png rename to docs/source/_static/figures/Globus_Screenshot_3.png diff --git a/docs/source/globus/Globus_Screenshot_4.png b/docs/source/_static/figures/Globus_Screenshot_4.png similarity index 100% rename from docs/source/globus/Globus_Screenshot_4.png rename to docs/source/_static/figures/Globus_Screenshot_4.png diff --git a/docs/source/_static/docs-version-selector.png b/docs/source/_static/figures/docs-version-selector.png similarity index 100% rename from docs/source/_static/docs-version-selector.png rename to docs/source/_static/figures/docs-version-selector.png diff --git a/docs/source/dev_guide/git-flow.svg b/docs/source/_static/figures/git-flow.svg similarity index 100% rename from docs/source/dev_guide/git-flow.svg rename to docs/source/_static/figures/git-flow.svg diff --git a/docs/source/dev_guide/github_release.png b/docs/source/_static/figures/github_release.png similarity index 100% rename from docs/source/dev_guide/github_release.png rename to docs/source/_static/figures/github_release.png diff --git a/docs/source/dev_guide/pre-commit-flow.svg b/docs/source/_static/figures/pre-commit-flow.svg similarity index 100% rename from docs/source/dev_guide/pre-commit-flow.svg rename to docs/source/_static/figures/pre-commit-flow.svg diff --git a/docs/source/_static/pre-commit-passing.png b/docs/source/_static/figures/pre-commit-passing.png similarity index 100% rename from docs/source/_static/pre-commit-passing.png rename to docs/source/_static/figures/pre-commit-passing.png diff --git a/docs/source/contributing.rst b/docs/source/dev_guide/contributing_to_docs.rst similarity index 98% rename from docs/source/contributing.rst rename to docs/source/dev_guide/contributing_to_docs.rst index f47868f6..9176b5eb 100644 --- a/docs/source/contributing.rst +++ b/docs/source/dev_guide/contributing_to_docs.rst @@ -72,7 +72,7 @@ using Sphinx, you can refer to $ # Check the `_build/html` folder for all generated versioned docs $ # Open `_build/html//index.html` to view in browser - .. figure:: _static/docs-version-selector.png + .. figure:: _static/figures/docs-version-selector.png :alt: Docs version selector Docs version selector dropdown in the bottom left-hand corner diff --git a/docs/source/dev_guide/index.rst b/docs/source/dev_guide/index.rst index 93148bfe..2a5982aa 100644 --- a/docs/source/dev_guide/index.rst +++ b/docs/source/dev_guide/index.rst @@ -8,11 +8,10 @@ implementation details that are mainly useful to contributors. .. toctree:: :maxdepth: 2 - ../contributing - ../design - project-standards + project_standards + ci tar_tracking_modes testing - ci release_testing release + contributing_to_docs diff --git a/docs/source/dev_guide/project-standards.rst b/docs/source/dev_guide/project_standards.rst similarity index 97% rename from docs/source/dev_guide/project-standards.rst rename to docs/source/dev_guide/project_standards.rst index be276e3e..f2363d5e 100644 --- a/docs/source/dev_guide/project-standards.rst +++ b/docs/source/dev_guide/project_standards.rst @@ -11,7 +11,7 @@ Version Control (VC) The repository uses a fork-based Git workflow with tag releases. -.. figure:: git-flow.svg +.. figure:: /_static/figures/git-flow.svg :alt: Git Flow Diagram Guidelines for VC @@ -39,7 +39,7 @@ The repository uses the ``pre-commit`` package to manage pre-commit hooks. These hooks help enforce quality assurance standards and identify simple issues at the commit level before submitting code reviews. -.. figure:: pre-commit-flow.svg +.. figure:: /_static/figures/pre-commit-flow.svg :alt: Pre-commit Flow Diagram ``pre-commit`` Flow @@ -57,7 +57,7 @@ Automatically run all pre-commit hooks (just commit) :: # Tip: If there is an issue with pre-commit, you can bypass with the `--no-verify` flag. Please do NOT use this on a regular basis. git commit -m '...' -.. figure:: ../_static/pre-commit-passing.png +.. figure:: ../_static/figures/pre-commit-passing.png :alt: pre-commit Output ``pre-commit`` Output diff --git a/docs/source/dev_guide/release.rst b/docs/source/dev_guide/release.rst index 5c870764..e67cb08a 100644 --- a/docs/source/dev_guide/release.rst +++ b/docs/source/dev_guide/release.rst @@ -69,6 +69,9 @@ Releasing on GitHub: release candidates Releasing on GitHub: production releases ---------------------------------------- +.. figure:: /_static/figures/github_release.png + :alt: GitHub Release Diagram + 1. Draft a new release `here `_. You can save this and come back to it later, if need be. 2. Set `Tag version` to ``v``, **including the "v"**. `@Target` should be ``main``. 3. Set `Release title` to ``v``, **including the "v"**. diff --git a/docs/source/dev_guide/tar_tracking_modes.rst b/docs/source/dev_guide/tar_tracking_modes.rst index bf51c01f..2cb3300e 100644 --- a/docs/source/dev_guide/tar_tracking_modes.rst +++ b/docs/source/dev_guide/tar_tracking_modes.rst @@ -103,5 +103,5 @@ Graphviz source The following ``.dot`` file summarizes the relationships between local tar creation, database updates, and transfer handling in each mode: -.. literalinclude:: tar_tracking_modes.dot +.. literalinclude:: /_static/code_snippets/tar_tracking_modes.dot :language: dot diff --git a/docs/source/index.rst b/docs/source/index.rst index ed799e14..127bf521 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -57,14 +57,15 @@ User Guide pages The user-facing documentation is organized under :doc:`user_guide/index` and includes: -* :doc:`getting_started` for installation and first-time setup -* :doc:`user_guide/globus` for Globus account setup and transfer workflows -* :doc:`user_guide/configuration` for ``.zstash.ini`` configuration details -* :doc:`tutorial` for a full archive creation and extraction walkthrough -* :doc:`usage` for command-line usage details -* :doc:`best_practices` for archive management recommendations -* :doc:`database` for the archive index database layout -* :doc:`support` for where to ask questions or report issues +* :doc:`user_guide/design` for the high-level architecture and implementation overview +* :doc:`user_guide/getting_started` for installation and first-time setup +* :doc:`user_guide/usage` for command-line usage details +* :doc:`user_guide/globus` for Globus account setup and transfer workflows and ``.zstash.ini`` configuration details +* :doc:`user_guide/best_practices` for archive management recommendations +* :doc:`user_guide/database` for the archive index database layout +* :doc:`user_guide/support` for where to ask questions or report issues +* :doc:`user_guide/archived_documentation` for a full archive creation and extraction walkthrough + Developer Guide pages ===================== @@ -72,16 +73,15 @@ Developer Guide pages The contributor and maintainer documentation is organized under :doc:`dev_guide/index` and includes: -* :doc:`contributing` for development environment setup and contribution - workflow -* :doc:`design` for the high-level architecture and implementation overview -* :doc:`dev_guide/project-standards` for coding standards and conventions +* :doc:`dev_guide/project_standards` for coding standards and conventions +* :doc:`dev_guide/ci` for continuous integration details * :doc:`dev_guide/tar_tracking_modes` for tar tracking behavior in each storage mode * :doc:`dev_guide/testing` for the test layout and execution guidance -* :doc:`dev_guide/ci` for continuous integration details * :doc:`dev_guide/release_testing` for release validation steps * :doc:`dev_guide/release` for the release process +* :doc:`dev_guide/contributing_to_docs` for development environment setup and contribution + workflow .. toctree:: :maxdepth: 2 diff --git a/docs/source/user_guide/archived_documentation/index.rst b/docs/source/user_guide/archived_documentation/index.rst new file mode 100644 index 00000000..f5943fd8 --- /dev/null +++ b/docs/source/user_guide/archived_documentation/index.rst @@ -0,0 +1,10 @@ +###################### +Archived Documentation +###################### + +These pages document older versions of ``zstash`` but may still be useful as a reference. + +.. toctree:: + :maxdepth: 2 + + tutorial diff --git a/docs/source/tutorial.rst b/docs/source/user_guide/archived_documentation/tutorial.rst similarity index 100% rename from docs/source/tutorial.rst rename to docs/source/user_guide/archived_documentation/tutorial.rst diff --git a/docs/source/best_practices.rst b/docs/source/user_guide/best_practices.rst similarity index 97% rename from docs/source/best_practices.rst rename to docs/source/user_guide/best_practices.rst index 63c3e5c8..d5304426 100644 --- a/docs/source/best_practices.rst +++ b/docs/source/user_guide/best_practices.rst @@ -166,7 +166,7 @@ Transfer all zstash files to NERSC HPSS using Globus. * On the leftmost pane, select 'ENDPOINT' * Search for 'NERSC HPSS'. Click on Green power button to activate endpoint. -.. image:: globus/Globus_Screenshot_1.png +.. image:: /_static/figures/Globus_Screenshot_1.png :scale: 50% :alt: Globus screenshot, NERSC HPSS endpoint @@ -174,14 +174,14 @@ Transfer all zstash files to NERSC HPSS using Globus. * Search for 'compy-dtn'. Click on Green power button to activate endpoint. Login using your compy credentials (username, PIN+RSA). -.. image:: globus/Globus_Screenshot_2.png +.. image:: /_static/figures/Globus_Screenshot_2.png :scale: 50% :alt: Globus screenshot, compy-dtn endpoint * In the file manager, navigate to your local zstash directory. * Click on 'Transfer or Sync...' -.. image:: globus/Globus_Screenshot_3.png +.. image:: /_static/figures/Globus_Screenshot_3.png :scale: 50% :alt: Globus screenshot, file manager @@ -198,7 +198,7 @@ Transfer all zstash files to NERSC HPSS using Globus. * Click 'Start ->'. -.. image:: globus/Globus_Screenshot_4.png +.. image:: /_static/figures/Globus_Screenshot_4.png :scale: 50% :alt: Globus screenshot, sync diff --git a/docs/source/database.rst b/docs/source/user_guide/database.rst similarity index 100% rename from docs/source/database.rst rename to docs/source/user_guide/database.rst diff --git a/docs/source/design.rst b/docs/source/user_guide/design.rst similarity index 100% rename from docs/source/design.rst rename to docs/source/user_guide/design.rst diff --git a/docs/source/getting_started.rst b/docs/source/user_guide/getting_started.rst similarity index 99% rename from docs/source/getting_started.rst rename to docs/source/user_guide/getting_started.rst index 16b22a28..b4607533 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/user_guide/getting_started.rst @@ -275,7 +275,7 @@ Furthermore, the dev environment includes quality assurance (QA) tools such as c git commit -m "commit-message" - .. figure:: _static/pre-commit-passing.png + .. figure:: _static/figures/pre-commit-passing.png :alt: pre-commit Output ``pre-commit`` Output diff --git a/docs/source/user_guide/configuration.rst b/docs/source/user_guide/globus/configuration.rst similarity index 100% rename from docs/source/user_guide/configuration.rst rename to docs/source/user_guide/globus/configuration.rst diff --git a/docs/source/user_guide/globus/index.rst b/docs/source/user_guide/globus/index.rst new file mode 100644 index 00000000..c3292105 --- /dev/null +++ b/docs/source/user_guide/globus/index.rst @@ -0,0 +1,9 @@ +###### +Globus +###### + +.. toctree:: + :maxdepth: 2 + + intro + configuration diff --git a/docs/source/user_guide/globus.rst b/docs/source/user_guide/globus/intro.rst similarity index 98% rename from docs/source/user_guide/globus.rst rename to docs/source/user_guide/globus/intro.rst index 9b18e8e7..ac6f34ba 100644 --- a/docs/source/user_guide/globus.rst +++ b/docs/source/user_guide/globus/intro.rst @@ -1,6 +1,6 @@ -############ -Globus Guide -############ +################### +Globus Introduction +################### This page is intended for users who want to run ``zstash`` on a machine that does not have direct HPSS access, or who prefer to move zstash archives through diff --git a/docs/source/user_guide/index.rst b/docs/source/user_guide/index.rst index 940fe5b4..8916901c 100644 --- a/docs/source/user_guide/index.rst +++ b/docs/source/user_guide/index.rst @@ -8,11 +8,11 @@ working with HPSS or Globus in day-to-day workflows. .. toctree:: :maxdepth: 2 - ../getting_started + design + getting_started + usage globus - configuration - ../tutorial - ../usage - ../best_practices - ../database - ../support + best_practices + database + support + archived_documentation diff --git a/docs/source/support.rst b/docs/source/user_guide/support.rst similarity index 100% rename from docs/source/support.rst rename to docs/source/user_guide/support.rst diff --git a/docs/source/usage.rst b/docs/source/user_guide/usage.rst similarity index 100% rename from docs/source/usage.rst rename to docs/source/user_guide/usage.rst From 3e8de43c118f248475f6d9f4d93be558c82214bc Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 22 Jun 2026 16:16:42 -0700 Subject: [PATCH 04/10] Update release docs --- docs/source/dev_guide/index.rst | 2 +- docs/source/dev_guide/release.rst | 146 ------------------ docs/source/dev_guide/releases/index.rst | 16 ++ .../releases/production_releases.rst | 61 ++++++++ .../dev_guide/releases/release_candidates.rst | 106 +++++++++++++ docs/source/index.rst | 2 +- 6 files changed, 185 insertions(+), 148 deletions(-) delete mode 100644 docs/source/dev_guide/release.rst create mode 100644 docs/source/dev_guide/releases/index.rst create mode 100644 docs/source/dev_guide/releases/production_releases.rst create mode 100644 docs/source/dev_guide/releases/release_candidates.rst diff --git a/docs/source/dev_guide/index.rst b/docs/source/dev_guide/index.rst index 2a5982aa..29ad932b 100644 --- a/docs/source/dev_guide/index.rst +++ b/docs/source/dev_guide/index.rst @@ -13,5 +13,5 @@ implementation details that are mainly useful to contributors. tar_tracking_modes testing release_testing - release + releases contributing_to_docs diff --git a/docs/source/dev_guide/release.rst b/docs/source/dev_guide/release.rst deleted file mode 100644 index e67cb08a..00000000 --- a/docs/source/dev_guide/release.rst +++ /dev/null @@ -1,146 +0,0 @@ -How to Prepare a Release -======================== - -In this guide, we'll cover: - -* Bumping the Version -* Releasing On GitHub -* Releasing The Software On Anaconda -* Creating a New Version of the Documentation - -Bumping the Version -------------------- - -1. Checkout a branch with the name of the version. - - :: - - git fetch upstream main - # Prepend "v" to - # For release candidates, append "rc" to - git checkout -b v upstream/main - -2. Bump version using tbump. - - :: - - # Exclude "v" and should match the above step. - # --no-tag is required since tagging is handled in "Releasing on GitHub" - $ tbump --no-tag - - :: Bumping from 1.1.0 to 1.2.0 - => Would patch these files - - setup.py:26 version="1.1.0", - + setup.py:26 version="1.2.0", - - zstash/__init__.py:1 __version__ = "v1.1.0" - + zstash/__init__.py:1 __version__ = "v1.2.0" - - conda/meta.yaml:2 {% set version = "1.1.0" %} - + conda/meta.yaml:2 {% set version = "1.2.0" %} - - tbump.toml:5 current = "1.1.0" - + tbump.toml:5 current = "1.2.0" - => Would run these git commands - $ git add --update - $ git commit --message Bump to 1.2.0 - $ git push origin v1.2.0 - :: Looking good? (y/N) - > - -3. If you encounter ``Error: Command `git push upstream main` failed``, as in `zppy issue 470 `_, you can run ``git push upstream `` yourself. - -4. Create a pull request to the main repo and merge it. Mark yourself as the assignee, and mark "Update version" as the label. - -.. _github-release: - -Releasing on GitHub: release candidates ---------------------------------------- - -1. Create a tag for the release candidate at https://github.com/E3SM-Project/zstash/tags. Example, bumping to v1.2.0rc1: - - :: - - $ git checkout main - $ git fetch upstream - $ git reset --hard upstream/main - $ git tag -a v1.2.0rc1 -m "v1.2.0rc1" - # Delete the branch from the tbump step. Otherwise, the push command won't work. - $ git branch -D v1.2.0rc1 - $ git push upstream v1.2.0rc1 - -Releasing on GitHub: production releases ----------------------------------------- - -.. figure:: /_static/figures/github_release.png - :alt: GitHub Release Diagram - -1. Draft a new release `here `_. You can save this and come back to it later, if need be. -2. Set `Tag version` to ``v``, **including the "v"**. `@Target` should be ``main``. -3. Set `Release title` to ``v``, **including the "v"**. -4. Use `Describe this release` to summarize the changelog. - - * You can scroll through `zstash commits `_ for a list of changes. - * You can look at the last release to get an idea of how to format the description. - -5. Click `Publish release`. -6. CI/CD release workflow is automatically triggered. - - -Releasing on conda-forge: release candidates --------------------------------------------- - -1. If you don't have a local version of the conda-forge repo, run: :: - - git clone git@github.com:conda-forge/zstash-feedstock.git - git remote add upstream git@github.com:conda-forge/zstash-feedstock.git - -2. If you don't have a fork of the conda-forge repo, on `conda-forge `_, click the "Fork" button in the upper right hand corner. Then, on your fork, click the green "Code" button, and copy the SSH path. Run: :: - - git remote add - -3. Get the sha256 of the tag you made in "Releasing on GitHub: release candidates": :: - - curl -sL https://github.com/E3SM-Project/zstash/archive/v1.2.0rc1.tar.gz | openssl sha256 - -4. Make changes on a local branch. Example, bumping to v1.2.0rc1: :: - - $ git fetch upstream dev - $ git checkout -b v1.2.0rc1 upstream/dev # You can name the branch anything you want - # In `recipe/meta.yaml`, update the version and sha256 (and the build number if needed): - {% set version = "1.2.0rc1" %} # Set to your version - sha256: ... # The sha256 from the previous step - number: 0 # build > number should always be 0 - $ git add -A - $ git commit -m "v1.2.0rc1" - $ git push v1.2.0rc1 - -5. Note that the conda-forge bot does not work for release candidates. So, make a PR manually from your fork of the feedstock to the ``dev`` branch of `conda-forge `_. Then, the package build on conda-forge will end up with the ``zstash_dev`` label. You can add the "automerge" label to have the PR automatically merge once CI checks pass. - -6. 6. After merging, CI runs again (in a slightly different way). Then, check the https://anaconda.org/conda-forge/zstash page to view the newly updated package. Release candidates are assigned the ``zstash_dev`` label. Note that it takes about 15 minutes for the files to propagate across conda-forge's mirroring services, which must happen before you can use the files. - -Releasing on conda-forge: production releases ------------------- - -1. Be sure to have already completed :ref:`Releasing On GitHub `. This triggers the CI/CD workflow that handles Anaconda releases. -2. Wait for a bot PR to come up automatically on conda-forge after the GitHub release. This can happen anywhere from 1 hour to 1 day later. -3. Re-render the PR (see `docs `_). -4. Merge the PR on conda-forge. -5. Check the https://anaconda.org/conda-forge/zstash page to view the newly updated package. Production releases are assigned the ``main`` label. -6. Notify the maintainers of the unified E3SM environment about the new release on the `E3SM Confluence site `_. - - * Be sure to only update the ``zstash`` version number in the correct version(s) of the E3SM Unified environment. - * This is almost certainly one of the E3SM Unified versions listed under “Next versions”. If you are uncertain of which to update, leave a comment on the page asking. - -Creating a New Version of the Documentation -------------------------------------------- - -1. Be sure to have already completed :ref:`Releasing On GitHub `. This triggers the CI/CD workflow that handles publishing documentation versions. -2. Wait until the CI/CD build is successful. You can view all workflows at `All Workflows `_. -3. Changes will be available on the `zstash documentation page `_. - -Extra Resources ---------------- - -Conda-forge: - -* https://conda-forge.org/docs/user/introduction.html#why-conda-forge -* https://conda-forge.org/docs/maintainer/infrastructure.html#admin-web-services -* https://acme-climate.atlassian.net/wiki/spaces/IPD/pages/3616735236/Releasing+E3SM+Software+on+Anaconda+conda-forge+channel diff --git a/docs/source/dev_guide/releases/index.rst b/docs/source/dev_guide/releases/index.rst new file mode 100644 index 00000000..149aa8aa --- /dev/null +++ b/docs/source/dev_guide/releases/index.rst @@ -0,0 +1,16 @@ +How to Prepare a Release +======================== + + +.. toctree:: + :maxdepth: 1 + + release_candidates + production_releases + +Extra Resources +--------------- + +- Conda-forge docs: https://conda-forge.org/docs/user/introduction.html +- Admin web services: https://conda-forge.org/docs/maintainer/infrastructure.html#admin-web-services +- E3SM Anaconda release guide: https://acme-climate.atlassian.net/wiki/spaces/IPD/pages/3616735236/Releasing+E3SM+Software+on+Anaconda+conda-forge+channel diff --git a/docs/source/dev_guide/releases/production_releases.rst b/docs/source/dev_guide/releases/production_releases.rst new file mode 100644 index 00000000..13851e6a --- /dev/null +++ b/docs/source/dev_guide/releases/production_releases.rst @@ -0,0 +1,61 @@ +.. _production_releases: + +Preparing a production release +============================== + +Step 1: Testing +--------------- + +Be sure to run the entire integration test suite before making a production release. + +Step 2: Confluence +------------------ + +This step should already have been completed during the release-candidate phase, however it is good practice to double check that the `E3SM Unified version tracking page `_ has had the next E3SM Unified version updated with the new ``zstash`` version number. + +Step 3: tbump +------------- + +Similar to the release-candidate directions, we'll use ``v1.2.3`` as an example version number here. + + .. code-block:: bash + + cd zstash + git status # Confirm there's no uncommitted changes + git fetch upstream main # This assumes you've named your remote for the main repo as "upstream" + git checkout -b v1.2.3 upstream/main + git log --oneline | head -n 5 + # Check that the latest commits match what's on https://github.com/E3SM-Project/zstash/commits/main/ + conda activate env-name # Activate any zstash dev environment you have; we just need `tbump` + tbump 1.2.3 --no-tag + # This creates a commit, but won't push it (because the branch isn't named `main`) + git push upstream v1.2.3 + # Create, and "Update version" label" to, and merge the PR; delete the branch on GitHub + +Step 4: Make the release on the zstash repo +-------------------------------------------- + +.. figure:: /_static/figures/github_release.png + :alt: GitHub Release Diagram + +1. Draft a new release `here `_. Click "Draft a new release". +2. Set Tag version to ``v1.2.3``, including the "v". ``@Target`` should be ``main``. Click "Tag", then "Create new tag" and enter "v1.2.3". +3. Set Release title to ``v1.2.3``, including the "v". +4. Use "Describe this release" to summarize the changelog. Write two sections: "Summary of changes" (the high-level summary) & "Full list of changes" (the categorized list of commits, from reviewing the `zstash commits `_). +5. Make sure "Set as the latest release" is checked. +6. Click "Publish release". Unlike the RCs, ``v1.2.3`` should now appear on _both_ `Tags `_ and `Releases `_. +7. CI/CD release workflow will be automatically triggered. The docs workflow is just for the docs. Clicking "Publish release" is responsible for triggering the bot PR on conda-forge. + +Step 5: zstash-feedstock repo +------------------------------ + +1. Wait for a bot PR to come up automatically on conda-forge after the GitHub release. This can happen anywhere from 1 hour to 1 day later. Check https://github.com/conda-forge/zstash-feedstock/pulls. (Alternative: open an issue with the bot command: ``@conda-forge-admin, please update version`` and the PR will be opened.) +2. Complete any requirements to merge the PR. +3. Check the https://anaconda.org/conda-forge/zstash/files/manage page to view the newly updated package. Check it has the ``main`` label. + +Step 6: Check the docs +---------------------- + +1. Wait for the docs workflow to complete successfully. +2. Wait until the CI/CD build is successful. You can view all workflows at `All Workflows `_. +3. Changes will be available on the `zstash documentation page `_. diff --git a/docs/source/dev_guide/releases/release_candidates.rst b/docs/source/dev_guide/releases/release_candidates.rst new file mode 100644 index 00000000..2dff8420 --- /dev/null +++ b/docs/source/dev_guide/releases/release_candidates.rst @@ -0,0 +1,106 @@ +.. _release_candidates: + +Preparing a release candidate +============================= + +Step 1: Testing +--------------- + +Be sure to run the entire integration test suite before making a release candidate. + +Step 2: Determine what the new version should be +------------------------------------------------- + +This step only needs to be done for rc1. + +Review `zstash commits `_. Identify what changes since the last production release would require a patch, minor, or major version update (as in ``vMAJOR.MINOR.PATCH``). + +The highest-level version update will be used. For example, if you found 1 commit requiring a patch update, 1 commit requiring a minor update, and 0 commits requiring a major update, then you'd want to increment the minor version. + +Step 3: Confluence +------------------ + +This step only needs to be done for rc1. + +Update the `E3SM Unified version tracking page `_ with the new ``zstash`` version number. Be sure to update the section for the next E3SM Unified environment, not the current one. + +Step 4: tbump +------------- + +In this example, ``v1.2.3rc4`` is used -- so, the major version is 1, the minor version is 2, and the patch version is 3. The release candidate number is 4. + + .. code-block:: bash + + cd zstash + git status # Make sure you don't have any uncommitted changes + git fetch upstream main # This assumes you've named your remote for the main repo as "upstream" + git checkout -b v1.2.3rc4 upstream/main + conda activate env-name # Activate any zstash dev environment you have; we just need `tbump` + tbump 1.2.3rc4 --no-tag + # This creates a commit, but won't push it (because the branch isn't named `main`) + git diff HEAD^ HEAD | cat + # Review the change `tbump` added + git push upstream v1.2.3rc4 + # Create, and "Update version" label" to, and merge the PR; delete the branch on GitHub + +Step 5: Tag the RC on the zstash repo +-------------------------------------- + + .. code-block:: bash + + git checkout main + git fetch upstream + git reset --hard upstream/main + git tag -a v1.2.3rc4 -m "v1.2.3rc4" # Add the tag for this RC + # Delete the branch from the tbump step. Otherwise, the push command won't work. + git branch -D v1.2.3rc4 + git push upstream v1.2.3rc4 # Push the new RC tag to GitHub + +``v1.2.3rc4`` should now appear on `Tags `__, but _not_ on `Releases `_. + +Step 6: zstash-feedstock repo +------------------------------ + +If you don't have a fork of zstash-feedstock, first go to the `zstash-feedstock repo `__, and click the "Fork" button in the top right. + + .. code-block:: bash + + # If you don't have a clone of zstash-feedstock, first run: + # Clone using the SSH from the green "Code" button in the top right of the repo home page. + git clone git@github.com:conda-forge/zstash-feedstock.git + cd zstash-feedstock + git remote -v # See your remotes + # Optional; this lets you use upstream to refer to the main repo: + git remote add upstream git@github.com:conda-forge/zstash-feedstock.git + # Required; this allows you to push to your fork: + # Copy the SSH path from your fork's green "Code" button and run: + git remote add your-fork-name git@github.com:your-fork-name/zstash-feedstock.git + + # If you already have your clone set up, just run: + cd zstash-feedstock + + # In all cases: + curl -sL https://github.com/E3SM-Project/zstash/archive/v1.2.3rc4.tar.gz | openssl sha256 + # SHA2-256(stdin)= long hex string + git status # Check for uncommitted changes + git fetch upstream dev # Make sure you fetch the dev branch, not the main branch! + git checkout -b v1.2.3rc4 upstream/dev + emacs recipe/meta.yaml + # Update the version and sha256 (and the build number if needed): + # {% set version = "1.2.3rc4" %} + # sha256: ... # The sha256 from a few commands earlier + # number: 0 # build >>> number should always be 0 + + # Check the diff since the last release/RC. Examples: + # - https://github.com/E3SM-Project/zstash/compare/v1.2.2...v1.2.3rc1 + # - https://github.com/E3SM-Project/zstash/compare/v1.2.3rc3...v1.2.3rc4 + # If there are changes in dependencies there (e.g., in `conda/dev.yml`), + # you'll want to include them in this feedstock PR too. + + git add -A + git commit -m "v1.2.3rc4" + git push your-fork-name v1.2.3rc4 + +Then, create a pull request to the ``dev`` branch. Do _not_ set the pull request to merge to ``main``, as that will create a production release! RC packages get the ``zstash_dev`` label. + +Follow any further directions given by the ``conda-forge/zstash-feedstock`` PR template. Once that PR is reviewed and merged and CI completes, check https://anaconda.org/conda-forge/zstash for the new package (allow ~15 minutes for mirroring). diff --git a/docs/source/index.rst b/docs/source/index.rst index 127bf521..2d5084bf 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -79,7 +79,7 @@ The contributor and maintainer documentation is organized under mode * :doc:`dev_guide/testing` for the test layout and execution guidance * :doc:`dev_guide/release_testing` for release validation steps -* :doc:`dev_guide/release` for the release process +* :doc:`dev_guide/releases` for the release process * :doc:`dev_guide/contributing_to_docs` for development environment setup and contribution workflow From e0704e7bb7de2c2047a917ea504577bf4bde4ad7 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 22 Jun 2026 17:24:14 -0700 Subject: [PATCH 05/10] Update testing docs --- docs/source/dev_guide/release_testing.rst | 95 ---------- docs/source/dev_guide/testing.rst | 209 ++++++++++++++++++++-- docs/source/index.rst | 1 - 3 files changed, 194 insertions(+), 111 deletions(-) delete mode 100644 docs/source/dev_guide/release_testing.rst diff --git a/docs/source/dev_guide/release_testing.rst b/docs/source/dev_guide/release_testing.rst deleted file mode 100644 index 85e8c533..00000000 --- a/docs/source/dev_guide/release_testing.rst +++ /dev/null @@ -1,95 +0,0 @@ -*************************************** -Testing directions for making a release -*************************************** - -1. Have three shells open: one on Chrysalis, one on Compy, and one on Perlmutter. Do the following steps on each machine. - - * If running on Perlmutter, it is preferable to run from ``$CSCRATCH`` rather than ``/global/homes``. Running from the latter may result in a 'Resource temporarily unavailable' error. - * If running on Compy, it is necessary to run ``test_globus.py`` from a sub-directory of ``/compyfs`` rather than ``/qfs``. - -2. ``cd`` to the ``zstash`` directory. - -3. Check out a branch to test on. - - a. test dev (run before making a new zstash RC) :: - - git fetch upstream main - git checkout -b test_pre_zstash_rc<#> upstream/main - git log # check the commits match https://github.com/E3SM-Project/zstash/commits/main - - b. test new Unified RC :: - - git fetch upstream main - git checkout -b test_unified_rc<#>_ upstream/main - git log # check the commits match https://github.com/E3SM-Project/zstash/commits/main - - c. test final Unified :: - - git fetch upstream main - git checkout -b test_unified_<#>_ upstream/main - git log # check the commits match https://github.com/E3SM-Project/zstash/commits/main - -4. Set up your environment. - - a. test dev (run before making a new zstash RC): Set up a new development environment. This ensures that testing will use the latest conda changes. Note that you will need to run ``conda remove -n zstash_dev_pre_rc<#> --all`` first if you have previously done this step. :: - - mamba clean --all - mamba env create -f conda/dev.yml -n zstash_dev_pre_rc<#> - conda activate zstash_dev_pre_rc<#> - pip install . - - b. test new Unified RC: Launch the E3SM Unified environment for the machine you're on. Change out the version numbers below. :: - - * Chrysalis: ``source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.9.0rc16_chrysalis.sh`` - * Compy: ``source /share/apps/E3SM/conda_envs/test_e3sm_unified_1.9.0rc16_compy.sh`` - * Perlmutter: ``source /global/common/software/e3sm/anaconda_envs/test_e3sm_unified_1.9.0rc16_pm-cpu.sh`` - - c. test final Unified :: - - * Chrysalis: ``source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh`` - * Compy: ``source /share/apps/E3SM/conda_envs/load_latest_e3sm_unified_compy.sh`` - * Perlmutter: ``source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh`` - -5. Activate Globus. Go to https://www.globus.org/. Log in with your NERSC credentials. Activate the following Globus endpoints using the corresponding credentials: - - * `Chrysalis `_ - * `Compy `_ - * `Perlmutter/NERSC `_ - -6. Run the unit tests with ``python -m unittest tests/test_*.py``. - - a. test dev (run before making a new zstash RC): - - * If there are any failures, fix the code (or tests). If you make any conda changes, go back to step 4a. If you otherwise change zstash source code, run ``pip install .`` and then redo step 6. If you only make changes to tests, you can immediately redo step 6. - - b. test new Unified RC: - - * If there are any failures, fix the code and go back to step 1, following the (a: test dev (run before making a new zstash RC)) directions. - - c. test final Unified: - - * There should be no failures. If there are, a patch release of E3SM Unified may be required. - - For a, b, and c: - - * If there are no failures, proceed to the next step. - -7. Run ``git diff``. You should not have any changes. If you do, you have probably made code changes to get the tests to pass. Make a pull request to merge the changes. Add the "semver: bug" label. - -8. Wrap up release testing: - - a. test dev (run before making a new zstash RC): Create the next zstash RC by following the "release candidates" directions at https://e3sm-project.github.io/zstash/_build/html/main/dev_guide/release.html. - - b. test new Unified RC: Create the next zstash RC by following the "production releases" directions at https://e3sm-project.github.io/zstash/_build/html/main/dev_guide/release.html. - - c. test final Unified: You can now safely remove old branches and environments. At https://github.com/E3SM-Project/zstash/branches, delete any branches that are no longer needed. Also, run: :: - - # Branches - $ cd - $ git branch # Look at all branch names - $ git branch -D - - # Environments - $ conda env list - # For each environment you want to delete, run: - $ conda remove -n --all diff --git a/docs/source/dev_guide/testing.rst b/docs/source/dev_guide/testing.rst index 4ed81701..4c02ccbd 100644 --- a/docs/source/dev_guide/testing.rst +++ b/docs/source/dev_guide/testing.rst @@ -39,9 +39,81 @@ the repository root:: python -m unittest tests/integration/python_tests/group_by_command/test_*.py python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py +Example of expected output when all tests pass:: + + # pytest tests/unit/test_*.py + # 1 passed in 0.19s + + # python -m unittest tests/integration/python_tests/group_by_command/test_*.py + # Ran 69 tests in 327.570s + # OK + + # python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py + # Ran 4 tests in 2.666s + # OK + Some integration tests are skipped automatically on systems that do not have ``hsi`` or HPSS access. +Unit tests +========== + +The ``tests/unit/`` directory holds pytest-based tests targeting isolated, +pure-Python logic — no HPSS, Globus, or filesystem side-effects required. + +Current unit test files +----------------------- + +``test_hpss.py`` + Exercises the ``get_files_to_archive_with_stats`` helper and the logic that + compares on-disk files against the zstash SQLite database. Key scenarios + covered: + + * Scanning a directory and returning size/mtime pairs for each file. + * Building the ``archived_files`` dictionary from database rows, keeping the + entry with the most recent modification time when duplicates exist. + * Detecting **new** files (present on disk but absent from the database). + * Detecting **modified** files whose size or modification time differs from + the archived record, using a configurable ``TIME_TOL`` tolerance (default + 1 second). + * Verifying that files within the tolerance window are *not* re-archived. + +``test_utils.py`` + Covers ``zstash.utils.run_command`` subprocess behavior: + + * When the command starts with ``hsi``, the function strips + ``LD_LIBRARY_PATH`` and ``LD_PRELOAD`` from the child environment to + avoid linker conflicts, while preserving other variables such as ``HOME``. + * For all other commands the loader variables are passed through unchanged. + +Run the unit suite from the repository root:: + + pytest tests/unit/test_*.py + +Python integration tests +======================== + +The ``tests/integration/python_tests/`` directory contains ``unittest``-based +tests that invoke zstash commands and inspect their output. They are split +into two sub-directories: + +``group_by_command/`` + Each test file focuses on a single zstash sub-command (e.g. ``create``, + ``update``, ``extract``, ``check``). This makes it straightforward to run + only the tests relevant to the command you have changed. + +``group_by_workflow/`` + End-to-end tests that exercise multi-step workflows: create an archive, + update it with new or changed files, extract files, and verify integrity. + +Run both groups:: + + python -m unittest tests/integration/python_tests/group_by_command/test_*.py + python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py + +Tests that require ``hsi`` or HPSS are automatically skipped when those tools +are unavailable. + Bash-based integration tests ============================ @@ -50,34 +122,93 @@ The bash tests are grouped by the machine or environment they require. Run from any machine -------------------- -The ``run_from_any`` directory contains bash tests that can be exercised -without a specific facility, although Globus authentication may still be part -of the workflow. The README specifically calls out reviewing the instructions in -``globus_auth.bash`` before running the related scripts. +The ``run_from_any/`` directory contains bash tests that can be exercised +on any machine, although Globus authentication is part of the +workflow. + +Before running these tests, review the instructions at the bottom of +``globus_auth.bash``, then authenticate and run the Globus tar-deletion test:: + + cd tests/integration/bash_tests/run_from_any/ + # Review globus_auth.bash and run with the appropriate parameters: + ./globus_auth.bash + ./test_globus_tar_deletion.bash Run from Perlmutter ------------------- -The ``run_from_perlmutter`` directory contains tests that depend on direct HPSS -access and Perlmutter-specific paths. +The ``run_from_perlmutter/`` directory contains tests that depend on direct +HPSS access and Perlmutter-specific paths. Update any hardcoded paths to +match your username before running. + +Steps:: + + cd tests/integration/bash_tests/run_from_perlmutter/ + + # Symlink-following test (edit paths for your username first) + time ./follow_symlinks.sh + # real 0m31.851s — No errors + + # HPSS update test + time ./test_update_non_empty_hpss.bash + # real 0m10.062s — No errors + + # Globus ls test + # 1. Log into globus.org + # 2. In File Manager, add both endpoints: + # - NERSC Perlmutter + # - Globus Tutorial Collection 1 + time ./test_ls_globus.bash # You may be prompted to paste an auth-code + # real 0m40.297s — No errors Run from Chrysalis ------------------ -The ``run_from_chrysalis`` directory contains tests that depend on Chrysalis, +The ``run_from_chrysalis/`` directory contains tests that depend on Chrysalis, Globus setup, and in some cases explicit cleanup of previous authentication state before rerunning. -Choosing the right scope -======================== +Steps:: + + cd tests/integration/bash_tests/run_from_chrysalis/ -Use the smallest test scope that matches the change: + # If not already done: + # 1. Log into globus.org + # 2. In File Manager add both endpoints: + # - LCRC Improv DTN + # - NERSC Perlmutter -* unit tests for isolated logic changes -* Python integration tests for command behavior -* bash and machine-specific tests for HPSS- or Globus-specific workflows -* release testing for end-to-end release validation; see - :doc:`release_testing` + # --- database_corruption.bash --- + # To reset completely before this test: + # Revoke consents: https://auth.globus.org/v2/web/consents + # > Globus Endpoint Performance Monitoring > rescind all + # + # Set up the required remote state (one-time): + rm ~/.zstash_globus_tokens.json + mkdir zstash_demo + echo 'file0 stuff' > zstash_demo/file0.txt + # NERSC_PERLMUTTER_ENDPOINT=6bdc7956-fc0f-4ad2-989c-7aa5ee643a79 + zstash create \ + --hpss=globus://6bdc7956-fc0f-4ad2-989c-7aa5ee643a79//global/homes//zstash/tests/test_database_corruption_setup23 \ + zstash_demo + # Paste the auth-code when prompted. This pre-authentication means the + # database_corruption test itself will not stop for user input. + rm -rf zstash_demo/ + # + # Pick a unique_id to avoid collisions with a previous run, or delete the + # remote directory on Perlmutter first: + # rm -rf /global/homes//zstash/tests/test_database_corruption_ + # + # Edit paths for your username, then run: + time ./database_corruption.bash + # Success count: 25 + # Fail count: 0 + # real 6m43.994s + + # --- symlinks.sh --- + # Edit paths for your username first. + time ./symlinks.sh + # real 0m1.346s — No errors GitHub Actions ============== @@ -91,3 +222,51 @@ GitHub Actions runs the machine-independent test suite in That workflow is the baseline CI safety net, while the machine-specific bash tests remain primarily manual. + +Testing for a release +===================== + +First, run on Chrysalis: + +Steps :: + + cd zstash + pytest tests/unit/test_*.py + python -m unittest tests/integration/python_tests/group_by_command/test_*.py + python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py + + cd tests/integration/bash_tests/run_from_any/ + ./globus_auth.bash unique_id chrysalis path_to_repo chrysalis_dst_basedir perlmutter_dst_basedir hpss_dst_basedir compy_dst_basedir + ./test_globus_tar_deletion.bash unique_id path_to_repo dst_basedir LCRC_IMPROV_DTN_ENDPOINT + + cd - + cd tests/integration/bash_tests/run_from_chrysalis/ + # You should still have Globus set up from the globus auth test. + time ./database_corruption.bash unique_id # NOTE: you will have to change out paths for your username + time ./symlinks.sh # NOTE: you will have to change out paths for your username + +Then, run on Perlmutter: + +Steps :: + + cd zstash + pytest tests/unit/test_*.py + python -m unittest tests/integration/python_tests/group_by_command/test_*.py + python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py + + cd - + cd tests/integration/bash_tests/run_from_perlmutter/ + time ./follow_symlinks.sh # NOTE: you will have to change out paths for your username + time ./test_update_non_empty_hpss.bash + # Log into globus.org + # Log into endpoints (NERSC Perlmutter, Globus Tutorial Collection 1) at globus.org: File Manager > Add the endpoints in the "Collection" fields + time ./test_ls_globus.bash # NOTE: You may be asked to paste an auth-code + +Lastly, run on Compy: + +Steps :: + + cd zstash + pytest tests/unit/test_*.py + python -m unittest tests/integration/python_tests/group_by_command/test_*.py + python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py diff --git a/docs/source/index.rst b/docs/source/index.rst index 2d5084bf..98ddd9ec 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -78,7 +78,6 @@ The contributor and maintainer documentation is organized under * :doc:`dev_guide/tar_tracking_modes` for tar tracking behavior in each storage mode * :doc:`dev_guide/testing` for the test layout and execution guidance -* :doc:`dev_guide/release_testing` for release validation steps * :doc:`dev_guide/releases` for the release process * :doc:`dev_guide/contributing_to_docs` for development environment setup and contribution workflow From 0e452ee57c11a510557909babe257d5ce793ab30 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 22 Jun 2026 17:51:12 -0700 Subject: [PATCH 06/10] Use code blocks --- .../source/dev_guide/contributing_to_docs.rst | 74 ++-- docs/source/dev_guide/project_standards.rst | 75 ++-- docs/source/dev_guide/testing.rst | 250 ++++++------ docs/source/user_guide/best_practices.rst | 108 +++--- docs/source/user_guide/database.rst | 20 +- docs/source/user_guide/getting_started.rst | 82 ++-- docs/source/user_guide/globus/intro.rst | 6 +- docs/source/user_guide/usage.rst | 357 ++++++++++-------- 8 files changed, 557 insertions(+), 415 deletions(-) diff --git a/docs/source/dev_guide/contributing_to_docs.rst b/docs/source/dev_guide/contributing_to_docs.rst index 9176b5eb..4bf4a68a 100644 --- a/docs/source/dev_guide/contributing_to_docs.rst +++ b/docs/source/dev_guide/contributing_to_docs.rst @@ -106,20 +106,26 @@ for a new repository. (Adapted from `Sphinx documentation on GitHub Create Sphinx conda environment (see above). -Create a new git branch (gh-pages): :: +Create a new git branch (gh-pages): - $ git branch gh-pages - $ git checkout gh-pages + .. code-block:: bash -Clear out anything from the main branch and start fresh :: + git branch gh-pages + git checkout gh-pages - $ git symbolic-ref HEAD refs/heads/gh-pages - $ rm .git/index - $ git clean -fdx +Clear out anything from the main branch and start fresh -Create documentation :: + .. code-block:: bash - $ sphinx-quickstart + git symbolic-ref HEAD refs/heads/gh-pages + rm .git/index + git clean -fdx + +Create documentation + + .. code-block:: bash + + sphinx-quickstart accept suggested default options, except :: @@ -129,40 +135,54 @@ Edit Makefile and change BUILDIR :: BUILDDIR = docs -Remove old build directory :: +Remove old build directory + + .. code-block:: bash + + rmdir build - $ rmdir build +Change the Sphinx theme to 'ReadTheDocs'. Edit 'source/conf.py and change -Change the Sphinx theme to 'ReadTheDocs'. Edit 'source/conf.py and change :: + .. code-block:: python - html_theme = 'alabaster' + html_theme = 'alabaster' -to :: +to - import sphinx_rtd_theme - html_theme = "sphinx_rtd_theme" - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + .. code-block:: python -Try building documentation :: + import sphinx_rtd_theme + html_theme = "sphinx_rtd_theme" + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - $ make html +Try building documentation + + .. code-block:: bash + + make html Create an empty .nojekyll file to indicate to Github.com that this -is not a Jekyll static website: :: +is not a Jekyll static website: + + .. code-block:: bash - $ touch .nojekyll + touch .nojekyll -Create a top-level re-direction file: :: +Create a top-level re-direction file: - $ vi index.html + .. code-block:: bash + + vi index.html with the following: :: -Commit and push back to Github: :: +Commit and push back to Github: + + .. code-block:: bash - $ git add . - $ git commit - $ git push origin gh-pages + git add . + git commit + git push origin gh-pages diff --git a/docs/source/dev_guide/project_standards.rst b/docs/source/dev_guide/project_standards.rst index f2363d5e..50d6ea8f 100644 --- a/docs/source/dev_guide/project_standards.rst +++ b/docs/source/dev_guide/project_standards.rst @@ -47,12 +47,16 @@ at the commit level before submitting code reviews. Helpful ``pre-commit`` Commands ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Install into your cloned repo :: +Install into your cloned repo - conda activate zstash_dev - pre-commit install + .. code-block:: bash -Automatically run all pre-commit hooks (just commit) :: + conda activate zstash_dev + pre-commit install + +Automatically run all pre-commit hooks (just commit) + + .. code-block:: bash # Tip: If there is an issue with pre-commit, you can bypass with the `--no-verify` flag. Please do NOT use this on a regular basis. git commit -m '...' @@ -62,14 +66,18 @@ Automatically run all pre-commit hooks (just commit) :: ``pre-commit`` Output -Manually run all pre-commit hooks :: +Manually run all pre-commit hooks + + .. code-block:: bash + + pre-commit run --all-files - pre-commit run --all-files +Run individual hook -Run individual hook :: + .. code-block:: bash - # Available hook ids: trailing-whitespace, end-of-file-fixer, check-yaml, black, isort, flake8, mypy - pre-commit run + # Available hook ids: trailing-whitespace, end-of-file-fixer, check-yaml, black, isort, flake8, mypy + pre-commit run Squash and Rebase Commits ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -93,36 +101,44 @@ How to squash and rebase commits Assuming that you followed :ref:`"(b) Development Environment" `: -1. Sync ``main`` with the main repo's ``main`` :: +1. Sync ``main`` with the main repo's ``main`` - git checkout main - git rebase /main - git push -f main + .. code-block:: bash -2. Get the SHA of the commit OR number of commits to rebase to :: + git checkout main + git rebase /main + git push -f main - git log --graph --decorate --pretty=oneline --abbrev-commit +2. Get the SHA of the commit OR number of commits to rebase to -3. Squash commits:: + .. code-block:: bash - git rebase -i [SHA] + git log --graph --decorate --pretty=oneline --abbrev-commit - # OR +3. Squash commits - git rebase -i HEAD~[NUMBER OF COMMITS] + .. code-block:: bash + + git rebase -i [SHA] + # OR: + git rebase -i HEAD~[NUMBER OF COMMITS] 4. Make sure your squashed commit messages are refined -5. Rebase branch onto ``main`` :: +5. Rebase branch onto ``main`` + + .. code-block:: bash - git checkout - git rebase main - git push -f + git checkout + git rebase main + git push -f -6. Force push to remote branch :: +6. Force push to remote branch - # You have to force push because the rebase rewrites the commit SHAs - git push -f + .. code-block:: bash + + # You have to force push because the rebase rewrites the commit SHAs + git push -f Source: https://blog.carbonfive.com/always-squash-and-rebase-your-git-commits/ @@ -142,10 +158,11 @@ Helpful Commands ~~~~~~~~~~~~~~~~ Run a tool - :: - # Available tool names: black, flake8, isort, mypy - . + .. code-block:: bash + + # Available tool names: black, flake8, isort, mypy + . .. _ci-cd: diff --git a/docs/source/dev_guide/testing.rst b/docs/source/dev_guide/testing.rst index 4c02ccbd..a0764c22 100644 --- a/docs/source/dev_guide/testing.rst +++ b/docs/source/dev_guide/testing.rst @@ -27,30 +27,34 @@ Recommended baseline workflow ============================= For a normal development change, start with the machine-independent checks from -the repository root:: +the repository root - rm -rf build - conda clean --all --y - conda env create -f conda/dev.yml -n zstash_dev_test - conda activate zstash_dev_test - pre-commit run --all-files - python -m pip install . - pytest tests/unit/test_*.py - python -m unittest tests/integration/python_tests/group_by_command/test_*.py - python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py + .. code-block:: bash -Example of expected output when all tests pass:: + rm -rf build + conda clean --all --y + conda env create -f conda/dev.yml -n zstash_dev_test + conda activate zstash_dev_test + pre-commit run --all-files + python -m pip install . + pytest tests/unit/test_*.py + python -m unittest tests/integration/python_tests/group_by_command/test_*.py + python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py - # pytest tests/unit/test_*.py - # 1 passed in 0.19s +Example of expected output when all tests pass - # python -m unittest tests/integration/python_tests/group_by_command/test_*.py - # Ran 69 tests in 327.570s - # OK + .. code-block:: bash - # python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py - # Ran 4 tests in 2.666s - # OK + # pytest tests/unit/test_*.py + # 1 passed in 0.19s + + # python -m unittest tests/integration/python_tests/group_by_command/test_*.py + # Ran 69 tests in 327.570s + # OK + + # python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py + # Ran 4 tests in 2.666s + # OK Some integration tests are skipped automatically on systems that do not have ``hsi`` or HPSS access. @@ -86,9 +90,11 @@ Current unit test files avoid linker conflicts, while preserving other variables such as ``HOME``. * For all other commands the loader variables are passed through unchanged. -Run the unit suite from the repository root:: +Run the unit suite from the repository root + + .. code-block:: bash - pytest tests/unit/test_*.py + pytest tests/unit/test_*.py Python integration tests ======================== @@ -106,10 +112,12 @@ into two sub-directories: End-to-end tests that exercise multi-step workflows: create an archive, update it with new or changed files, extract files, and verify integrity. -Run both groups:: +Run both groups - python -m unittest tests/integration/python_tests/group_by_command/test_*.py - python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py + .. code-block:: bash + + python -m unittest tests/integration/python_tests/group_by_command/test_*.py + python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py Tests that require ``hsi`` or HPSS are automatically skipped when those tools are unavailable. @@ -127,12 +135,14 @@ on any machine, although Globus authentication is part of the workflow. Before running these tests, review the instructions at the bottom of -``globus_auth.bash``, then authenticate and run the Globus tar-deletion test:: +``globus_auth.bash``, then authenticate and run the Globus tar-deletion test + + .. code-block:: bash - cd tests/integration/bash_tests/run_from_any/ - # Review globus_auth.bash and run with the appropriate parameters: - ./globus_auth.bash - ./test_globus_tar_deletion.bash + cd tests/integration/bash_tests/run_from_any/ + # Review globus_auth.bash and run with the appropriate parameters: + ./globus_auth.bash + ./test_globus_tar_deletion.bash Run from Perlmutter ------------------- @@ -141,25 +151,27 @@ The ``run_from_perlmutter/`` directory contains tests that depend on direct HPSS access and Perlmutter-specific paths. Update any hardcoded paths to match your username before running. -Steps:: +Steps - cd tests/integration/bash_tests/run_from_perlmutter/ + .. code-block:: bash - # Symlink-following test (edit paths for your username first) - time ./follow_symlinks.sh - # real 0m31.851s — No errors + cd tests/integration/bash_tests/run_from_perlmutter/ - # HPSS update test - time ./test_update_non_empty_hpss.bash - # real 0m10.062s — No errors + # Symlink-following test (edit paths for your username first) + time ./follow_symlinks.sh + # real 0m31.851s — No errors - # Globus ls test - # 1. Log into globus.org - # 2. In File Manager, add both endpoints: - # - NERSC Perlmutter - # - Globus Tutorial Collection 1 - time ./test_ls_globus.bash # You may be prompted to paste an auth-code - # real 0m40.297s — No errors + # HPSS update test + time ./test_update_non_empty_hpss.bash + # real 0m10.062s — No errors + + # Globus ls test + # 1. Log into globus.org + # 2. In File Manager, add both endpoints: + # - NERSC Perlmutter + # - Globus Tutorial Collection 1 + time ./test_ls_globus.bash # You may be prompted to paste an auth-code + # real 0m40.297s — No errors Run from Chrysalis ------------------ @@ -168,47 +180,49 @@ The ``run_from_chrysalis/`` directory contains tests that depend on Chrysalis, Globus setup, and in some cases explicit cleanup of previous authentication state before rerunning. -Steps:: - - cd tests/integration/bash_tests/run_from_chrysalis/ - - # If not already done: - # 1. Log into globus.org - # 2. In File Manager add both endpoints: - # - LCRC Improv DTN - # - NERSC Perlmutter - - # --- database_corruption.bash --- - # To reset completely before this test: - # Revoke consents: https://auth.globus.org/v2/web/consents - # > Globus Endpoint Performance Monitoring > rescind all - # - # Set up the required remote state (one-time): - rm ~/.zstash_globus_tokens.json - mkdir zstash_demo - echo 'file0 stuff' > zstash_demo/file0.txt - # NERSC_PERLMUTTER_ENDPOINT=6bdc7956-fc0f-4ad2-989c-7aa5ee643a79 - zstash create \ - --hpss=globus://6bdc7956-fc0f-4ad2-989c-7aa5ee643a79//global/homes//zstash/tests/test_database_corruption_setup23 \ - zstash_demo - # Paste the auth-code when prompted. This pre-authentication means the - # database_corruption test itself will not stop for user input. - rm -rf zstash_demo/ - # - # Pick a unique_id to avoid collisions with a previous run, or delete the - # remote directory on Perlmutter first: - # rm -rf /global/homes//zstash/tests/test_database_corruption_ - # - # Edit paths for your username, then run: - time ./database_corruption.bash - # Success count: 25 - # Fail count: 0 - # real 6m43.994s - - # --- symlinks.sh --- - # Edit paths for your username first. - time ./symlinks.sh - # real 0m1.346s — No errors +Steps + + .. code-block:: bash + + cd tests/integration/bash_tests/run_from_chrysalis/ + + # If not already done: + # 1. Log into globus.org + # 2. In File Manager add both endpoints: + # - LCRC Improv DTN + # - NERSC Perlmutter + + # --- database_corruption.bash --- + # To reset completely before this test: + # Revoke consents: https://auth.globus.org/v2/web/consents + # > Globus Endpoint Performance Monitoring > rescind all + # + # Set up the required remote state (one-time): + rm ~/.zstash_globus_tokens.json + mkdir zstash_demo + echo 'file0 stuff' > zstash_demo/file0.txt + # NERSC_PERLMUTTER_ENDPOINT=6bdc7956-fc0f-4ad2-989c-7aa5ee643a79 + zstash create \ + --hpss=globus://6bdc7956-fc0f-4ad2-989c-7aa5ee643a79//global/homes//zstash/tests/test_database_corruption_setup23 \ + zstash_demo + # Paste the auth-code when prompted. This pre-authentication means the + # database_corruption test itself will not stop for user input. + rm -rf zstash_demo/ + # + # Pick a unique_id to avoid collisions with a previous run, or delete the + # remote directory on Perlmutter first: + # rm -rf /global/homes//zstash/tests/test_database_corruption_ + # + # Edit paths for your username, then run: + time ./database_corruption.bash + # Success count: 25 + # Fail count: 0 + # real 6m43.994s + + # --- symlinks.sh --- + # Edit paths for your username first. + time ./symlinks.sh + # real 0m1.346s — No errors GitHub Actions ============== @@ -228,45 +242,51 @@ Testing for a release First, run on Chrysalis: -Steps :: +Steps + + .. code-block:: bash - cd zstash - pytest tests/unit/test_*.py - python -m unittest tests/integration/python_tests/group_by_command/test_*.py - python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py + cd zstash + pytest tests/unit/test_*.py + python -m unittest tests/integration/python_tests/group_by_command/test_*.py + python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py - cd tests/integration/bash_tests/run_from_any/ - ./globus_auth.bash unique_id chrysalis path_to_repo chrysalis_dst_basedir perlmutter_dst_basedir hpss_dst_basedir compy_dst_basedir - ./test_globus_tar_deletion.bash unique_id path_to_repo dst_basedir LCRC_IMPROV_DTN_ENDPOINT + cd tests/integration/bash_tests/run_from_any/ + ./globus_auth.bash unique_id chrysalis path_to_repo chrysalis_dst_basedir perlmutter_dst_basedir hpss_dst_basedir compy_dst_basedir + ./test_globus_tar_deletion.bash unique_id path_to_repo dst_basedir LCRC_IMPROV_DTN_ENDPOINT - cd - - cd tests/integration/bash_tests/run_from_chrysalis/ - # You should still have Globus set up from the globus auth test. - time ./database_corruption.bash unique_id # NOTE: you will have to change out paths for your username - time ./symlinks.sh # NOTE: you will have to change out paths for your username + cd - + cd tests/integration/bash_tests/run_from_chrysalis/ + # You should still have Globus set up from the globus auth test. + time ./database_corruption.bash unique_id # NOTE: you will have to change out paths for your username + time ./symlinks.sh # NOTE: you will have to change out paths for your username Then, run on Perlmutter: -Steps :: +Steps - cd zstash - pytest tests/unit/test_*.py - python -m unittest tests/integration/python_tests/group_by_command/test_*.py - python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py + .. code-block:: bash - cd - - cd tests/integration/bash_tests/run_from_perlmutter/ - time ./follow_symlinks.sh # NOTE: you will have to change out paths for your username - time ./test_update_non_empty_hpss.bash - # Log into globus.org - # Log into endpoints (NERSC Perlmutter, Globus Tutorial Collection 1) at globus.org: File Manager > Add the endpoints in the "Collection" fields - time ./test_ls_globus.bash # NOTE: You may be asked to paste an auth-code + cd zstash + pytest tests/unit/test_*.py + python -m unittest tests/integration/python_tests/group_by_command/test_*.py + python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py + + cd - + cd tests/integration/bash_tests/run_from_perlmutter/ + time ./follow_symlinks.sh # NOTE: you will have to change out paths for your username + time ./test_update_non_empty_hpss.bash + # Log into globus.org + # Log into endpoints (NERSC Perlmutter, Globus Tutorial Collection 1) at globus.org: File Manager > Add the endpoints in the "Collection" fields + time ./test_ls_globus.bash # NOTE: You may be asked to paste an auth-code Lastly, run on Compy: -Steps :: +Steps + + .. code-block:: bash - cd zstash - pytest tests/unit/test_*.py - python -m unittest tests/integration/python_tests/group_by_command/test_*.py - python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py + cd zstash + pytest tests/unit/test_*.py + python -m unittest tests/integration/python_tests/group_by_command/test_*.py + python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py diff --git a/docs/source/user_guide/best_practices.rst b/docs/source/user_guide/best_practices.rst index d5304426..1565a2d9 100644 --- a/docs/source/user_guide/best_practices.rst +++ b/docs/source/user_guide/best_practices.rst @@ -15,33 +15,39 @@ of the data transfer nodes (dtn<01..15>.nersc.gov). Also, because archiving large amount of data with zstash can take several days, it is recommended to invoke zstash within a UNIX `screen` session to which you can detach and re-attach without killing zstash. You -can access zstash on the data transfer nodes by loading the E3SM unified environment: :: +can access zstash on the data transfer nodes by loading the E3SM unified environment: - $ ssh dtn01.nersc.gov - $ screen - $ bash - $ source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh + .. code-block:: bash + + ssh dtn01.nersc.gov + screen + bash + source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh To detach from the screen session, use CTRL-A followed by D (for detach). You can then safely close your window. To re-attach to an existing session -later: :: +later: + + .. code-block:: bash - $ ssh dtn01.nersc.gov - $ screen -r + ssh dtn01.nersc.gov + screen -r Archive ------- Typically, you should consider archiving the entire directory structure of a simulation. The first time, this is accomplished with ``zstash create``. -For example: :: +For example: + + .. code-block:: bash - $ ssh dtn01.nersc.gov - $ screen -r - $ cd /global/cscratch1/sd/golaz/E3SM/simulations/20180129.DECKv1b_piControl.ne30_oEC.edison - $ mkdir zstash - $ zstash create --hpss=2018/E3SM_simulations/20180129.DECKv1b_piControl.ne30_oEC.edison \ - --maxsize 128 . 2>&1 | tee zstash/zstash_create_20190226.log + ssh dtn01.nersc.gov + screen -r + cd /global/cscratch1/sd/golaz/E3SM/simulations/20180129.DECKv1b_piControl.ne30_oEC.edison + mkdir zstash + zstash create --hpss=2018/E3SM_simulations/20180129.DECKv1b_piControl.ne30_oEC.edison \ + --maxsize 128 . 2>&1 | tee zstash/zstash_create_20190226.log The command above will archive the entire directory structure under `/global/cscratch1/sd/golaz/E3SM/simulations/20180129.DECKv1b_piControl.ne30_oEC.edison`. @@ -51,36 +57,42 @@ recommends file size between 100 and 500 GB for best performance. If your model output has been reorganized using the CIME short-term archive utility, you can easily archive only a subset of the restart files to conserve space. For example, to **archive -restart files every 5 years** only: :: +restart files every 5 years** only: - $ ssh dtn01.nersc.gov - $ screen -r - $ cd /global/cscratch1/sd/golaz/E3SM/simulations/20180129.DECKv1b_piControl.ne30_oEC.edison - $ mkdir zstash - $ zstash create --hpss=2018/E3SM_simulations/20180129.DECKv1b_piControl.ne30_oEC.edison \ - --exclude="archive/rest/???[!05]-*/" \ - --maxsize 128 . 2>&1 | tee zstash/zstash_create_20190226.log + .. code-block:: bash + + ssh dtn01.nersc.gov + screen -r + cd /global/cscratch1/sd/golaz/E3SM/simulations/20180129.DECKv1b_piControl.ne30_oEC.edison + mkdir zstash + zstash create --hpss=2018/E3SM_simulations/20180129.DECKv1b_piControl.ne30_oEC.edison \ + --exclude="archive/rest/???[!05]-*/" \ + --maxsize 128 . 2>&1 | tee zstash/zstash_create_20190226.log Update ------ You can also add newly created files to an existing archive, or restart archiving after a -failure using the ``zstash update`` functionality: :: +failure using the ``zstash update`` functionality: + + .. code-block:: bash - $ ssh dtn01.nersc.gov - $ screen -r - $ cd /global/cscratch1/sd/golaz/E3SM/simulations/20180129.DECKv1b_piControl.ne30_oEC.edison - $ mkdir zstash - $ zstash update --hpss=2018/E3SM_simulations/20180129.DECKv1b_piControl.ne30_oEC.edison \ - --exclude="archive/rest/???[!05]-*/" 2>&1 | tee zstash/zstash_update_20190226.log + ssh dtn01.nersc.gov + screen -r + cd /global/cscratch1/sd/golaz/E3SM/simulations/20180129.DECKv1b_piControl.ne30_oEC.edison + mkdir zstash + zstash update --hpss=2018/E3SM_simulations/20180129.DECKv1b_piControl.ne30_oEC.edison \ + --exclude="archive/rest/???[!05]-*/" 2>&1 | tee zstash/zstash_update_20190226.log Check ----- After archiving or updating, it is **highly recommended** that you verify the integrity -of the tar files. The safest way to do so is go to a new, empty directory and run: :: +of the tar files. The safest way to do so is go to a new, empty directory and run: - $ zstash check --hpss=2018/E3SM_simulations/20180129.DECKv1b_piControl.ne30_oEC.edison + .. code-block:: bash + + zstash check --hpss=2018/E3SM_simulations/20180129.DECKv1b_piControl.ne30_oEC.edison ``zstash check`` will download the tar archives to the local disk cache (under the zstash/ subdirectory) and verify the md5 checksum of every file against the @@ -125,10 +137,12 @@ With the second error, you might see something like: :: ReadError: unexpected end of data This seems to be caused by the filesystem. Simply run ``zstash check`` again. -To save time, like ``zstash extract``, you can check for specific files or tar archives: :: +To save time, like ``zstash extract``, you can check for specific files or tar archives: + + .. code-block:: bash - $ zstash check --hpss=/path/to/hpss/archive "archive/ocn/hist/mpaso.hist.am.timeSeriesStatsMonthly.1892-04-01.nc" - $ zstash check --hpss=/path/to/hpss/archive "000012.tar" + zstash check --hpss=/path/to/hpss/archive "archive/ocn/hist/mpaso.hist.am.timeSeriesStatsMonthly.1892-04-01.nc" + zstash check --hpss=/path/to/hpss/archive "000012.tar" Compy/Anvil =========== @@ -146,13 +160,15 @@ Archive ------- Starting with v0.4, zstash supports the creation of local archives only (using the -``--hpss=none`` command line option). For example :: +``--hpss=none`` command line option). For example - $ screen - $ cd /compyfs/gola749/E3SM_simulations/20191216.alpha20.piControl.ne30_r05_oECv3_ICG.compy - $ mkdir zstash - $ zstash create --hpss=none --maxsize 128 . 2>&1 | tee zstash/zstash_create_20200224.log - ctrl-a d # to disconnect from screen session + .. code-block:: bash + + screen + cd /compyfs/gola749/E3SM_simulations/20191216.alpha20.piControl.ne30_r05_oECv3_ICG.compy + mkdir zstash + zstash create --hpss=none --maxsize 128 . 2>&1 | tee zstash/zstash_create_20200224.log + ctrl-a d # to disconnect from screen session .. _globus-compy: @@ -205,8 +221,10 @@ Transfer all zstash files to NERSC HPSS using Globus. Check ----- -Once archiving is complete, run ``zstash check`` on NERSC to verify integrity of the archive: :: +Once archiving is complete, run ``zstash check`` on NERSC to verify integrity of the archive: + + .. code-block:: bash - $ ssh dtn01.nersc.gov - $ cd - $ zstash check --hpss= + ssh dtn01.nersc.gov + cd + zstash check --hpss= diff --git a/docs/source/user_guide/database.rst b/docs/source/user_guide/database.rst index c2852470..6e32a4b7 100644 --- a/docs/source/user_guide/database.rst +++ b/docs/source/user_guide/database.rst @@ -59,20 +59,26 @@ Exploring content Direct interaction with the database can be useful to explore content of an archive, beyond what might be available with :ref:`zstash list`. -To list **all the files** in an archive: :: +To list **all the files** in an archive: - $ cd - $ sqlite3 zstash/index.db "select * from files;" + .. code-block:: bash + + cd + sqlite3 zstash/index.db "select * from files;" For each file, the following information will be printed :: file # | path | size | modification time |md5 checksum |tar archive | offset (within tar) -To list **files matching a specified pattern** (for example \*/run/\*.nc): :: +To list **files matching a specified pattern** (for example \*/run/\*.nc): + + .. code-block:: bash + + sqlite3 zstash/index.db "select * from files where name glob '*/run/*.nc';" - $ sqlite3 zstash/index.db "select * from files where name glob '*/run/*.nc';" +To list **all the files in a specific tar fole** (for example 00000a.tar): -To list **all the files in a specific tar fole** (for example 00000a.tar): :: + .. code-block:: bash - $ sqlite3 zstash/index.db "select * from files where tar is '00000a.tar';" + sqlite3 zstash/index.db "select * from files where tar is '00000a.tar';" diff --git a/docs/source/user_guide/getting_started.rst b/docs/source/user_guide/getting_started.rst index b4607533..e787fc20 100644 --- a/docs/source/user_guide/getting_started.rst +++ b/docs/source/user_guide/getting_started.rst @@ -18,44 +18,44 @@ a conda environment that pulls together Python and other E3SM tools such as The paths to ``e3sm_unified`` activation scripts are machine dependent: **Compy** - :: + .. code-block:: bash - source /share/apps/E3SM/conda_envs/load_latest_e3sm_unified_compy.sh + source /share/apps/E3SM/conda_envs/load_latest_e3sm_unified_compy.sh **NERSC** - :: + .. code-block:: bash - source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh + source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh **Anvil** - :: + .. code-block:: bash - source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_anvil.sh + source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_anvil.sh **Chrysalis** - :: + .. code-block:: bash - source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh + source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh **Cooley** - :: + .. code-block:: bash - source /lus/theta-fs0/projects/ccsm/acme/tools/e3sm-unified/load_latest_e3sm_unified_cooley.sh + source /lus/theta-fs0/projects/ccsm/acme/tools/e3sm-unified/load_latest_e3sm_unified_cooley.sh **acme1** - :: + .. code-block:: bash - source /p/user_pub/e3sm_unified/envs/load_latest_e3sm_unified_acme1.sh + source /p/user_pub/e3sm_unified/envs/load_latest_e3sm_unified_acme1.sh **Andes** - :: + .. code-block:: bash - source /gpfs/alpine/proj-shared/cli115/e3sm-unified/load_latest_e3sm_unified_andes.sh + source /gpfs/alpine/proj-shared/cli115/e3sm-unified/load_latest_e3sm_unified_andes.sh Change ``.sh`` to ``.csh`` for ``csh`` shells. @@ -84,27 +84,27 @@ If the system doesn't come with conda pre-installed, follow these instructions: 1. Download Mambaforge Linux - :: + .. code-block:: bash - wget https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh + wget https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh MacOS x86_64 (note that ``zstash`` is not supported on MacOS, but it may be useful to contribute to the documentation on MacOS) - :: + .. code-block:: bash - wget https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-MacOSX-x86_64.sh + wget https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-MacOSX-x86_64.sh 2. Install Mambaforge Linux - :: + .. code-block:: bash - bash ./Mambaforge-Linux-x86_64.sh + bash ./Mambaforge-Linux-x86_64.sh MacOS x86_64 - :: + .. code-block:: bash - bash ./Mambaforge-MacOSX-x86_64.sh + bash ./Mambaforge-MacOSX-x86_64.sh When you see: :: @@ -117,7 +117,7 @@ If the system doesn't come with conda pre-installed, follow these instructions: 3. If you are working on a machine/network that intercepts SSL communications (such as acme1), you will get an SSL error unless you disable the SSL verification: - :: + .. code-block:: bash conda config --set ssl_verify false binstar config --set ssl_verify False @@ -131,9 +131,11 @@ create a **(b) Development Environment**. Installation using mamba ------------------------ -First, make sure that you're using ``bash``. :: +First, make sure that you're using ``bash``. - bash + .. code-block:: bash + + bash You must have a conda base enviornment installed as well. See :ref:`"Installation in a Conda Environment" ` section above for @@ -153,9 +155,11 @@ Create a new conda environment with ``zstash`` installed and activate it: :: conda activate zstash_env Or (less recommended because of potential conflicts) you can install ``zstash`` -in an existing environment. :: +in an existing environment. + + .. code-block:: bash - mamba install zppy + mamba install zppy Installation on NERSC --------------------- @@ -170,9 +174,11 @@ Updating -------- If you **installed into your own conda environment** (e.g., not through the -unified environment), you can update ``zstash`` by doing the following: :: +unified environment), you can update ``zstash`` by doing the following: + + .. code-block:: bash - mamba update zstash + mamba update zstash .. _dev-env: @@ -191,7 +197,7 @@ Furthermore, the dev environment includes quality assurance (QA) tools such as c 2. Clone your fork and keep it in sync with the main repo's ``main`` - :: + .. code-block:: bash # Go to https://github.com/E3SM-Project/zstash # Click "Fork" in the upper right hand corner. This will fork the main repo. @@ -207,7 +213,7 @@ Furthermore, the dev environment includes quality assurance (QA) tools such as c or if you already have a clone of your fork, rebase your fork on the main repo's ``main`` to keep it in sync: - :: + .. code-block:: bash # Add the main repo as a remote. # You can call it anything but "upstream" is recommended. @@ -231,19 +237,19 @@ Furthermore, the dev environment includes quality assurance (QA) tools such as c Checkout a new branch from ``main``: - :: + .. code-block:: bash git checkout -b /main 3. Remove any cached conda packages. This will ensure that you always get the latest packages. - :: + .. code-block:: bash mamba clean --all 4. Enter the fork's clone. - :: + .. code-block:: bash cd zstash @@ -252,26 +258,26 @@ Furthermore, the dev environment includes quality assurance (QA) tools such as c - Tip: Add the flag ``-n `` to customize the name of the environment - :: + .. code-block:: bash mamba env create -f conda/dev.yml conda activate zstash_dev 6. Install ``pre-commit``. - :: + .. code-block:: bash pre-commit install 7. Make the desired changes to ``zstash``, then rebuild and install with: - :: + .. code-block:: bash pip install . 8. Commit changes and make sure ``pre-commit`` checks pass - :: + .. code-block:: bash git commit -m "commit-message" diff --git a/docs/source/user_guide/globus/intro.rst b/docs/source/user_guide/globus/intro.rst index ac6f34ba..df84ec4c 100644 --- a/docs/source/user_guide/globus/intro.rst +++ b/docs/source/user_guide/globus/intro.rst @@ -35,9 +35,11 @@ For a first Globus-based archive, the safest approach is: 4. Start with a small archive so you can confirm that authentication, endpoint activation, and path selection are correct. -If you are creating a new archive, a minimal first test looks like:: +If you are creating a new archive, a minimal first test looks like - zstash create --hpss=globus://nersc/~/test/my_archive . + .. code-block:: bash + + zstash create --hpss=globus://nersc/~/test/my_archive . After the transfer completes, verify it with ``zstash check`` or retrieve a small file with ``zstash extract``. diff --git a/docs/source/user_guide/usage.rst b/docs/source/user_guide/usage.rst index 5d36e999..74066a49 100644 --- a/docs/source/user_guide/usage.rst +++ b/docs/source/user_guide/usage.rst @@ -26,9 +26,11 @@ If running on Cori, it is preferable to run from ``$CSCRATCH`` rather than Create ====== -To create a new zstash archive: :: +To create a new zstash archive: - $ zstash create --hpss= + .. code-block:: bash + + zstash create --hpss= where @@ -71,17 +73,21 @@ Basic example ------------- To **archive** output from an E3SM simulation located -under `$CSCRATCH/ACME_simulations/20170731.F20TR.ne30_ne30.edison`:: +under `$CSCRATCH/ACME_simulations/20170731.F20TR.ne30_ne30.edison` + + .. code-block:: bash + + cd $CSCRATCH/ACME_simulations/20170731.F20TR.ne30_ne30.edison + zstash create --hpss=test/E3SM_simulations/20170731.F20TR.ne30_ne30.edison . - $ cd $CSCRATCH/ACME_simulations/20170731.F20TR.ne30_ne30.edison - $ zstash create --hpss=test/E3SM_simulations/20170731.F20TR.ne30_ne30.edison . +Once done, you should see the archive files on hsi: -Once done, you should see the archive files on hsi: :: + .. code-block:: bash - $ hsi - > cd test/E3SM_simulations/20170731.F20TR.ne30_ne30.edison - > ls - 000000.tar index.db + hsi + cd test/E3SM_simulations/20170731.F20TR.ne30_ne30.edison + ls + # 000000.tar index.db The data from this test simulation is small, so in this case there is only a single tar file (000000.tar) and the index database (index.db). @@ -91,17 +97,21 @@ Examples excluding some files You may decide that certain files do not need to be archived. For example, if you want to **exclude \*.o and \*.mod files** under the build -subdirectory: :: +subdirectory: - $ cd $CSCRATCH/ACME_simulations/20170731.F20TR.ne30_ne30.edison - $ zstash create --hpss=test/ACME_simulations/20170731.F20TR.ne30_ne30.edison \ + .. code-block:: bash + + cd $CSCRATCH/ACME_simulations/20170731.F20TR.ne30_ne30.edison + zstash create --hpss=test/ACME_simulations/20170731.F20TR.ne30_ne30.edison \ --exclude="build/*/*.o","build/*/*.mod" . Or you may decide that you only want to **archive restart files every 5 years** -to conserve storage space: :: +to conserve storage space: + + .. code-block:: bash - $ cd $CSCRATCH/ACME_simulations/20170731.F20TR.ne30_ne30.edison - $ zstash create --hpss=test/ACME_simulations/20170731.F20TR.ne30_ne30.edison \ + cd $CSCRATCH/ACME_simulations/20170731.F20TR.ne30_ne30.edison + zstash create --hpss=test/ACME_simulations/20170731.F20TR.ne30_ne30.edison \ --exclude="archive/rest/???[!05]-*/" . This exclude pattern will skip all restart subdirectories under the short-term archive, @@ -110,18 +120,22 @@ except for those with years ending in '0' or '5'. Example with Globus ------------------- If you run zstash on the system without the HPSS file system, but has a `Globus `_ endpoint set up, -you can use a Globus URL: :: +you can use a Globus URL: - $ cd $CSCRATCH/ACME_simulations/20170731.F20TR.ne30_ne30.anvil - $ zstash create --hpss=globus://9cd89cfd-6d04-11e5-ba46-22000b92c6ec/~/test/E3SM_simulations/20170731.F20TR.ne30_ne30.anvil . + .. code-block:: bash + + cd $CSCRATCH/ACME_simulations/20170731.F20TR.ne30_ne30.anvil + zstash create --hpss=globus://9cd89cfd-6d04-11e5-ba46-22000b92c6ec/~/test/E3SM_simulations/20170731.F20TR.ne30_ne30.anvil . 9cd89cfd-6d04-11e5-ba46-22000b92c6ec is the NERSC HPSS Globus endpoint UUID. Two names ``nersc`` and ``alcf`` are recognized by zstash and substituted internally with a corresponding Globus UUID for the NERSC HPSS Globus endpoint (9cd89cfd-6d04-11e5-ba46-22000b92c6ec) and the ALCF HPSS Globus endpoint (de463ec4-6d04-11e5-ba46-22000b92c6ec) endpoint. -If you want to store zstash archive on these two remote HPSS file systems, you can use the names instead of UUIDs: :: +If you want to store zstash archive on these two remote HPSS file systems, you can use the names instead of UUIDs: + + .. code-block:: bash - $ zstash create --hpss=globus://nersc/~/test/E3SM_simulations/20170731.F20TR.ne30_ne30.anvil . + zstash create --hpss=globus://nersc/~/test/E3SM_simulations/20170731.F20TR.ne30_ne30.anvil . .. note:: If you are a new Globus user, you should first do a small transfer to test functionality. @@ -138,9 +152,11 @@ Check Note: Most of the commands for this are the same for ``zstash extract`` and ``zstash ls``. To verify that your files were uploaded on HPSS successfully, -go to a **new, empty directory** and run: :: +go to a **new, empty directory** and run: + + .. code-block:: bash - $ zstash check --hpss= [--workers=] [--cache=] [--keep] [-v] [files] + zstash check --hpss= [--workers=] [--cache=] [--keep] [-v] [files] where @@ -199,41 +215,47 @@ You may need to reupload it via ``zstash create``. Please contact the zstash development team, we're working on identifying what causes these issues. -Example using ``--hpss=none``:: - - $ mkdir zstash_demo - $ echo 'file0 stuff' > zstash_demo/file0.txt - $ zstash create --hpss=none zstash_demo - $ ls zstash_demo/ - file0.txt zstash - $ ls zstash_demo/zstash/ - 000000.tar index.db - $ cd zstash_demo - $ zstash check --hpss=none - INFO: Opening tar archive zstash/000000.tar - INFO: Checking file0.txt - INFO: No failures detected when checking the files. If you have a log file, run "grep -i Exception " to double check. - -Example usage of ``--tars``:: - - # Starting at 00005a until the end - zstash check --tars=00005a- - # Starting from the beginning to 00005a (included) - zstash check --tars=-00005a - # Specific range - zstash check --tars=00005a-00005c - # Selected tar files - zstash check --tars=00003e,00004e,000059 - # Mix and match - zstash check --tars=000030-00003e,00004e,00005a- +Example using ``--hpss=none`` + + .. code-block:: bash + + mkdir zstash_demo + echo 'file0 stuff' > zstash_demo/file0.txt + zstash create --hpss=none zstash_demo + ls zstash_demo/ + # file0.txt zstash + ls zstash_demo/zstash/ + # 000000.tar index.db + cd zstash_demo + zstash check --hpss=none + # INFO: Opening tar archive zstash/000000.tar + # INFO: Checking file0.txt + # INFO: No failures detected when checking the files. If you have a log file, run "grep -i Exception " to double check. + +Example usage of ``--tars`` + + .. code-block:: bash + + # Starting at 00005a until the end + zstash check --tars=00005a- + # Starting from the beginning to 00005a (included) + zstash check --tars=-00005a + # Specific range + zstash check --tars=00005a-00005c + # Selected tar files + zstash check --tars=00003e,00004e,000059 + # Mix and match + zstash check --tars=000030-00003e,00004e,00005a- Update ====== -An existing zstash archive can be updated to add new or modified files: :: +An existing zstash archive can be updated to add new or modified files: - $ cd - $ zstash update --hpss= [--cache=] [--dry-run] [--exclude] [--keep] [-v] + .. code-block:: bash + + cd + zstash update --hpss= [--cache=] [--dry-run] [--exclude] [--keep] [-v] where @@ -263,23 +285,29 @@ Example ------- Following the '**zstash create**' example above, we now run zstash again with the -'**update**' functionality: :: +'**update**' functionality: + + .. code-block:: bash - $ cd $CSCRATCH/ACME_simulations/20170731.F20TR.ne30_ne30.edison - $ zstash update --hpss=test/ACME_simulations/20170731.F20TR.ne30_ne30.edison + cd $CSCRATCH/ACME_simulations/20170731.F20TR.ne30_ne30.edison + zstash update --hpss=test/ACME_simulations/20170731.F20TR.ne30_ne30.edison Since nothing has changed, zstash simply returns :: INFO: Nothing to update -Now, let's add a new file :: +Now, let's add a new file - $ mkdir new - $ echo "This is a new file..." > new/file.txt + .. code-block:: bash -and rerun zstash update :: + mkdir new + echo "This is a new file..." > new/file.txt - $ zstash update --hpss=test/ACME_simulations/20170731.F20TR.ne30_ne30.edison +and rerun zstash update + + .. code-block:: bash + + zstash update --hpss=test/ACME_simulations/20170731.F20TR.ne30_ne30.edison Zstash recognizes the presence of a new file and adds it to the archive: :: @@ -302,10 +330,12 @@ Extract Note: Most of the commands for this are the same for ``zstash check`` and ``zstash ls``. -To extract files from an existing zstash archive into current : :: +To extract files from an existing zstash archive into current : + + .. code-block:: bash - $ cd - $ zstash extract --hpss= [--workers=] [--cache=] [--keep] [-v] [files] + cd + zstash extract --hpss= [--workers=] [--cache=] [--keep] [-v] [files] where @@ -375,55 +405,62 @@ Extracting a single file by its full path ``archive/logs/atm.log.8229335.180130- DEBUG: Closing index database If the index database is already in the local disk cache (zstash/index.db), you can leave out the ``--hpss`` -path. For example: :: +path. For example: - $ zstash extract archive/logs/atm.log.8229335.180130-143234.gz + .. code-block:: bash + + zstash extract archive/logs/atm.log.8229335.180130-143234.gz However, recall that wildcards are supported, so this full path isn't needed when using them. Instead, you could download files matching ``"*atm.log.8229335.180130-143234.gz*"``. Note -the use of double quotes (") to avoid shell level substitution. :: +the use of double quotes (") to avoid shell level substitution. - $ zstash extract --hpss=/home/g/golaz/2018/E3SM_simulations/20180129.DECKv1b_piControl.ne30_oEC.edison "*atm.log.8229335.180130-143234.gz*" - DEBUG: Opening index database - DEBUG: Running zstash extract - DEBUG: Local path : /global/cscratch1/sd/golaz/ACME_simulations/20180129.DECKv1b_piControl.ne30_oEC.edison - DEBUG: HPSS path : /home/g/golaz/2018/E3SM_simulations/20180129.DECKv1b_piControl.ne30_oEC.edison - DEBUG: Max size : 274877906944 - DEBUG: Keep local tar files : False - INFO: Opening tar archive zstash/000018.tar - INFO: Extracting archive/logs/atm.log.8229335.180130-143234.gz - DEBUG: Valid md5: e8161bba53500848dc917258d1d8f56a archive/logs/atm.log.8229335.180130-143234.gz - DEBUG: Closing tar archive zstash/000018.tar - INFO: Opening tar archive zstash/000047.tar - INFO: Extracting case_scripts/logs/atm.log.8229335.180130-143234.gz - DEBUG: Valid md5: e8161bba53500848dc917258d1d8f56a case_scripts/logs/atm.log.8229335.180130-143234.gz - DEBUG: Closing tar archive zstash/000047.tar - DEBUG: Closing index database + .. code-block:: bash + + zstash extract --hpss=/home/g/golaz/2018/E3SM_simulations/20180129.DECKv1b_piControl.ne30_oEC.edison "*atm.log.8229335.180130-143234.gz*" + # DEBUG: Opening index database + # DEBUG: Running zstash extract + # DEBUG: Local path : /global/cscratch1/sd/golaz/ACME_simulations/20180129.DECKv1b_piControl.ne30_oEC.edison + # DEBUG: HPSS path : /home/g/golaz/2018/E3SM_simulations/20180129.DECKv1b_piControl.ne30_oEC.edison + # DEBUG: Max size : 274877906944 + # DEBUG: Keep local tar files : False + # INFO: Opening tar archive zstash/000018.tar + # INFO: Extracting archive/logs/atm.log.8229335.180130-143234.gz + # DEBUG: Valid md5: e8161bba53500848dc917258d1d8f56a archive/logs/atm.log.8229335.180130-143234.gz + # DEBUG: Closing tar archive zstash/000018.tar + # INFO: Opening tar archive zstash/000047.tar + # INFO: Extracting case_scripts/logs/atm.log.8229335.180130-143234.gz + # DEBUG: Valid md5: e8161bba53500848dc917258d1d8f56a case_scripts/logs/atm.log.8229335.180130-143234.gz + # DEBUG: Closing tar archive zstash/000047.tar + # DEBUG: Closing index database In this particular example, the pattern matches two specific files, one under `archive/logs/` and another one under `case_scripts/logs/`. If you didn't intend to retrieve both of them, a more efficient approach would have been to first identify the desired files with 'zstash ls'. Another example of wildcards would be to retrieve all **cam.h0** (monthly atmosphere output files) -between **years 0030 and 0069** for the DECKv1 piControl simulation. The zstash command would be: :: +between **years 0030 and 0069** for the DECKv1 piControl simulation. The zstash command would be: + + .. code-block:: bash - $ zstash extract --hpss=/home/g/golaz/2018/E3SM_simulations/20180129.DECKv1b_piControl.ne30_oEC.edison \ - "*.cam.h0.00[3-6]?-??.nc" + zstash extract --hpss=/home/g/golaz/2018/E3SM_simulations/20180129.DECKv1b_piControl.ne30_oEC.edison "*.cam.h0.00[3-6]?-??.nc" You may specify the cache with the ``--cache`` option. Notice that there is no need to include -``--keep`` when not using HPSS. :: +``--keep`` when not using HPSS + + .. code-block:: bash - $ zstash extract --hpss=none \ - --cache=/p/user_pub/e3sm/archive/1_1/BGC-v1/20181217.BCRC_CNPCTC20TR_OIBGC.ne30_oECv3.edison \ - "*cam.h3.1906-01-*-*.nc" + zstash extract --hpss=none --cache=/p/user_pub/e3sm/archive/1_1/BGC-v1/20181217.BCRC_CNPCTC20TR_OIBGC.ne30_oECv3.edison "*cam.h3.1906-01-*-*.nc" Example with Globus ------------------- -To extract from the archive created with Globus in the ``zstash create`` example, you would run: :: +To extract from the archive created with Globus in the ``zstash create`` example, you would run: - $ zstash extract --hpss=globus://9cd89cfd-6d04-11e5-ba46-22000b92c6ec/~/test/E3SM_simulations/20170731.F20TR.ne30_ne30.anvil + .. code-block:: bash + + zstash extract --hpss=globus://9cd89cfd-6d04-11e5-ba46-22000b92c6ec/~/test/E3SM_simulations/20170731.F20TR.ne30_ne30.anvil .. _zstash-list: @@ -432,9 +469,11 @@ List Note: Most of the commands for this are the same for ``zstash extract`` and ``zstash check``. -You can view the files in an existing zstash archive: :: +You can view the files in an existing zstash archive: + + .. code-block:: bash - $ zstash ls --hpss= [-l] [--cache=] [--tars] [-v] [files] + zstash ls --hpss= [-l] [--cache=] [--tars] [-v] [files] where @@ -451,86 +490,100 @@ where to avoid shell substitution. * Names of specific tar archives to list all files within these tar archives. -Below is an example. Note the names of the columns: :: +Below is an example. Note the names of the columns: - $ zstash ls -l --hpss=/home/g/golaz/2018/E3SM_simulations/20180129.DECKv1b_piControl.ne30_oEC.edison "*atm.log.8229335.180130-143234.gz*" - DEBUG: Opening index database - DEBUG: Running zstash ls - DEBUG: HPSS path : /home/g/golaz/2018/E3SM_simulations/20180129.DECKv1b_piControl.ne30_oEC.edison - id name size mtime md5 tar offset - 30482 archive/logs/atm.log.8229335.180130-143234.gz 20156521 2018-02-01 10:02:35 e8161bba53500848dc917258d1d8f56a 000018.tar 131697281536 - 51608 case_scripts/logs/atm.log.8229335.180130-143234.gz 20156521 2018-02-01 10:02:52 e8161bba53500848dc917258d1d8f56a 000047.tar 202381473280 + .. code-block:: bash -Below is an example of using ``ls`` to look at the tars in addition to the files: :: + zstash ls -l --hpss=/home/g/golaz/2018/E3SM_simulations/20180129.DECKv1b_piControl.ne30_oEC.edison "*atm.log.8229335.180130-143234.gz*" + # DEBUG: Opening index database + # DEBUG: Running zstash ls + # DEBUG: HPSS path : /home/g/golaz/2018/E3SM_simulations/20180129.DECKv1b_piControl.ne30_oEC.edison + # id name size mtime md5 tar offset + # 30482 archive/logs/atm.log.8229335.180130-143234.gz 20156521 2018-02-01 10:02:35 e8161bba53500848dc917258d1d8f56a 000018.tar 131697281536 + # 51608 case_scripts/logs/atm.log.8229335.180130-143234.gz 20156521 2018-02-01 10:02:52 e8161bba53500848dc917258d1d8f56a 000047.tar 202381473280 - $ mkdir source_directory - $ touch source_directory/file0.txt - $ zstash create --hpss=hpss_archive source_directory - INFO: Gathering list of files to archive - INFO: Creating new tar archive 000000.tar - INFO: Archiving file0.txt - INFO: tar name=000000.tar, tar size=10240, tar md5=97d3e0ffaff4880251c77699d7438fe2 - INFO: Transferring file to HPSS: zstash/000000.tar - INFO: Transferring file to HPSS: zstash/index.db +Below is an example of using ``ls`` to look at the tars in addition to the files: - $ zstash ls --hpss=hpss_archive --tars - INFO: Transferring file from HPSS: zstash/index.db - file0.txt + .. code-block:: bash - Tars: - 000000.tar + mkdir source_directory + touch source_directory/file0.txt + zstash create --hpss=hpss_archive source_directory + # INFO: Gathering list of files to archive + # INFO: Creating new tar archive 000000.tar + # INFO: Archiving file0.txt + # INFO: tar name=000000.tar, tar size=10240, tar md5=97d3e0ffaff4880251c77699d7438fe2 + # INFO: Transferring file to HPSS: zstash/000000.tar + # INFO: Transferring file to HPSS: zstash/index.db + + zstash ls --hpss=hpss_archive --tars + # INFO: Transferring file from HPSS: zstash/index.db + # file0.txt + + # Tars: + # 000000.tar .. warning:: Running ``zstash ls`` outside the source directory (the directory you're archiving) is not advised. ``zstash`` will only retrieve ``index.db`` from the HPSS archive if a local archive (cache) is not present. -Example 1 -- changing the HPSS archive: :: +Example 1 -- changing the HPSS archive: + + .. code-block:: bash - $ zstash create --hpss=hpss_archive source_directory # Creates an HPSS archive named `hpss_archive` and a local archive (cache) `source_directory/zstash`. - $ zstash ls --hpss=hpss_archive # List the contents of `hpss_archive` and creates a cache `zstash` at the same level of `source_directory`. + zstash create --hpss=hpss_archive source_directory # Creates an HPSS archive named `hpss_archive` and a local archive (cache) `source_directory/zstash`. + zstash ls --hpss=hpss_archive # List the contents of `hpss_archive` and creates a cache `zstash` at the same level of `source_directory`. # Add `source_directory/new_file.txt` - $ zstash create --hpss=different_hpss_archive source_directory # Create a different HPSS archive of the source directory. This overwrites the local archive (cache) `source_directory/zstash`. - $ zstash ls --hpss=different_hpss_archive # `new_file.txt` will NOT be shown. The existing cache `zstash` (same level as `source_directory`) is being used. - $ rm -rf zstash # Delete the cache. (You could instead change to another directory). - $ zstash ls --hpss=different_hpss_archive # `new_file.txt` will be shown. + zstash create --hpss=different_hpss_archive source_directory # Create a different HPSS archive of the source directory. This overwrites the local archive (cache) `source_directory/zstash`. + zstash ls --hpss=different_hpss_archive # `new_file.txt` will NOT be shown. The existing cache `zstash` (same level as `source_directory`) is being used. + rm -rf zstash # Delete the cache. (You could instead change to another directory). + zstash ls --hpss=different_hpss_archive # `new_file.txt` will be shown. -Example 2 -- updating the HPSS archive: :: +Example 2 -- updating the HPSS archive: - $ zstash create --hpss=hpss_archive source_directory # Creates an HPSS archive named `hpss_archive` and a local archive (cache) `source_directory/zstash`. - $ zstash ls --hpss=hpss_archive # List the contents of `hpss_archive` and creates a cache `zstash` at the same level of `source_directory`. + .. code-block:: bash + + zstash create --hpss=hpss_archive source_directory # Creates an HPSS archive named `hpss_archive` and a local archive (cache) `source_directory/zstash`. + zstash ls --hpss=hpss_archive # List the contents of `hpss_archive` and creates a cache `zstash` at the same level of `source_directory`. # Add `source_directory/new_file.txt` - $ cd source_directory - $ zstash update --hpss=hpss_archive # Add `new_file.txt` to the HPSS archive. This updates the cache `zstash` (in `source_directory`). - $ cd .. - $ zstash ls --hpss=hpss_archive # `new_file.txt` will NOT be shown. The existing cache `zstash` (same level as `source_directory`) is being used. - $ rm -rf zstash # Delete the cache. (You could instead change to another directory). - $ zstash ls --hpss=hpss_archive # `new_file.txt` will be shown. - -Example 3 -- changing the HPSS archive, running ``zstash_ls`` from the source directory: :: - - $ zstash create --hpss=hpss_archive source_directory # Creates an HPSS archive named `hpss_archive` and a local archive (cache) `source_directory/zstash`. - $ cd source_directory # This is the directory we are archiving. - $ zstash ls --hpss=hpss_archive # List the contents of `hpss_archive` and uses the existing cache `zstash` (in `source_directory`). + cd source_directory + zstash update --hpss=hpss_archive # Add `new_file.txt` to the HPSS archive. This updates the cache `zstash` (in `source_directory`). + cd .. + zstash ls --hpss=hpss_archive # `new_file.txt` will NOT be shown. The existing cache `zstash` (same level as `source_directory`) is being used. + rm -rf zstash # Delete the cache. (You could instead change to another directory). + zstash ls --hpss=hpss_archive # `new_file.txt` will be shown. + +Example 3 -- changing the HPSS archive, running ``zstash_ls`` from the source directory: + + .. code-block:: bash + + zstash create --hpss=hpss_archive source_directory # Creates an HPSS archive named `hpss_archive` and a local archive (cache) `source_directory/zstash`. + cd source_directory # This is the directory we are archiving. + zstash ls --hpss=hpss_archive # List the contents of `hpss_archive` and uses the existing cache `zstash` (in `source_directory`). # Add `new_file.txt` - $ cd .. - $ zstash create --hpss=different_hpss_archive source_directory # Create a different HPSS archive of the source directory. This overwrites the local archive (cache) `source_directory/zstash`. - $ cd source_directory - $ zstash ls --hpss=different_archive # `new_file.txt` will be shown. + cd .. + zstash create --hpss=different_hpss_archive source_directory # Create a different HPSS archive of the source directory. This overwrites the local archive (cache) `source_directory/zstash`. + cd source_directory + zstash ls --hpss=different_archive # `new_file.txt` will be shown. -Example 4 -- updating the HPSS archive, running ``zstash_ls`` from the source directory: :: +Example 4 -- updating the HPSS archive, running ``zstash_ls`` from the source directory: - $ zstash create --hpss=hpss_archive source_directory # Creates an HPSS archive named `hpss_archive` and a local archive (cache) `source_directory/zstash`. - $ cd source_directory # This is the directory we are archiving. - $ zstash ls --hpss=hpss_archive # List the contents of `hpss_archive` and uses the existing cache `zstash` (in `source_directory`). + .. code-block:: bash + + zstash create --hpss=hpss_archive source_directory # Creates an HPSS archive named `hpss_archive` and a local archive (cache) `source_directory/zstash`. + cd source_directory # This is the directory we are archiving. + zstash ls --hpss=hpss_archive # List the contents of `hpss_archive` and uses the existing cache `zstash` (in `source_directory`). # Add new_file.txt - $ zstash update --hpss=hpss_archive # Add `new_file.txt` to the HPSS archive. This updates the cache `zstash` (in `source_directory`). - $ zstash ls --hpss=hpss_archive # `new_file.txt` will be shown. + zstash update --hpss=hpss_archive # Add `new_file.txt` to the HPSS archive. This updates the cache `zstash` (in `source_directory`). + zstash ls --hpss=hpss_archive # `new_file.txt` will be shown. Version ======= -Starting with version 0.3, you can check the version of zstash from the command line: :: +Starting with version 0.3, you can check the version of zstash from the command line: + + .. code-block:: bash - $ zstash version - v0.3.0 + zstash version + # v0.3.0 From 5c939eb0b3d084d4203ea6f6193616f70bf9f18f Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 22 Jun 2026 18:01:10 -0700 Subject: [PATCH 07/10] Update environment setup docs --- docs/source/user_guide/getting_started.rst | 304 +++++---------------- 1 file changed, 62 insertions(+), 242 deletions(-) diff --git a/docs/source/user_guide/getting_started.rst b/docs/source/user_guide/getting_started.rst index e787fc20..d12d1d76 100644 --- a/docs/source/user_guide/getting_started.rst +++ b/docs/source/user_guide/getting_started.rst @@ -4,289 +4,109 @@ Getting started *************** -.. highlight:: none - - Activate e3sm_unified environment ================================= -If you have an account on one of the E3SM supported machines (NERSC, Compy, Acme1, -LCRC, Cooley, Rhea), you can access ``zstash`` by activating ``e3sm_unified``, which is +If you have an account on one of the E3SM supported machines, you can access ``zstash`` by activating ``e3sm_unified``, which is a conda environment that pulls together Python and other E3SM tools such as -``e3sm_diags``. - -The paths to ``e3sm_unified`` activation scripts are machine dependent: - -**Compy** - .. code-block:: bash - - source /share/apps/E3SM/conda_envs/load_latest_e3sm_unified_compy.sh - - -**NERSC** - .. code-block:: bash - - source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh - - -**Anvil** - .. code-block:: bash - - source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_anvil.sh - -**Chrysalis** - .. code-block:: bash - - source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh - - -**Cooley** - .. code-block:: bash - - source /lus/theta-fs0/projects/ccsm/acme/tools/e3sm-unified/load_latest_e3sm_unified_cooley.sh - - -**acme1** - .. code-block:: bash - - source /p/user_pub/e3sm_unified/envs/load_latest_e3sm_unified_acme1.sh +``e3sm_diags`` and ``zppy``. +The paths to ``e3sm_unified`` activation scripts are machine dependent. As of E3SM Unified 1.13.0, the supported machines and their corresponding activation scripts are: **Andes** - .. code-block:: bash - - source /gpfs/alpine/proj-shared/cli115/e3sm-unified/load_latest_e3sm_unified_andes.sh - - -Change ``.sh`` to ``.csh`` for ``csh`` shells. - -Note that ``e3sm_unified``'s development cycle is not in phase with ``zstash``, -therefore the version of ``zstash`` included may not be the latest. -To install the latest stable release, refer to the following: - -.. _conda_environment: - -Installation in a Conda Environment -=================================== - -If the E3SM Unified environment doesn't serve your needs, you can alternatively -install the latest version in your own custom conda environment. - -First, activate conda or install it if it's not available. Details vary amongst machines. - -.. _conda_environment_others: - -Others/Local ------------- - -If the system doesn't come with conda pre-installed, follow these instructions: - -1. Download Mambaforge - - Linux - .. code-block:: bash - - wget https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh - - MacOS x86_64 (note that ``zstash`` is not supported on MacOS, but it may be useful to contribute to the documentation on MacOS) - .. code-block:: bash - - wget https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-MacOSX-x86_64.sh - -2. Install Mambaforge - - Linux - .. code-block:: bash - - bash ./Mambaforge-Linux-x86_64.sh + :: + source /ccs/proj/cli115/software/e3sm-unified/load_latest_e3sm_unified_andes.sh - MacOS x86_64 - .. code-block:: bash +**Aurora** + :: - bash ./Mambaforge-MacOSX-x86_64.sh + source /lus/flare/projects/E3SMinput/soft/e3sm-unified/load_latest_e3sm_unified_aurora.sh - When you see: :: - - by running conda init? [yes|no] - [no] >>> yes - - respond with ``yes`` so ``conda`` and ``mamba`` commands are available on - initializing a new bash terminal. - -3. If you are working on a machine/network that intercepts SSL communications (such as -acme1), you will get an SSL error unless you disable the SSL verification: - - .. code-block:: bash - - conda config --set ssl_verify false - binstar config --set ssl_verify False - -4. Once conda and mamba are properly working, you can install the **(a) Latest Stable Release** or -create a **(b) Development Environment**. - -(a) Latest Stable Release -========================= - -Installation using mamba ------------------------- - -First, make sure that you're using ``bash``. - - .. code-block:: bash - - bash - -You must have a conda base enviornment installed as well. -See :ref:`"Installation in a Conda Environment" ` section above for -installing conda. -Create a new Anaconda environment with zstash installed and activate it: :: +**Chrysalis** + :: -These steps should not be necessary if you installed Mambaforge as suggested -above but may be needed if you have previously installed Miniconda3 instead: :: + source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh - conda install -y -n base mamba - conda config --add channels conda-forge - conda config --set channel_priority strict +**Compy** + :: -Create a new conda environment with ``zstash`` installed and activate it: :: + source /share/apps/E3SM/conda_envs/load_latest_e3sm_unified_compy.sh - mamba create -n zstash_env zstash - conda activate zstash_env +**Dane** + :: -Or (less recommended because of potential conflicts) you can install ``zstash`` -in an existing environment. + source /usr/workspace/e3sm/apps/e3sm-unified/load_latest_e3sm_unified_dane.sh - .. code-block:: bash +**Frontier** + :: - mamba install zppy + source /ccs/proj/cli115/software/e3sm-unified/load_latest_e3sm_unified_frontier.sh -Installation on NERSC ---------------------- +**Perlmutter (login or CPU nodes)** + :: -After installing on NERSC, you may see improved performance -running **zstash on the data transfer nodes** (dtn{01..15}.nersc.gov). However, modules are -not directly available there, so you will need to manually activate Anaconda before running -``conda activate zstash_env``. + source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh +**ALCF Polaris** + :: -Updating --------- + source /lus/grand/projects/E3SMinput/soft/e3sm-unified/load_latest_e3sm_unified_polaris.sh -If you **installed into your own conda environment** (e.g., not through the -unified environment), you can update ``zstash`` by doing the following: - .. code-block:: bash +Change ``.sh`` to ``.csh`` for ``csh`` shells. - mamba update zstash +E3SM Unified and zstash versions +============================== -.. _dev-env: +``zstash`` development is largely synced with ``e3sm_unified``. The last several releases have been: -(b) Development Environment -=========================== +* E3SM Unified 1.13.0: ``zstash 1.6.0`` +* E3SM Unified 1.12.0: ``zstash 1.5.0`` -Unlike the latest stable release (i.e., the user environment), the development -environment does not include ``zstash``. -Instead, the developer will ``pip install .`` to build ``zstash`` with changes -(see step 6 below). +To use ``zstash`` features/fixes not yet in a production release, you can use a development environment. -Furthermore, the dev environment includes quality assurance (QA) tools such as code formatters, linters, and ``pre-commit``. -**NOTE**: These QA tools are enforced using ``pre-commit`` checks in the continuous integration/continuous delivery (CI/CD) build, so you must use the dev environment for all contributions. +Nevertheless, it is possible that the version of ``zstash`` included in ``e3sm_unified`` may not be the latest. To install the latest stable release, refer to the following: -1. Follow :ref:`"Others/Local" ` section for installing conda. +Setting up a development environment +==================================== -2. Clone your fork and keep it in sync with the main repo's ``main`` +.. code-block:: bash - .. code-block:: bash + # Get the code ######################################################################## + # Set up your fork: # Go to https://github.com/E3SM-Project/zstash - # Click "Fork" in the upper right hand corner. This will fork the main repo. # Click the green "Code" button - # Choose the HTTPS or SSH option. - # (To use the SSH option, you need to have a SSH connection to GitHub set up). - # Click the clipboard icon to copy the path. - # On your command line: - git clone - git remote -v - # You should see your fork listed as `origin` - - - or if you already have a clone of your fork, rebase your fork on the main repo's ``main`` to keep it in sync: - - .. code-block:: bash - - # Add the main repo as a remote. - # You can call it anything but "upstream" is recommended. - # We'll use `` here. - git remote add - - # Fetch all the branches of that remote into remote-tracking branches - git fetch - - # Make sure that you're on your main branch: - git checkout main - - # Rewrite your main branch so that any of your commits that - # aren't already in /main are replayed on top of that branch: - git rebase /main - - # Push your main branch to your GitHub fork: - # Note that should be `origin` if you cloned your fork as above. - git push -f main - - - Checkout a new branch from ``main``: - - .. code-block:: bash - - git checkout -b /main - -3. Remove any cached conda packages. This will ensure that you always get the latest packages. - - .. code-block:: bash - - mamba clean --all - -4. Enter the fork's clone. - - .. code-block:: bash - + # Choose the SSH option and paste it here: + git clone git@github.com:E3SM-Project/zstash.git cd zstash + git remote -v # You should see the main repo listed as `origin` -5. Use conda to create a new dev environment. -(``zstash`` **is not included in this environment**). - - - Tip: Add the flag ``-n `` to customize the name of the environment - - .. code-block:: bash - - mamba env create -f conda/dev.yml - conda activate zstash_dev - -6. Install ``pre-commit``. - - .. code-block:: bash - - pre-commit install - -7. Make the desired changes to ``zstash``, then rebuild and install with: - - .. code-block:: bash + # A couple optional steps: + git remote add upstream git@github.com:E3SM-Project/zstash.git # Use the name "upstream" instead of origin + git remote add your-fork-name git@github.com:your-fork-name/zstash.git # Use your fork, if you have one - pip install . + # To use the latest code: + git fetch upstream + git checkout -b main upstream/main + # To use code from a specific branch: + git checkout -b that-branch-name upstream/that-branch-name -8. Commit changes and make sure ``pre-commit`` checks pass + # Set up the environment ############################################################## + # First, make sure you have conda activated. Then: + rm -rf build # Sometimes an existing `build` directory can cause problems. + conda clean --all --y # This makes sure conda will pick up the latest information. + conda env create -f conda/dev.yml -n env-name + conda activate env-name + pre-commit run --all-files # This is only necessary if you've made changes + python -m pip install . # Install the code into your development environment (env-name) - .. code-block:: bash +Note: if you'd like to contribute to ``zstash`` rather than just using the latest code, please refer to the Developer Guide instead. - git commit -m "commit-message" - .. figure:: _static/figures/pre-commit-passing.png - :alt: pre-commit Output - ``pre-commit`` Output +Running +======= -Archiving -========= -For archiving E3SM simulations, we recommend following the -:ref:`Best practices for E3SM`. +To run ``zstash``, refer to :doc:`user_guide/usage`. From 58390420a1363c996c526a0e345088468c35d195 Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 22 Jun 2026 18:24:09 -0700 Subject: [PATCH 08/10] Revise Copilot-edited docs --- .../code_snippets/tar_tracking_modes.dot | 45 ------------------- docs/source/dev_guide/tar_tracking_modes.rst | 9 ---- docs/source/index.rst | 2 +- .../user_guide/globus/configuration.rst | 2 +- docs/source/user_guide/globus/intro.rst | 4 +- 5 files changed, 4 insertions(+), 58 deletions(-) delete mode 100644 docs/source/_static/code_snippets/tar_tracking_modes.dot diff --git a/docs/source/_static/code_snippets/tar_tracking_modes.dot b/docs/source/_static/code_snippets/tar_tracking_modes.dot deleted file mode 100644 index 29b4118b..00000000 --- a/docs/source/_static/code_snippets/tar_tracking_modes.dot +++ /dev/null @@ -1,45 +0,0 @@ -digraph tar_tracking_modes { - rankdir=LR; - node [shape=box]; - - source [label="source files"]; - local_tar [label="local tar in cache"]; - index_db [label="index.db"]; - files_table [label="files table"]; - tars_table [label="tars table"]; - - source -> local_tar [label="construct_tars"]; - local_tar -> files_table [label="record members"]; - local_tar -> tars_table [label="record tar metadata"]; - files_table -> index_db; - tars_table -> index_db; - - subgraph cluster_none { - label="--hpss=none"; - local_archive [label="local archive kept in cache"]; - } - - subgraph cluster_hpss { - label="HPSS path"; - hsi_put [label="hsi put"]; - remote_hpss [label="remote HPSS archive"]; - } - - subgraph cluster_globus { - label="globus:// path"; - batch [label="TransferBatch / TransferData"]; - task [label="Globus task"]; - remote_globus [label="remote archive via Globus"]; - } - - local_tar -> local_archive [label="keep local"]; - local_tar -> hsi_put [label="close then transfer"]; - hsi_put -> remote_hpss; - - local_tar -> batch [label="add tar to batch"]; - batch -> task [label="submit"]; - task -> remote_globus [label="success"]; - - index_db -> remote_hpss [label="upload last"]; - index_db -> remote_globus [label="transfer last"]; -} diff --git a/docs/source/dev_guide/tar_tracking_modes.rst b/docs/source/dev_guide/tar_tracking_modes.rst index 2cb3300e..f1038b02 100644 --- a/docs/source/dev_guide/tar_tracking_modes.rst +++ b/docs/source/dev_guide/tar_tracking_modes.rst @@ -96,12 +96,3 @@ Summary table - Globus transfer task - after Globus success and finalization, unless ``--keep`` - local cache plus remote Globus-backed destination - -Graphviz source -=============== - -The following ``.dot`` file summarizes the relationships between local tar -creation, database updates, and transfer handling in each mode: - -.. literalinclude:: /_static/code_snippets/tar_tracking_modes.dot - :language: dot diff --git a/docs/source/index.rst b/docs/source/index.rst index 98ddd9ec..2e56ac20 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -64,7 +64,7 @@ includes: * :doc:`user_guide/best_practices` for archive management recommendations * :doc:`user_guide/database` for the archive index database layout * :doc:`user_guide/support` for where to ask questions or report issues -* :doc:`user_guide/archived_documentation` for a full archive creation and extraction walkthrough +* :doc:`user_guide/archived_documentation` for older documentation that may still be useful as a reference. Developer Guide pages diff --git a/docs/source/user_guide/globus/configuration.rst b/docs/source/user_guide/globus/configuration.rst index 1e7a8832..a9afd956 100644 --- a/docs/source/user_guide/globus/configuration.rst +++ b/docs/source/user_guide/globus/configuration.rst @@ -14,7 +14,7 @@ The file uses an INI format such as: .. code-block:: ini [local] - globus_endpoint_uuid = 6bdc7956-fc0f-4ad2-989c-7aa5ee643a79 + globus_endpoint_uuid = 6bdc7956-fc0f-4ad2-989c-7aa5ee643a79 # NERSC_PERLMUTTER_ENDPOINT If the file does not exist, zstash creates it the first time it needs Globus configuration. diff --git a/docs/source/user_guide/globus/intro.rst b/docs/source/user_guide/globus/intro.rst index df84ec4c..bedfc857 100644 --- a/docs/source/user_guide/globus/intro.rst +++ b/docs/source/user_guide/globus/intro.rst @@ -17,7 +17,7 @@ Examples include: * ``globus://nersc/~/my_archive`` * ``globus://alcf/~/my_archive`` -* ``globus://9cd89cfd-6d04-11e5-ba46-22000b92c6ec/~/my_archive`` +* ``globus://9cd89cfd-6d04-11e5-ba46-22000b92c6ec/~/my_archive`` (NERSC HPSS Globus endpoint) The names ``nersc`` and ``alcf`` are built-in shortcuts for the NERSC HPSS and ALCF HPSS Globus endpoints. @@ -50,7 +50,7 @@ Authentication flow The first time zstash needs Globus credentials, it will print an authorization URL and ask you to paste back the returned code. After a successful login, zstash stores refresh-token state in ``~/.zstash_globus_tokens.json`` so future -commands usually do not need another interactive login. +Globus transfers between the same machines usually do not need another interactive login. zstash also checks ``~/.zstash.ini`` for the local endpoint UUID. See :doc:`configuration` for details on when that file needs to be created or From b11cd30cc7ca460f4c1db2c23b8ec71d50bf10bf Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 22 Jun 2026 19:02:10 -0700 Subject: [PATCH 09/10] Address Copilot review comments --- docs/source/dev_guide/index.rst | 3 +-- docs/source/index.rst | 6 +++--- docs/source/user_guide/database.rst | 2 +- docs/source/user_guide/index.rst | 4 ++-- docs/source/user_guide/usage.rst | 14 +++++++------- 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/docs/source/dev_guide/index.rst b/docs/source/dev_guide/index.rst index 29ad932b..4dac454f 100644 --- a/docs/source/dev_guide/index.rst +++ b/docs/source/dev_guide/index.rst @@ -12,6 +12,5 @@ implementation details that are mainly useful to contributors. ci tar_tracking_modes testing - release_testing - releases + releases/index contributing_to_docs diff --git a/docs/source/index.rst b/docs/source/index.rst index 2e56ac20..4a499905 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -60,11 +60,11 @@ includes: * :doc:`user_guide/design` for the high-level architecture and implementation overview * :doc:`user_guide/getting_started` for installation and first-time setup * :doc:`user_guide/usage` for command-line usage details -* :doc:`user_guide/globus` for Globus account setup and transfer workflows and ``.zstash.ini`` configuration details +* :doc:`user_guide/globus/index` for Globus account setup and transfer workflows and ``.zstash.ini`` configuration details * :doc:`user_guide/best_practices` for archive management recommendations * :doc:`user_guide/database` for the archive index database layout * :doc:`user_guide/support` for where to ask questions or report issues -* :doc:`user_guide/archived_documentation` for older documentation that may still be useful as a reference. +* :doc:`user_guide/archived_documentation/index` for older documentation that may still be useful as a reference. Developer Guide pages @@ -78,7 +78,7 @@ The contributor and maintainer documentation is organized under * :doc:`dev_guide/tar_tracking_modes` for tar tracking behavior in each storage mode * :doc:`dev_guide/testing` for the test layout and execution guidance -* :doc:`dev_guide/releases` for the release process +* :doc:`dev_guide/releases/index` for the release process * :doc:`dev_guide/contributing_to_docs` for development environment setup and contribution workflow diff --git a/docs/source/user_guide/database.rst b/docs/source/user_guide/database.rst index 6e32a4b7..72f96ac2 100644 --- a/docs/source/user_guide/database.rst +++ b/docs/source/user_guide/database.rst @@ -76,7 +76,7 @@ To list **files matching a specified pattern** (for example \*/run/\*.nc): sqlite3 zstash/index.db "select * from files where name glob '*/run/*.nc';" -To list **all the files in a specific tar fole** (for example 00000a.tar): +To list **all the files in a specific tar file** (for example 00000a.tar): .. code-block:: bash diff --git a/docs/source/user_guide/index.rst b/docs/source/user_guide/index.rst index 8916901c..37ac47a7 100644 --- a/docs/source/user_guide/index.rst +++ b/docs/source/user_guide/index.rst @@ -11,8 +11,8 @@ working with HPSS or Globus in day-to-day workflows. design getting_started usage - globus + globus/index best_practices database support - archived_documentation + archived_documentation/index diff --git a/docs/source/user_guide/usage.rst b/docs/source/user_guide/usage.rst index 74066a49..62d1458e 100644 --- a/docs/source/user_guide/usage.rst +++ b/docs/source/user_guide/usage.rst @@ -42,7 +42,7 @@ where Then zstash will use `Globus `_ to store a new zstash archive on a Globus endpoint. Names ``alcf`` and ``nersc`` are recognized as referring to the ALCF HPSS and NERSC HPSS endpoints, e.g. ``globus://nersc/~/my_archive``. - See :doc:`user_guide/globus` and :doc:`user_guide/configuration` for first-time Globus setup details. + See :doc:`user_guide/globus/intro` and :doc:`user_guide/globus/configuration` for first-time Globus setup details. * ```` specifies the path to the local directory that should be archived. Additional optional arguments: @@ -85,8 +85,8 @@ Once done, you should see the archive files on hsi: .. code-block:: bash hsi - cd test/E3SM_simulations/20170731.F20TR.ne30_ne30.edison - ls + hsi> cd test/E3SM_simulations/20170731.F20TR.ne30_ne30.edison + hsi> ls # 000000.tar index.db The data from this test simulation is small, so in this case there is only a single tar @@ -144,7 +144,7 @@ If you want to store zstash archive on these two remote HPSS file systems, you c Always activate Globus endpoints via the Globus web interface before running ``zstash``. For a more complete first-time Globus workflow, including ``~/.zstash.ini``, -see :doc:`user_guide/globus` and :doc:`user_guide/configuration`. +see :doc:`user_guide/globus/intro` and :doc:`user_guide/globus/configuration`. Check ===== @@ -554,7 +554,7 @@ Example 2 -- updating the HPSS archive: rm -rf zstash # Delete the cache. (You could instead change to another directory). zstash ls --hpss=hpss_archive # `new_file.txt` will be shown. -Example 3 -- changing the HPSS archive, running ``zstash_ls`` from the source directory: +Example 3 -- changing the HPSS archive, running ``zstash ls`` from the source directory: .. code-block:: bash @@ -565,9 +565,9 @@ Example 3 -- changing the HPSS archive, running ``zstash_ls`` from the source di cd .. zstash create --hpss=different_hpss_archive source_directory # Create a different HPSS archive of the source directory. This overwrites the local archive (cache) `source_directory/zstash`. cd source_directory - zstash ls --hpss=different_archive # `new_file.txt` will be shown. + zstash ls --hpss=different_hpss_archive # `new_file.txt` will be shown. -Example 4 -- updating the HPSS archive, running ``zstash_ls`` from the source directory: +Example 4 -- updating the HPSS archive, running ``zstash ls`` from the source directory: .. code-block:: bash From b21fcd968660d65812e9573c74ff9b952a55074a Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Mon, 22 Jun 2026 19:09:28 -0700 Subject: [PATCH 10/10] Fix usage docs link --- docs/source/user_guide/getting_started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/user_guide/getting_started.rst b/docs/source/user_guide/getting_started.rst index d12d1d76..5bb768f9 100644 --- a/docs/source/user_guide/getting_started.rst +++ b/docs/source/user_guide/getting_started.rst @@ -109,4 +109,4 @@ Note: if you'd like to contribute to ``zstash`` rather than just using the lates Running ======= -To run ``zstash``, refer to :doc:`user_guide/usage`. +To run ``zstash``, refer to :doc:`usage`.