From def4c4ad58f1e9e89b3ab6ff9fcba20b31ebf915 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Wed, 9 Sep 2026 14:05:47 +0200 Subject: [PATCH 1/6] Source one shared library from the get_* install scripts The tools/get_*.sh scripts each carried their own copy of the same shell logic: portable_realpath in nine of them, the dirty-checkout guard in seven, the --force argument split in six, and the GitHub SSH probe in four, plus the repo-root derivation in nearly all of them. A fix to one copy left the others untouched, which is how a portable_realpath that rejects a path git clone has yet to create survived in nine places at once. tools/_get_common.sh now holds one copy of each helper, and every script sources it through its own directory, so a run by path (the docs, install.sh, the CI setup action, and proteus install-all through data.py) finds it without resolving a path first. A script whose library is absent stops with the missing file named instead of continuing with undefined helpers. The shared portable_realpath resolves a destination that does not exist yet, which is what an install path is before the clone, and get_socrates.sh stops when its install path cannot be resolved rather than passing an empty work-tree name to git clone. Variations that differ per script are parameters, not separate copies: guard_dirty_checkout takes a git pathspec so get_socrates.sh keeps excluding its regenerable make/Mk_cmd, get_parse_args reports both the --force switch and the optional install path, and the SSH probe honours GIT_SSH_COMMAND for every caller rather than only for SOCRATES. The helpers target bash 3.2 and behave the same with or without set -euo pipefail, because the scripts differ on that. Destinations are unchanged, with one exception: get_agni.sh and get_spider.sh now derive the checkout root through the same symlink-resolving path as the other scripts, so a checkout reached through a symlink resolves to the same root everywhere. --- docs/How-to/development_standards.md | 26 +++++ tools/_get_common.sh | 151 +++++++++++++++++++++++++++ tools/get_agni.sh | 14 ++- tools/get_aragog.sh | 44 ++------ tools/get_boreas.sh | 64 +++--------- tools/get_lavatmos.sh | 64 +++--------- tools/get_lovepy.sh | 19 ++-- tools/get_petsc.sh | 17 ++- tools/get_socrates.sh | 85 ++++++--------- tools/get_spider.sh | 65 ++++-------- tools/get_thermoenginelite.sh | 64 +++--------- tools/get_vulcan.sh | 52 +++------ tools/get_zalmoxis.sh | 44 ++------ 13 files changed, 341 insertions(+), 368 deletions(-) create mode 100644 tools/_get_common.sh diff --git a/docs/How-to/development_standards.md b/docs/How-to/development_standards.md index 85e058aa6..88cce3326 100644 --- a/docs/How-to/development_standards.md +++ b/docs/How-to/development_standards.md @@ -62,6 +62,32 @@ field sets) are written one entry per line with a trailing comma, grouped by module under a header comment, and ordered alphabetically within each group. This keeps two independent additions on different lines so they merge cleanly. +**Module install scripts** + +The `tools/get_*.sh` scripts share one sourced library, `tools/_get_common.sh`. +It provides `portable_realpath`, the checkout root (`proteus_root`) and tools +directory (`proteus_tools_dir`), the `--force` and install-path argument split +(`get_parse_args`), the dirty-checkout guard (`guard_dirty_checkout`), the +GitHub SSH probe (`github_use_ssh`), the https-to-SSH URL rewrite +(`github_ssh_url`), and the pin reader (`resolve_module_pin`). + +A new install script starts with the same bootstrap the existing ones use: + +```bash +_get_common="$(dirname "${BASH_SOURCE[0]}")/_get_common.sh" +if [ ! -f "$_get_common" ]; then + echo "ERROR: $_get_common is missing; use a complete PROTEUS checkout." >&2 + exit 1 +fi +source "$_get_common" +``` + +Add a helper to the library rather than copying one into a script, and give it +a parameter for the variation a caller needs: `get_socrates.sh` passes a git +pathspec to `guard_dirty_checkout` so its regenerable `make/Mk_cmd` does not +count as local work. The helpers target bash 3.2 and run the same with or +without `set -euo pipefail`, because the scripts differ on that. + **Editing shared files** - When adding to the main coupling loop, add a stage function and call it rather diff --git a/tools/_get_common.sh b/tools/_get_common.sh new file mode 100644 index 000000000..d693df575 --- /dev/null +++ b/tools/_get_common.sh @@ -0,0 +1,151 @@ +# shellcheck shell=bash +# +# Shared shell helpers for the tools/get_*.sh module install scripts. +# +# This file is sourced, never executed. Source it near the top of a get_* +# script, before the first helper call: +# +# _get_common="$(dirname "${BASH_SOURCE[0]}")/_get_common.sh" +# [ -f "$_get_common" ] || { +# echo "ERROR: $_get_common is missing." >&2 +# exit 1 +# } +# source "$_get_common" +# +# Every caller runs a get_* script by path (docs, install.sh, the CI setup +# action, and proteus install-all through src/proteus/utils/data.py), so +# ${BASH_SOURCE[0]} locates this file without needing a resolved path +# first, which is what makes the bootstrap above safe to run before +# portable_realpath exists. +# +# The helpers target bash 3.2, the version macOS ships, and behave the same +# whether or not the sourcing script enables `set -euo pipefail`. Two of +# them exit the script on failure (guard_dirty_checkout, resolve_module_pin) +# and so must be called as plain commands: inside a command substitution +# the exit would only leave the subshell. +# +# Sourcing this file sets: +# proteus_tools_dir absolute path of the tools/ directory +# proteus_root absolute path of the PROTEUS checkout root +# +# and defines: +# portable_realpath resolve a path, whether or not it exists yet +# get_parse_args split --force from an optional install path +# guard_dirty_checkout refuse to delete a checkout holding local work +# github_use_ssh report whether GitHub accepts an SSH key +# github_ssh_url rewrite an https GitHub URL for SSH transport +# resolve_module_pin read a module's pinned clone URL and ref + +# Resolve a path to an absolute one, with symlinks and .. expanded. +# +# macOS before 13 (Catalina through Monterey) does not ship the coreutils +# realpath, so fall back to python3, which is always present in PROTEUS's +# Python environment. A path that does not exist yet is also rejected by +# realpath (BSD refuses a missing leaf, GNU a missing parent) and takes +# the same fallback: install destinations are resolved before git clone +# creates them, so a missing path must resolve rather than fail. +portable_realpath() { + if command -v realpath >/dev/null 2>&1 && realpath "$1" 2>/dev/null; then + return 0 + fi + python3 -c "import os,sys; print(os.path.realpath(sys.argv[1]))" "$1" +} + +# The directory holding this file, without calling out to dirname, so that +# sourcing it depends on nothing but bash itself. +_get_common_dir="${BASH_SOURCE[0]%/*}" +if [ "$_get_common_dir" = "${BASH_SOURCE[0]}" ]; then + _get_common_dir="." +fi +proteus_tools_dir=$(portable_realpath "$_get_common_dir") +proteus_root=$(portable_realpath "$proteus_tools_dir/..") + +# Split the script's arguments into the --force switch and an optional +# install-path positional, reported as get_force (true / false) and +# get_install_path (empty when no path was given). Scripts with a fixed +# destination read get_force only; --force may appear before or after the +# path. guard_dirty_checkout honours get_force. +get_parse_args() { + get_force=false + get_install_path="" + for arg in "$@"; do + if [ "$arg" = "--force" ]; then + get_force=true + elif [ -z "$get_install_path" ]; then + get_install_path="$arg" + fi + done +} + +# Refuse to delete a checkout that holds local work, unless --force was +# given. Guarded states: modified tracked files, and commits that are on +# no remote. Untracked files (build artifacts, egg-info) do not block the +# refresh, because they are routine in a refreshed checkout. +# +# Usage: guard_dirty_checkout [git pathspec...] +# +# The script name goes into the recovery command in the error message. +# Trailing arguments are passed to git status, which get_socrates.sh uses +# to exclude its regenerable build config from the dirty test. Exits 1 +# when the checkout is guarded, so call it as a plain command. +guard_dirty_checkout() { + local workpath="$1" + local script="$2" + shift 2 + + if [ "${get_force:-false}" = true ]; then + return 0 + fi + if [ ! -d "$workpath/.git" ]; then + return 0 + fi + + local dirty unpushed + dirty=$(git -C "$workpath" status --porcelain --untracked-files=no "$@" 2>/dev/null | head -1) + unpushed=$(git -C "$workpath" log HEAD --not --remotes --oneline 2>/dev/null | head -1) + if [ -n "$dirty" ] || [ -n "$unpushed" ]; then + echo "ERROR: $workpath has uncommitted changes or commits not on a remote." >&2 + echo " Refusing to delete it. Commit and push your work, or run" >&2 + echo " bash tools/$script --force to discard the checkout." >&2 + exit 1 + fi +} + +# Report whether GitHub accepts an SSH key, as "true" or "false" on stdout. +# +# `ssh -T git@github.com` exits 1 when the key is accepted (GitHub refuses +# the interactive shell it was asked for) and 255 when it is not, so exit +# code 1 is the success signal. The probe honours GIT_SSH_COMMAND, so a +# caller such as CI can make it non-interactive and fast-failing. Its own +# output is sent to stderr, where the user still sees it, so that it +# cannot contaminate the answer read by a command substitution. +github_use_ssh() { + local rc=0 + ${GIT_SSH_COMMAND:-ssh} -T git@github.com >&2 || rc=$? + if [ "$rc" -eq 1 ]; then + echo true + else + echo false + fi +} + +# Rewrite an https GitHub URL to its SSH form for cloning over SSH. +# A URL that is not on github.com is returned unchanged. +github_ssh_url() { + printf '%s\n' "${1/https:\/\/github.com\//git@github.com:}" +} + +# Read a module's pinned clone URL and ref from pyproject.toml, through +# tools/_module_pins.py, into module_url and module_ref. A module whose +# pin is missing or empty stops the install here, where the cause is +# named, rather than at a git clone with an empty argument. Exits 1 in +# that case, so call it as a plain command. +resolve_module_pin() { + local module="$1" + module_url=$(python "$proteus_tools_dir/_module_pins.py" "$module" url) + module_ref=$(python "$proteus_tools_dir/_module_pins.py" "$module" ref) + if [ -z "$module_url" ] || [ -z "$module_ref" ]; then + echo "ERROR: could not resolve $module url/ref from pyproject.toml" >&2 + exit 1 + fi +} diff --git a/tools/get_agni.sh b/tools/get_agni.sh index 5d91cd4d1..2b4325024 100755 --- a/tools/get_agni.sh +++ b/tools/get_agni.sh @@ -17,16 +17,22 @@ if ! command -v julia >/dev/null 2>&1; then exit 1 fi -script_root="$(cd "$(dirname "$0")/.." && pwd)" +# Shared helpers: see tools/_get_common.sh. +_get_common="$(dirname "${BASH_SOURCE[0]}")/_get_common.sh" +if [ ! -f "$_get_common" ]; then + echo "ERROR: $_get_common is missing; use a complete PROTEUS checkout." >&2 + exit 1 +fi +source "$_get_common" -ag_url="${AGNI_GIT_URL:-$(python "$script_root/tools/_module_pins.py" agni url)}" -ag_ref="${AGNI_GIT_REF:-$(python "$script_root/tools/_module_pins.py" agni ref)}" +ag_url="${AGNI_GIT_URL:-$(python "$proteus_tools_dir/_module_pins.py" agni url)}" +ag_ref="${AGNI_GIT_REF:-$(python "$proteus_tools_dir/_module_pins.py" agni ref)}" # First positional arg can be either "0" (skip AGNI test step) or a path. # Preserve AGNI's upstream get_agni.sh interface: passing "0" tells it # to skip Pkg.test. Anything else is treated as a destination path. skip_tests="" -dest="$script_root/AGNI" +dest="$proteus_root/AGNI" if [ "${1:-}" = "0" ]; then skip_tests="0" elif [ -n "${1:-}" ]; then diff --git a/tools/get_aragog.sh b/tools/get_aragog.sh index a19971298..bc7d94bee 100755 --- a/tools/get_aragog.sh +++ b/tools/get_aragog.sh @@ -16,17 +16,16 @@ echo "Set up Aragog..." -portable_realpath() { - if command -v realpath >/dev/null 2>&1; then - realpath "$1" - else - python3 -c "import os,sys; print(os.path.realpath(sys.argv[1]))" "$1" - fi -} +# Shared helpers: see tools/_get_common.sh. +_get_common="$(dirname "${BASH_SOURCE[0]}")/_get_common.sh" +if [ ! -f "$_get_common" ]; then + echo "ERROR: $_get_common is missing; use a complete PROTEUS checkout." >&2 + exit 1 +fi +source "$_get_common" # Path to PROTEUS folder -root=$(dirname $(portable_realpath $0)) -root=$(portable_realpath "$root/..") +root="$proteus_root" # Paired-branch detection. When PROTEUS is on a feature branch and aragog has a # branch of the same name, install that aragog branch so CI exercises the real @@ -39,36 +38,15 @@ if [ -z "$proteus_branch" ]; then proteus_branch=$(git -C "$root" rev-parse --abbrev-ref HEAD 2>/dev/null) fi -# Refuse to delete a checkout holding local work unless --force is given. -# Keep this guard in sync across the get_* scripts that refresh checkouts. -# Guarded states: modified tracked files, and commits not on any remote. -# Untracked files (build artifacts, egg-info) do not block the refresh. -force=false -for arg in "$@"; do - [ "$arg" = "--force" ] && force=true -done +get_parse_args "$@" workpath=$root/aragog/ -if [ -d "$workpath/.git" ] && [ "$force" != true ]; then - dirty=$(git -C "$workpath" status --porcelain --untracked-files=no 2>/dev/null | head -1) - unpushed=$(git -C "$workpath" log HEAD --not --remotes --oneline 2>/dev/null | head -1) - if [ -n "$dirty" ] || [ -n "$unpushed" ]; then - echo "ERROR: $workpath has uncommitted changes or commits not on a remote." >&2 - echo " Refusing to delete it. Commit and push your work, or run" >&2 - echo " bash tools/get_aragog.sh --force to discard the checkout." >&2 - exit 1 - fi -fi +guard_dirty_checkout "$workpath" get_aragog.sh # Make room rm -rf $workpath # Check SSH access to GitHub -ssh -T git@github.com -if [ $? -eq 1 ]; then - use_ssh=true -else - use_ssh=false -fi +use_ssh=$(github_use_ssh) # Download echo "Cloning from GitHub" diff --git a/tools/get_boreas.sh b/tools/get_boreas.sh index f86bf9eab..2bfa5a79f 100755 --- a/tools/get_boreas.sh +++ b/tools/get_boreas.sh @@ -12,67 +12,35 @@ set -euo pipefail echo "Set up BOREAS..." -portable_realpath() { - if command -v realpath >/dev/null 2>&1; then - realpath "$1" - else - python3 -c "import os,sys; print(os.path.realpath(sys.argv[1]))" "$1" - fi -} +# Shared helpers: see tools/_get_common.sh. +_get_common="$(dirname "${BASH_SOURCE[0]}")/_get_common.sh" +if [ ! -f "$_get_common" ]; then + echo "ERROR: $_get_common is missing; use a complete PROTEUS checkout." >&2 + exit 1 +fi +source "$_get_common" # Path to PROTEUS folder -root=$(dirname "$(portable_realpath "$0")") -root=$(portable_realpath "$root/..") +root="$proteus_root" -# Refuse to delete a checkout holding local work unless --force is given. -# Keep this guard in sync across the get_* scripts that refresh checkouts. -# Guarded states: modified tracked files, and commits not on any remote. -# Untracked files (build artifacts, egg-info) do not block the refresh. -force=false -for arg in "$@"; do - [ "$arg" = "--force" ] && force=true -done +get_parse_args "$@" workpath="$root/BOREAS/" -if [ -d "$workpath/.git" ] && [ "$force" != true ]; then - dirty=$(git -C "$workpath" status --porcelain --untracked-files=no 2>/dev/null | head -1) - unpushed=$(git -C "$workpath" log HEAD --not --remotes --oneline 2>/dev/null | head -1) - if [ -n "$dirty" ] || [ -n "$unpushed" ]; then - echo "ERROR: $workpath has uncommitted changes or commits not on a remote." >&2 - echo " Refusing to delete it. Commit and push your work, or run" >&2 - echo " bash tools/get_boreas.sh --force to discard the checkout." >&2 - exit 1 - fi -fi +guard_dirty_checkout "$workpath" get_boreas.sh # Make room rm -rf "$workpath" -# Detect SSH access to GitHub. `ssh -T git@github.com` exits 1 when -# authentication succeeds (GitHub refuses the shell), so a plain call -# would trip `set -e`; keeping it as the `if` condition keeps it in -# scope where a non-zero exit is expected rather than fatal. -if ssh -T git@github.com; then - use_ssh=false -else - if [ $? -eq 1 ]; then - use_ssh=true - else - use_ssh=false - fi -fi +# Detect SSH access to GitHub. +use_ssh=$(github_use_ssh) # Resolve the pinned URL + ref from pyproject.toml. -b_url=$(python "$root/tools/_module_pins.py" boreas url) -b_ref=$(python "$root/tools/_module_pins.py" boreas ref) -if [ -z "$b_url" ] || [ -z "$b_ref" ]; then - echo "ERROR: could not resolve boreas url/ref from pyproject.toml" >&2 - exit 1 -fi +resolve_module_pin boreas +b_url="$module_url" +b_ref="$module_ref" echo "Cloning from GitHub" if [ "$use_ssh" = true ]; then - # Rewrite https://github.com/ -> git@github.com: for SSH transport. - uri=${b_url/https:\/\/github.com\//git@github.com:} + uri=$(github_ssh_url "$b_url") else uri="$b_url" fi diff --git a/tools/get_lavatmos.sh b/tools/get_lavatmos.sh index b7a7ca819..c39bb182a 100755 --- a/tools/get_lavatmos.sh +++ b/tools/get_lavatmos.sh @@ -3,26 +3,18 @@ echo "Set up LavAtmos..." -portable_realpath() { - if command -v realpath >/dev/null 2>&1; then - realpath "$1" - else - python3 -c "import os,sys; print(os.path.realpath(sys.argv[1]))" "$1" - fi -} +# Shared helpers: see tools/_get_common.sh. +_get_common="$(dirname "${BASH_SOURCE[0]}")/_get_common.sh" +if [ ! -f "$_get_common" ]; then + echo "ERROR: $_get_common is missing; use a complete PROTEUS checkout." >&2 + exit 1 +fi +source "$_get_common" # Path to PROTEUS folder -root=$(dirname "$(portable_realpath "$0")") -root=$(portable_realpath "$root/..") +root="$proteus_root" -# Refuse to delete a checkout holding local work unless --force is given. -# Keep this guard in sync across the get_* scripts that refresh checkouts. -# Guarded states: modified tracked files, and commits not on any remote. -# Untracked files (build artifacts, egg-info) do not block the refresh. -force=false -for arg in "$@"; do - [ "$arg" = "--force" ] && force=true -done +get_parse_args "$@" workpath="$root/LavAtmos/" # Already setup? @@ -35,46 +27,22 @@ if [ -n "$LAVA_DIR" ]; then sleep 5 fi -if [ -d "$workpath/.git" ] && [ "$force" != true ]; then - dirty=$(git -C "$workpath" status --porcelain --untracked-files=no 2>/dev/null | head -1) - unpushed=$(git -C "$workpath" log HEAD --not --remotes --oneline 2>/dev/null | head -1) - if [ -n "$dirty" ] || [ -n "$unpushed" ]; then - echo "ERROR: $workpath has uncommitted changes or commits not on a remote." >&2 - echo " Refusing to delete it. Commit and push your work, or run" >&2 - echo " bash tools/get_lavatmos.sh --force to discard the checkout." >&2 - exit 1 - fi -fi +guard_dirty_checkout "$workpath" get_lavatmos.sh # Make room rm -rf "$workpath" -# Detect SSH access to GitHub. `ssh -T git@github.com` exits 1 when -# authentication succeeds (GitHub refuses the shell), so a plain call -# would trip `set -e`; keeping it as the `if` condition keeps it in -# scope where a non-zero exit is expected rather than fatal. -if ssh -T git@github.com; then - use_ssh=false -else - if [ $? -eq 1 ]; then - use_ssh=true - else - use_ssh=false - fi -fi +# Detect SSH access to GitHub. +use_ssh=$(github_use_ssh) # Resolve the pinned URL + ref from pyproject.toml. -l_url=$(python "$root/tools/_module_pins.py" lavatmos url) -l_ref=$(python "$root/tools/_module_pins.py" lavatmos ref) -if [ -z "$l_url" ] || [ -z "$l_ref" ]; then - echo "ERROR: could not resolve lavatmos url/ref from pyproject.toml" >&2 - exit 1 -fi +resolve_module_pin lavatmos +l_url="$module_url" +l_ref="$module_ref" echo "Cloning from GitHub" if [ "$use_ssh" = true ]; then - # Rewrite https://github.com/ -> git@github.com: for SSH transport. - uri=${l_url/https:\/\/github.com\//git@github.com:} + uri=$(github_ssh_url "$l_url") else uri="$l_url" fi diff --git a/tools/get_lovepy.sh b/tools/get_lovepy.sh index 9b3236c56..b61c0d609 100755 --- a/tools/get_lovepy.sh +++ b/tools/get_lovepy.sh @@ -7,12 +7,19 @@ if ! [ -x "$(command -v julia)" ]; then exit 1 fi -# Resolve the pinned LovePy URL + ref from pyproject.toml. Julia's -# Pkg.add accepts a `rev=` kwarg for a specific commit / tag / branch; -# `main` is the default if no ref is configured. -script_root="$(cd "$(dirname "$0")/.." && pwd)" -lp_url=$(python "$script_root/tools/_module_pins.py" lovepy url) -lp_ref=$(python "$script_root/tools/_module_pins.py" lovepy ref) +# Shared helpers: see tools/_get_common.sh. +_get_common="$(dirname "${BASH_SOURCE[0]}")/_get_common.sh" +if [ ! -f "$_get_common" ]; then + echo "ERROR: $_get_common is missing; use a complete PROTEUS checkout." >&2 + exit 1 +fi +source "$_get_common" + +# Resolve the pinned LovePy URL + ref from pyproject.toml. Julia's Pkg.add +# accepts a `rev=` kwarg for a specific commit / tag / branch. +resolve_module_pin lovepy +lp_url="$module_url" +lp_ref="$module_ref" echo "Installing LovePy into Julia environment ($lp_url @ $lp_ref)..." LD_LIBRARY_PATH="" julia -e "using Pkg; Pkg.add(url=\"$lp_url\", rev=\"$lp_ref\")" diff --git a/tools/get_petsc.sh b/tools/get_petsc.sh index 09266cc9c..bf431aa47 100755 --- a/tools/get_petsc.sh +++ b/tools/get_petsc.sh @@ -30,17 +30,14 @@ set -e # ----------------------------------------------------------------------------- -# Portable realpath: macOS <13 (Catalina through Monterey) does not ship -# GNU coreutils realpath. Fall back to python3, which is always available -# in PROTEUS's conda environment. +# Shared helpers, portable_realpath among them: see tools/_get_common.sh. # ----------------------------------------------------------------------------- -portable_realpath() { - if command -v realpath >/dev/null 2>&1; then - realpath "$1" - else - python3 -c "import os,sys; print(os.path.realpath(sys.argv[1]))" "$1" - fi -} +_get_common="$(dirname "${BASH_SOURCE[0]}")/_get_common.sh" +if [ ! -f "$_get_common" ]; then + echo "ERROR: $_get_common is missing; use a complete PROTEUS checkout." >&2 + exit 1 +fi +source "$_get_common" # ----------------------------------------------------------------------------- # Error handling: report which step failed on any non-zero exit diff --git a/tools/get_socrates.sh b/tools/get_socrates.sh index 4f08ab131..a7c05e25b 100755 --- a/tools/get_socrates.sh +++ b/tools/get_socrates.sh @@ -1,5 +1,10 @@ #!/bin/bash # Download and compile socrates +# +# Usage: +# tools/get_socrates.sh # install into ./socrates/ +# tools/get_socrates.sh some/path # custom destination, created if missing +# tools/get_socrates.sh --force # discard an existing checkout # Do we have NetCDF? if ! [ -x "$(command -v nc-config)" ]; then @@ -26,66 +31,43 @@ if [ -n "$RAD_DIR" ]; then sleep 5 fi -portable_realpath() { - if command -v realpath >/dev/null 2>&1; then - realpath "$1" - else - python3 -c "import os,sys; print(os.path.realpath(sys.argv[1]))" "$1" - fi -} - - -# Check SSH access to GitHub. The probe honours GIT_SSH_COMMAND so -# callers (e.g. CI) can make it non-interactive and fast-failing. -${GIT_SSH_COMMAND:-ssh} -T git@github.com -if [ $? -eq 1 ]; then - use_ssh=true -else - use_ssh=false +# Shared helpers: see tools/_get_common.sh. +_get_common="$(dirname "${BASH_SOURCE[0]}")/_get_common.sh" +if [ ! -f "$_get_common" ]; then + echo "ERROR: $_get_common is missing; use a complete PROTEUS checkout." >&2 + exit 1 fi +source "$_get_common" + +# Check SSH access to GitHub. +use_ssh=$(github_use_ssh) # Disable SSH (uncomment to allow SSH clone of SOCRATES) # use_ssh=false # Download -root=$(dirname $(portable_realpath $0)) -root=$(portable_realpath "$root/..") +root="$proteus_root" # Separate the --force flag from the optional install-path argument. -force=false -install_path="" -for arg in "$@"; do - if [ "$arg" = "--force" ]; then - force=true - elif [ -z "$install_path" ]; then - install_path="$arg" +get_parse_args "$@" + +if [ -n "$get_install_path" ]; then + socpath="$(portable_realpath "$get_install_path")" + # set -euo pipefail is not active yet, so an unresolvable path would + # otherwise reach git clone as an empty string. + if [ -z "$socpath" ]; then + echo "ERROR: could not resolve install path '$get_install_path'." >&2 + exit 1 fi -done - -if [ -n "$install_path" ]; then - socpath="$(portable_realpath "$install_path")" else socpath="$root/socrates" fi -# Refuse to delete a checkout holding local work unless --force is given. -# Keep this guard in sync across the get_* scripts that refresh checkouts. -# Guarded states: modified tracked files, and commits not on any remote. -# Untracked files (the compiled build tree) do not block the refresh. -# make/Mk_cmd is excluded: configure regenerates it on every build (compiler -# detection, host paths, and optimisation flags), so it is regenerable build -# config rather than user work and would otherwise block every refresh. -if [ -d "$socpath/.git" ] && [ "$force" != true ]; then - dirty=$(git -C "$socpath" status --porcelain --untracked-files=no \ - -- ':(exclude)make/Mk_cmd' 2>/dev/null | head -1) - unpushed=$(git -C "$socpath" log HEAD --not --remotes --oneline 2>/dev/null | head -1) - if [ -n "$dirty" ] || [ -n "$unpushed" ]; then - echo "ERROR: $socpath has uncommitted changes or commits not on a remote." >&2 - echo " Refusing to delete it. Commit and push your work, or run" >&2 - echo " bash tools/get_socrates.sh --force to discard the checkout." >&2 - exit 1 - fi -fi +# make/Mk_cmd is excluded from the dirty test: configure regenerates it on +# every build (compiler detection, host paths, and optimisation flags), so +# it is regenerable build config rather than user work and would otherwise +# block every refresh. +guard_dirty_checkout "$socpath" get_socrates.sh -- ':(exclude)make/Mk_cmd' rm -rf "$socpath" set -euo pipefail @@ -93,13 +75,12 @@ set -euo pipefail # Resolve the pinned URL + ref from pyproject.toml. The HTTPS URL is the # default; SSH is used only when ssh -T against github succeeded above. -soc_url=$(python "$root/tools/_module_pins.py" socrates url) -soc_ref=$(python "$root/tools/_module_pins.py" socrates ref) +resolve_module_pin socrates +soc_url="$module_url" +soc_ref="$module_ref" if [ "$use_ssh" = true ]; then - # Rewrite https://github.com/ -> git@github.com: for SSH transport. - soc_ssh_url=${soc_url/https:\/\/github.com\//git@github.com:} - git clone "$soc_ssh_url" "$socpath" + git clone "$(github_ssh_url "$soc_url")" "$socpath" else git clone "$soc_url" "$socpath" fi diff --git a/tools/get_spider.sh b/tools/get_spider.sh index 0514d9d60..4e48b9ef9 100755 --- a/tools/get_spider.sh +++ b/tools/get_spider.sh @@ -32,17 +32,14 @@ set -e # ----------------------------------------------------------------------------- -# Portable realpath: macOS <13 (Catalina through Monterey) does not ship -# GNU coreutils realpath. Fall back to python3, which is always available -# in PROTEUS's conda environment. +# Shared helpers, portable_realpath among them: see tools/_get_common.sh. # ----------------------------------------------------------------------------- -portable_realpath() { - if command -v realpath >/dev/null 2>&1; then - realpath "$1" - else - python3 -c "import os,sys; print(os.path.realpath(sys.argv[1]))" "$1" - fi -} +_get_common="$(dirname "${BASH_SOURCE[0]}")/_get_common.sh" +if [ ! -f "$_get_common" ]; then + echo "ERROR: $_get_common is missing; use a complete PROTEUS checkout." >&2 + exit 1 +fi +source "$_get_common" # ----------------------------------------------------------------------------- # Error handling: report which step failed on any non-zero exit @@ -108,14 +105,12 @@ fi # ----------------------------------------------------------------------------- current_step="Validating PETSc installation" -# Derive the repo root from this script's location (tools/get_spider.sh). -# This avoids dependence on the caller's CWD — important when invoked by -# data.py:get_spider() which does not set cwd. -script_dir="$(cd "$(dirname "$0")" && pwd)" -repo_root="$(dirname "$script_dir")" - -# PETSc is expected at /petsc/. -petsc_path="$repo_root/petsc" +# The repo root comes from this script's location, through the shared +# helpers, rather than from the caller's CWD: data.py:get_spider() invokes +# the script without setting cwd. +# +# PETSc is expected at /petsc/. +petsc_path="$proteus_root/petsc" if [[ ! -d "$petsc_path" ]]; then echo "ERROR: petsc/ directory not found at $petsc_path." echo "Run ./tools/get_petsc.sh first to install PETSc." @@ -202,34 +197,13 @@ current_step="Cloning SPIDER from GitHub" # Default install directory: ./SPIDER/ ; override via first argument. # The --force flag is separated from the optional path argument. -force=false -install_path="" -for arg in "$@"; do - if [ "$arg" = "--force" ]; then - force=true - elif [ -z "$install_path" ]; then - install_path="$arg" - fi -done +get_parse_args "$@" workpath="SPIDER" -if [[ -n "$install_path" ]]; then - workpath="$install_path" +if [[ -n "$get_install_path" ]]; then + workpath="$get_install_path" fi -# Refuse to delete a checkout holding local work unless --force is given. -# Keep this guard in sync across the get_* scripts that refresh checkouts. -# Guarded states: modified tracked files, and commits not on any remote. -# Untracked files (build artifacts) do not block the refresh. -if [ -d "$workpath/.git" ] && [ "$force" != true ]; then - dirty=$(git -C "$workpath" status --porcelain --untracked-files=no 2>/dev/null | head -1) - unpushed=$(git -C "$workpath" log HEAD --not --remotes --oneline 2>/dev/null | head -1) - if [ -n "$dirty" ] || [ -n "$unpushed" ]; then - echo "ERROR: $workpath has uncommitted changes or commits not on a remote." >&2 - echo " Refusing to delete it. Commit and push your work, or run" >&2 - echo " bash tools/get_spider.sh --force to discard the checkout." >&2 - exit 1 - fi -fi +guard_dirty_checkout "$workpath" get_spider.sh # Remove any previous installation if [[ -d "$workpath" ]]; then @@ -242,9 +216,8 @@ echo "Cloning SPIDER from GitHub..." # Resolve the pinned URL + ref from pyproject.toml. Allow override via # the SPIDER_GIT_URL / SPIDER_GIT_REF env vars for local dev. -script_root="$(cd "$(dirname "$0")/.." && pwd)" -sp_url="${SPIDER_GIT_URL:-$(python "$script_root/tools/_module_pins.py" spider url)}" -sp_ref="${SPIDER_GIT_REF:-$(python "$script_root/tools/_module_pins.py" spider ref)}" +sp_url="${SPIDER_GIT_URL:-$(python "$proteus_tools_dir/_module_pins.py" spider url)}" +sp_ref="${SPIDER_GIT_REF:-$(python "$proteus_tools_dir/_module_pins.py" spider ref)}" git clone "$sp_url" "$workpath" git -C "$workpath" checkout --quiet "$sp_ref" diff --git a/tools/get_thermoenginelite.sh b/tools/get_thermoenginelite.sh index 589c4069f..5b8249170 100755 --- a/tools/get_thermoenginelite.sh +++ b/tools/get_thermoenginelite.sh @@ -13,68 +13,36 @@ if ! [ -x "$(command -v pip)" ]; then exit 1 fi -portable_realpath() { - if command -v realpath >/dev/null 2>&1; then - realpath "$1" - else - python3 -c "import os,sys; print(os.path.realpath(sys.argv[1]))" "$1" - fi -} +# Shared helpers: see tools/_get_common.sh. +_get_common="$(dirname "${BASH_SOURCE[0]}")/_get_common.sh" +if [ ! -f "$_get_common" ]; then + echo "ERROR: $_get_common is missing; use a complete PROTEUS checkout." >&2 + exit 1 +fi +source "$_get_common" # Path to PROTEUS folder -root=$(dirname "$(portable_realpath "$0")") -root=$(portable_realpath "$root/..") +root="$proteus_root" -# Refuse to delete a checkout holding local work unless --force is given. -# Keep this guard in sync across the get_* scripts that refresh checkouts. -# Guarded states: modified tracked files, and commits not on any remote. -# Untracked files (build artifacts, egg-info) do not block the refresh. -force=false -for arg in "$@"; do - [ "$arg" = "--force" ] && force=true -done +get_parse_args "$@" workpath="$root/ThermoEngineLite/" -if [ -d "$workpath/.git" ] && [ "$force" != true ]; then - dirty=$(git -C "$workpath" status --porcelain --untracked-files=no 2>/dev/null | head -1) - unpushed=$(git -C "$workpath" log HEAD --not --remotes --oneline 2>/dev/null | head -1) - if [ -n "$dirty" ] || [ -n "$unpushed" ]; then - echo "ERROR: $workpath has uncommitted changes or commits not on a remote." >&2 - echo " Refusing to delete it. Commit and push your work, or run" >&2 - echo " bash tools/get_thermoenginelite.sh --force to discard the checkout." >&2 - exit 1 - fi -fi +guard_dirty_checkout "$workpath" get_thermoenginelite.sh # Make room rm -rf "$workpath" -# Detect SSH access to GitHub. `ssh -T git@github.com` exits 1 when -# authentication succeeds (GitHub refuses the shell), so a plain call -# would trip `set -e`; keeping it as the `if` condition keeps it in -# scope where a non-zero exit is expected rather than fatal. -if ssh -T git@github.com; then - use_ssh=false -else - if [ $? -eq 1 ]; then - use_ssh=true - else - use_ssh=false - fi -fi +# Detect SSH access to GitHub. +use_ssh=$(github_use_ssh) # Resolve the pinned URL + ref from pyproject.toml. -l_url=$(python "$root/tools/_module_pins.py" thermoenginelite url) -l_ref=$(python "$root/tools/_module_pins.py" thermoenginelite ref) -if [ -z "$l_url" ] || [ -z "$l_ref" ]; then - echo "ERROR: could not resolve thermoenginelite url/ref from pyproject.toml" >&2 - exit 1 -fi +resolve_module_pin thermoenginelite +l_url="$module_url" +l_ref="$module_ref" echo "Cloning from GitHub" if [ "$use_ssh" = true ]; then - # Rewrite https://github.com/ -> git@github.com: for SSH transport. - uri=${l_url/https:\/\/github.com\//git@github.com:} + uri=$(github_ssh_url "$l_url") else uri="$l_url" fi diff --git a/tools/get_vulcan.sh b/tools/get_vulcan.sh index 5bc9790ec..d8f3c88d0 100755 --- a/tools/get_vulcan.sh +++ b/tools/get_vulcan.sh @@ -14,54 +14,26 @@ set -euo pipefail echo "Set up VULCAN..." -portable_realpath() { - if command -v realpath >/dev/null 2>&1; then - realpath "$1" - else - python3 -c "import os,sys; print(os.path.realpath(sys.argv[1]))" "$1" - fi -} +# Shared helpers: see tools/_get_common.sh. +_get_common="$(dirname "${BASH_SOURCE[0]}")/_get_common.sh" +if [ ! -f "$_get_common" ]; then + echo "ERROR: $_get_common is missing; use a complete PROTEUS checkout." >&2 + exit 1 +fi +source "$_get_common" # Path to PROTEUS folder -root=$(dirname "$(portable_realpath "$0")") -root=$(portable_realpath "$root/..") +root="$proteus_root" -# Refuse to delete a checkout holding local work unless --force is given. -# Keep this guard in sync across the get_* scripts that refresh checkouts. -# Guarded states: modified tracked files, and commits not on any remote. -# Untracked files (build artifacts, egg-info) do not block the refresh. -force=false -for arg in "$@"; do - [ "$arg" = "--force" ] && force=true -done +get_parse_args "$@" workpath="$root/VULCAN/" -if [ -d "$workpath/.git" ] && [ "$force" != true ]; then - dirty=$(git -C "$workpath" status --porcelain --untracked-files=no 2>/dev/null | head -1) - unpushed=$(git -C "$workpath" log HEAD --not --remotes --oneline 2>/dev/null | head -1) - if [ -n "$dirty" ] || [ -n "$unpushed" ]; then - echo "ERROR: $workpath has uncommitted changes or commits not on a remote." >&2 - echo " Refusing to delete it. Commit and push your work, or run" >&2 - echo " bash tools/get_vulcan.sh --force to discard the checkout." >&2 - exit 1 - fi -fi +guard_dirty_checkout "$workpath" get_vulcan.sh # Make room rm -rf "$workpath" -# Detect SSH access to GitHub. `ssh -T git@github.com` exits 1 when -# authentication succeeds (GitHub refuses the shell), so a plain call -# would trip `set -e`; keeping it as the `if` condition keeps it in -# scope where a non-zero exit is expected rather than fatal. -if ssh -T git@github.com; then - use_ssh=false -else - if [ $? -eq 1 ]; then - use_ssh=true - else - use_ssh=false - fi -fi +# Detect SSH access to GitHub. +use_ssh=$(github_use_ssh) # Download echo "Cloning from GitHub" diff --git a/tools/get_zalmoxis.sh b/tools/get_zalmoxis.sh index 96e78293a..389b4a2a8 100755 --- a/tools/get_zalmoxis.sh +++ b/tools/get_zalmoxis.sh @@ -17,17 +17,16 @@ echo "Set up Zalmoxis..." -portable_realpath() { - if command -v realpath >/dev/null 2>&1; then - realpath "$1" - else - python3 -c "import os,sys; print(os.path.realpath(sys.argv[1]))" "$1" - fi -} +# Shared helpers: see tools/_get_common.sh. +_get_common="$(dirname "${BASH_SOURCE[0]}")/_get_common.sh" +if [ ! -f "$_get_common" ]; then + echo "ERROR: $_get_common is missing; use a complete PROTEUS checkout." >&2 + exit 1 +fi +source "$_get_common" # Path to PROTEUS folder -root=$(dirname $(portable_realpath $0)) -root=$(portable_realpath "$root/..") +root="$proteus_root" # Paired-branch detection. When PROTEUS is on a feature branch and Zalmoxis has # a branch of the same name, install that Zalmoxis branch so CI exercises the @@ -40,36 +39,15 @@ if [ -z "$proteus_branch" ]; then proteus_branch=$(git -C "$root" rev-parse --abbrev-ref HEAD 2>/dev/null) fi -# Refuse to delete a checkout holding local work unless --force is given. -# Keep this guard in sync across the get_* scripts that refresh checkouts. -# Guarded states: modified tracked files, and commits not on any remote. -# Untracked files (build artifacts, egg-info) do not block the refresh. -force=false -for arg in "$@"; do - [ "$arg" = "--force" ] && force=true -done +get_parse_args "$@" workpath=$root/Zalmoxis/ -if [ -d "$workpath/.git" ] && [ "$force" != true ]; then - dirty=$(git -C "$workpath" status --porcelain --untracked-files=no 2>/dev/null | head -1) - unpushed=$(git -C "$workpath" log HEAD --not --remotes --oneline 2>/dev/null | head -1) - if [ -n "$dirty" ] || [ -n "$unpushed" ]; then - echo "ERROR: $workpath has uncommitted changes or commits not on a remote." >&2 - echo " Refusing to delete it. Commit and push your work, or run" >&2 - echo " bash tools/get_zalmoxis.sh --force to discard the checkout." >&2 - exit 1 - fi -fi +guard_dirty_checkout "$workpath" get_zalmoxis.sh # Make room rm -rf $workpath # Check SSH access to GitHub -ssh -T git@github.com -if [ $? -eq 1 ]; then - use_ssh=true -else - use_ssh=false -fi +use_ssh=$(github_use_ssh) # Download echo "Cloning from GitHub" From 17cfad0a52e6da848cf629c56d488bf707be5c78 Mon Sep 17 00:00:00 2001 From: "E. G. Patrick Bos" Date: Wed, 9 Sep 2026 14:05:59 +0200 Subject: [PATCH 2/6] Cover the shared install-script helpers and every clone destination The cases for the install scripts read tools/_get_common.sh rather than carrying their own copy of the shell under test, so a change to a helper re-runs through them. They pin what a shared helper can silently change: the checkout root derived from the library's own location, the --force and install-path split in either argument order, the refusal to delete a checkout holding local work and the pathspec exclusion get_socrates.sh passes it, the SSH probe exit codes and its GIT_SSH_COMMAND override, the https-to-SSH URL rewrite, and a missing module pin stopping the install. Each runs under a plain shell and under set -euo pipefail, which is the split across the scripts themselves. Two invariants stand behind the rest: no get_*.sh carries a private copy of a helper, and every script that calls one also sources the file defining it. A script that grows a private copy again is outside the reach of these cases, which is what let the same four-line path fix be needed in nine places. Each script also runs whole against a stubbed git and ssh in a throwaway checkout, pinning the destination it resolves and the transport it selects, including a SOCRATES install path that does not exist yet and the https fallback when no key is accepted. --- tests/tools/test_install_scripts.py | 803 +++++++++++++++++++++++++--- 1 file changed, 723 insertions(+), 80 deletions(-) diff --git a/tests/tools/test_install_scripts.py b/tests/tools/test_install_scripts.py index f876704f6..4437c36c8 100644 --- a/tests/tools/test_install_scripts.py +++ b/tests/tools/test_install_scripts.py @@ -4,18 +4,35 @@ Reusable shell logic replicated inline from ``tools/get_petsc.sh`` and ``tools/get_spider.sh``: -- ``portable_realpath()``: cross-platform path resolution - ERR trap: exit-code and step-name capture - Platform detection: PETSC_ARCH assignment - Homebrew prefix fallback: architecture-aware default - Workpath argument handling: ``$1`` override vs default - PETSc library detection: versioned ``.so``, ``.dylib``, missing +``tools/_get_common.sh``, the helper library every ``get_*.sh`` sources, is +sourced by the cases below rather than copied into them, so they run against +the shipped text: +- ``portable_realpath()``: cross-platform path resolution, including a + destination that does not exist yet +- the checkout root and tools directory the library derives from its own + location +- ``get_parse_args``: the ``--force`` switch and the optional install path +- ``guard_dirty_checkout``: the refusal to delete local work, and the + pathspec exclusion ``get_socrates.sh`` passes it +- ``github_use_ssh`` and ``github_ssh_url``: the SSH probe and URL rewrite +- ``resolve_module_pin``: a missing pin stops the install +- the bootstrap in each script, which stops when the library is absent, and + the invariant that no script carries a private copy of a helper + Blocks lifted out of the shipped scripts at run time, so that rewording a script re-runs its cases against the new text: -- ``tools/get_aragog.sh``: the dirty-checkout guard shared across ``get_*.sh`` -- ``tools/get_socrates.sh``: the portable-flag rewrite, its post-build flag - check, and the conditional AGNI-wrapper rebuild note +- ``tools/get_socrates.sh``: the install-path resolution, the portable-flag + rewrite, its post-build flag check, and the conditional AGNI-wrapper + rebuild note + +Whole scripts run against stubbed ``git`` and ``ssh``, to pin the clone +destination and transport each one resolves. Also pins invariants that live in checked-in configuration and documentation rather than in shell, each of which fails silently when its counterpart moves: @@ -39,26 +56,61 @@ from __future__ import annotations import os +import shutil import subprocess import sys +from pathlib import Path import pytest +pytestmark = [pytest.mark.unit, pytest.mark.timeout(30)] + +TOOLS_DIR = Path(__file__).resolve().parents[2] / 'tools' +COMMON_LIB = TOOLS_DIR / '_get_common.sh' + # --------------------------------------------------------------------------- -# Helper: extract portable_realpath function from a script +# Helpers: run bash against the shipped helper library # --------------------------------------------------------------------------- -def _portable_realpath_fn() -> str: - """Return the bash source for ``portable_realpath()``.""" - return """\ -portable_realpath() { - if command -v realpath >/dev/null 2>&1; then - realpath "$1" - else - python3 -c "import os,sys; print(os.path.realpath(sys.argv[1]))" "$1" - fi -} -""" +def _with_common(body: str, strict: bool = False) -> str: + """Return a bash snippet that sources the shipped helper library. + + The library is sourced, not copied, so a change to it re-runs through + every case below. ``strict`` mirrors the ``get_*`` scripts that enable + ``set -euo pipefail``: the helpers must behave the same either way. + """ + prelude = 'set -euo pipefail\n' if strict else '' + return f'{prelude}source "{COMMON_LIB}"\n{body}' + + +def _run_bash(snippet: str, *argv: str, **kwargs) -> subprocess.CompletedProcess: + """Run ``snippet`` with ``argv`` as its positional parameters.""" + return subprocess.run( + ['bash', '-c', snippet, 'get_test.sh', *argv], + capture_output=True, + text=True, + **kwargs, + ) + + +def _extract_script_block(script: str, start_marker: str, end_marker: str) -> str: + """Return the shipped lines of ``tools/