From 0c4b72e2c5df096610989ac171212e22a08f1898 Mon Sep 17 00:00:00 2001 From: Andy Loftus Date: Mon, 23 Mar 2026 21:13:27 +0000 Subject: [PATCH 1/8] CTT-947 support for python 3.12 use importlib.metadata for version * pkg_resources was removed from setuptools * removes dependency on setuptools fixup grep WARNING utils to make lmod cache and compare spider to ipf faq how to compare module spider with ipf Renamed glue2/types.py so that in python 3.9 and greater, it doesn't conflict with functool's types.py. (Also changed IPF references from types to glue2types) update uv_installer to 0.12.5 set max python 3.12 set default python to 3.12 --- docs/CHANGELOG | 7 ++ docs/faq.md | 65 +++++++++++++----- ipf/bin/mk_lmod_cache.sh | 45 +++++++++++++ ipf/glue2/{types.py => glue2types.py} | 0 ipf/glue2/installedservices.py | 2 +- ipf/glue2/modules.py | 2 +- ipf/glue2/openstack.py | 6 +- ipf/glue2/valet.py | 2 +- ipf/ipfinfo.py | 8 +-- pyproject.toml | 2 +- setup.sh | 7 +- utils/ipf-setup-wrapper.sh | 81 +++++++++++++++++----- utils/ipf_pkgs.sh | 31 +++++++++ utils/mk_lmod_cache.sh | 96 +++++++++++++++++++++++++++ utils/module_spider.sh | 18 +++++ utils/uv_installer.sh | 50 +++++++------- 16 files changed, 349 insertions(+), 73 deletions(-) create mode 100644 ipf/bin/mk_lmod_cache.sh rename ipf/glue2/{types.py => glue2types.py} (100%) create mode 100755 utils/ipf_pkgs.sh create mode 100755 utils/mk_lmod_cache.sh create mode 100755 utils/module_spider.sh diff --git a/docs/CHANGELOG b/docs/CHANGELOG index 7fdb263..4bb9b71 100644 --- a/docs/CHANGELOG +++ b/docs/CHANGELOG @@ -1,3 +1,10 @@ +for 1.9.9 +* CTT-947 support for python 3.12 + . use importlib.metadata for version (instead of setuptool's pkg_resources) +* Renamed glue2/types.py so that in python 3.9 and greater, it doesn't conflict with functool's types.py. (Also changed IPF references from types to glue2types). +* utils to make lmod cache and compare spider to ipf +* faq how to compare module spider with ipf + for 1.9.8 CTT-986 include uv installer (instead of relying on cloud version) diff --git a/docs/faq.md b/docs/faq.md index fa93167..31e659e 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,6 +1,8 @@ # Frequently Asked Questions -## How do I upgrade to the latest version? +## INSTALLATION / UPGRADE + +### How do I upgrade to the latest version? 1. Run the install script again. The `pip` command will check for a newer version and install it. * ```bash @@ -11,8 +13,7 @@ Note: Check the pip output to verify the new version is installed. If a newer version is available but it wasn't installed, see the section below: "Pip won't install the latest version of ipf?". - -## How can I start over from scratch? +### How can I start over from scratch? 1. Stop any running workflows * ```bash bash ~/ipf/bin/wfm stop @@ -44,14 +45,35 @@ install the latest version of ipf?". * See steps in [Configure Software Modules Publishing](03_configure-extmodules-workflow.md) -## Can I configure multiple workflows of the same type? +### Pip won't install the latest version of ipf? +Two possible workarounds: +* `rm -rf ~/ipf/` + +OR + +* `rm -rf /tmp/pip-build*` + +Then re-run the installer. + +See also: +https://stackoverflow.com/questions/14617136/why-is-pip-installing-an-old-version-of-my-package + +### What version of ipf is currently installed? +```bash +~/ipf/.venv/bin/pip freeze +``` + + +## CONFIGURATION + +### Can I configure multiple workflows of the same type? Yes! The `configure_extmodules` script will look for config files matching the naming convention `configure_extmodules*.conf`. You can create multiple config files and a workflow definition will be created for each one. Just make sure that `RESOURCE_NAME` is unique in each config file. -## How can I backup my workflow configs? +### How can I backup my workflow configs? 1. Backup workflow configs * ```bash bash ~/ipf/bin/save_configs.sh @@ -64,20 +86,27 @@ On a re-install, the IPF installer will look for any backed up config files and re-make the symlinks. -## Pip won't install the latest version of ipf? -Two possible workarounds: -* `rm -rf ~/ipf/` -OR +## TROUBLESHOOTING -* `rm -rf /tmp/pip-build*` +### Modules are missing from IPF's published list +Check the permissions on the module lua file. Ipf must be able to read the +module lua file in order to get the data required for publishing. -Then re-run the installer. - -See also: -https://stackoverflow.com/questions/14617136/why-is-pip-installing-an-old-version-of-my-package +### Module name is spelled differently than what "module spider" reports +Check the spelling in the module lua file. Ipf reads the module name from the +lua file. -## What version if ipf is currently installed? -```bash -~/ipf/.venv/bin/pip freeze -``` +### General troubleshooting hints +* Check the ipf log file for warnings + 1. Log file location is given in the output of `~/ipf/bin/wfm ls` + 1. `grep -F WARNING $(~/ipf/bin/wfm ls | awk '$1=="LOG:" {print $2}')` +* Compare output from module spider with ipf collected data + 1. `mkdir -p ~/ipf/utils` + 1. `pushd ~/ipf/utils` + 1. `curl -O https://raw.githubusercontent.com/access-ci-org/ipf/refs/heads/CTT-893/lmod_cache/utils/module_spider.sh` + 1. `curl -O https://raw.githubusercontent.com/access-ci-org/ipf/refs/heads/CTT-893/lmod_cache/utils/ipf_pkgs.sh` + 1. `comm -3 <(./module_spider.sh) <(ipf_pkgs.sh)` + 1. For any unexpected differences: + 1. Check lmod lua file permissions and contents + 1. Check lmod cache file contents and age diff --git a/ipf/bin/mk_lmod_cache.sh b/ipf/bin/mk_lmod_cache.sh new file mode 100644 index 0000000..3672a4b --- /dev/null +++ b/ipf/bin/mk_lmod_cache.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +INSTALL_DIR=___INSTALL_DIR___ +. ${INSTALL_DIR}/lib/utils.sh + + +SYSTEM_FILE="${HOME}"/.lmod/mData/system.txt +CACHE_DIR="${HOME}"/.lmod/mData/cacheDir +CACHE_FILE="${CACHE_DIR}"/spiderT.lua + + +assert_cache_dir() { + mkdir -p "${CACHE_DIR}" + [[ -d "${CACHE_DIR}" ]] || die "cache dir not found: '${CACHE_DIR}' " +} + + +assert_environment() { + [[ -z "${LMOD_DIR}" ]] && die "LMOD_DIR not defined" + [[ -d "${LMOD_DIR}" ]] || die "LMDO_DIR not found: '${LMOD_DIR}'" +} + + +mk_lmod_cache() { + "${LMOD_DIR}"/update_lmod_system_cache_files \ + -d "${CACHE_DIR}" \ + -t "${SYSTEM_FILE}" \ + $MODULEPATH + + [[ -f "${CACHE_FILE}" ]] || die "cache file not found: '${CACHE_FILE}'" + [[ -f "${SYSTEM_FILE}" ]] || die "system timestamp file not found: '${SYSTEM_FILE}'" +} + + +### +# MAIN +### + +[[ $DEBUG -eq $YES ]] && set -x + +assert_cache_dir + +assert_environment + +mk_lmod_cache diff --git a/ipf/glue2/types.py b/ipf/glue2/glue2types.py similarity index 100% rename from ipf/glue2/types.py rename to ipf/glue2/glue2types.py diff --git a/ipf/glue2/installedservices.py b/ipf/glue2/installedservices.py index 4ff1ca1..87d292b 100644 --- a/ipf/glue2/installedservices.py +++ b/ipf/glue2/installedservices.py @@ -25,7 +25,7 @@ #from . import endpoint from .step import GlueStep #from .step import computing_service -from .types import AppEnvState, ApplicationHandle +from .glue2types import AppEnvState, ApplicationHandle from . import computing_activity from . import computing_manager from . import computing_service diff --git a/ipf/glue2/modules.py b/ipf/glue2/modules.py index 2b18cb6..30ddd38 100644 --- a/ipf/glue2/modules.py +++ b/ipf/glue2/modules.py @@ -24,7 +24,7 @@ from ipf.urnprefix import IPF_URN_PREFIX from . import application from . import service -from .types import AppEnvState, ApplicationHandle +from .glue2types import AppEnvState, ApplicationHandle ####################################################################################################################### diff --git a/ipf/glue2/openstack.py b/ipf/glue2/openstack.py index 06fec9f..59c9b01 100644 --- a/ipf/glue2/openstack.py +++ b/ipf/glue2/openstack.py @@ -34,7 +34,7 @@ import glue2.computing_service import glue2.computing_share import glue2.execution_environment -import glue2.types +import glue2.glue2types from . import accelerator_environment from . import computing_manager_accel_info @@ -412,9 +412,9 @@ def _addImage(self, image, apps): # env.Description = ??? # env.Repository = if image.status == "ACTIVE": - env.State = glue2.types.AppEnvState.INSTALLED_NOT_VERIFIED + env.State = glue2.glue2types.AppEnvState.INSTALLED_NOT_VERIFIED elif image.status == "SAVING": - env.State = glue2.types.AppEnvState.INSTALLABLE + env.State = glue2.glue2types.AppEnvState.INSTALLABLE else: self.warning("unknown image status: "+image.status) env.Extension["Updated"] = image.updated diff --git a/ipf/glue2/valet.py b/ipf/glue2/valet.py index 19e38e4..5015d5f 100644 --- a/ipf/glue2/valet.py +++ b/ipf/glue2/valet.py @@ -24,7 +24,7 @@ from ipf.error import StepError from ipf.urnprefix import IPF_URN_PREFIX from . import application -from .types import AppEnvState, ApplicationHandle +from .glue2types import AppEnvState, ApplicationHandle ####################################################################################################################### diff --git a/ipf/ipfinfo.py b/ipf/ipfinfo.py index 5b44afc..c0ae302 100644 --- a/ipf/ipfinfo.py +++ b/ipf/ipfinfo.py @@ -17,7 +17,7 @@ import platform import socket -import pkg_resources +import importlib.metadata import os import json @@ -37,7 +37,7 @@ class IPFVersionStep(Step): def __init__(self): Step.__init__(self) - self.description = "produces an IPF Version document using the pkg_resources version" + self.description = "produces an IPF Version document" self.time_out = 5 self.produces = [IPFVersion] self._acceptParameter( @@ -48,8 +48,8 @@ def run(self): ipf_version = self.params["ipf_version"] except KeyError: try: - ipf_version = pkg_resources.get_distribution("IPF").version - except: + ipf_version = importlib.metadata.version( "ipf" ) + except importlib.metadata.PackageNotFoundError: ipf_version = "unknown" self._output(IPFVersion(ipf_version)) diff --git a/pyproject.toml b/pyproject.toml index 639d073..fae9922 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta:__legacy__" [project] name = "ipf" -requires-python = ">=3.6" +requires-python = ">=3.6,<=3.12" dependencies = [ "amqp >=1.4", "lupa", diff --git a/setup.sh b/setup.sh index 1b4add0..fffc591 100644 --- a/setup.sh +++ b/setup.sh @@ -17,7 +17,7 @@ VERBOSE=$YES # NO USER CHANGES AFTER THIS -export UV_PYTHON=3.9 #python version required by IPF +export UV_PYTHON=3.12 #python version required by IPF UV_DIR="${INSTALL_DIR}"/uv UV="${UV_DIR}"/uv VENV="$INSTALL_DIR"/.venv @@ -78,6 +78,7 @@ mk_venv() { export UV_PYTHON_INSTALL_DIR="${UV_DIR}"/python export UV_NO_CACHE=1 export UV_MANAGED_PYTHON=1 + [[ -n "${IPF_UV_PYTHON}" ]] && export UV_PYTHON="${IPF_UV_PYTHON}" "$UV" venv "$VENV" success "Python venv created at '$VENV'" "${V_PYTHON}" -m ensurepip @@ -117,7 +118,7 @@ install_ipf_pre_release() { --pre \ --no-deps \ --index-url https://test.pypi.org/simple/ \ - ipf \ + ipf"${IPF_INSTALL_VERSION:+==$IPF_INSTALL_VERSION}" \ && success "Installed dev version of ipf" } @@ -126,7 +127,7 @@ install_ipf() { [[ $DEBUG -eq $YES ]] && set -x "${V_PYTHON}" -m pip install \ --upgrade \ - ipf \ + ipf"${IPF_INSTALL_VERSION:+==$IPF_INSTALL_VERSION}" \ && success "Ipf and dependencies installed" } diff --git a/utils/ipf-setup-wrapper.sh b/utils/ipf-setup-wrapper.sh index ba1f1d2..1afcb83 100644 --- a/utils/ipf-setup-wrapper.sh +++ b/utils/ipf-setup-wrapper.sh @@ -2,30 +2,79 @@ set -x -IPF_BIN="${HOME}"/ipf/bin -VENV="${HOME}"/ipf/.venv +# ========================= +# customize these as needed +# ========================= + +# where to save the installer script IPF_SETUP="${HOME}"/ipf-setup.sh -[[ -z "${IPF_GIT_BRANCH}" ]] && { - echo "Missing environment variable IPF_GIT_BRANCH" - exit 1 -} +# dont actually do anything, just print out a list of commands +# that can be run manually +# MANUAL=yes + +# set to an absolute path or leave unset (defaults to $HOME/ipf) +# IPF_INSTALL_DIR=/scratch/user/u.al8635/ipf + +# set to "yes" or leave unset (defaults to latest) +# IPF_ALLOW_PRE_RELEASE=yes + +# specify an exact version of IPF +# get version string from +# https://pypi.org/project/ipf/#history (production releases) +# OR https://test.pypi.org/project/ipf/#history (pre-release versions) +# IPF_INSTALL_VERSION=1.9.9.dev1783568317 + +# either a branch name or leave unset (defaults to master) +# IPF_GIT_BRANCH=ATS-30367 + +# specify a specific version of python if needed +# IPF_UV_PYTHON=3.12 + +# ===================================== +# shouldn't need to touch anything else +# ===================================== + + +[[ -z "${IPF_GIT_BRANCH}" ]] && IPF_GIT_BRANCH=master URL_BASE='https://raw.githubusercontent.com/access-ci-org/ipf/refs/heads' IPF_SETUP_URL="${URL_BASE}"/"${IPF_GIT_BRANCH}"/setup.sh +IPF_UV_URL="${URL_BASE}"/"${IPF_GIT_BRANCH}"/utils/uv_installer.sh +IPF_BIN="${HOME}"/ipf/bin +VENV="${HOME}"/ipf/.venv -bash "${IPF_BIN}"/wfm stop - -rm -rf ~/ipf +### +# MAIN +### -rm -f "${IPF_SETUP}" +# make exports (used by setup script) +export IPF_UV_URL +[[ -n "${IPF_INSTALL_DIR}" ]] && export IPF_INSTALL_DIR +[[ -n "${IPF_ALLOW_PRE_RELEASE}" ]] && export IPF_ALLOW_PRE_RELEASE +[[ -n "${IPF_UV_PYTHON}" ]] && export IPF_UV_PYTHON +[[ -n "${IPF_INSTALL_VERSION}" ]] && export IPF_INSTALL_VERSION -curl -o "${IPF_SETUP}" "${IPF_SETUP_URL}" +# set ACTION if doing manual run +if [[ -n "${MANUAL}" ]] ; then + ACTION=echo + set +x + echo 'COMMANDS THAT WOULD HAVE BEEN RUN' + echo '---------------------------------' +fi -export IPF_UV_URL="${URL_BASE}"/"${IPF_GIT_BRANCH}"/utils/uv_installer.sh -export IPF_ALLOW_PRE_RELEASE=yes +# stop anything running +[[ -x "${IPF_BIN}"/wfm ]] && ${ACTION} bash "${IPF_BIN}"/wfm stop -bash "${IPF_SETUP}" +# cleanup any existing installs +${ACTION} rm -rf ~/ipf +${ACTION} rm -rf ~/.cache/pip +${ACTION} rm -f "${IPF_SETUP}" -"${VENV}"/bin/python --version +# get new setup script +${ACTION} curl -o "${IPF_SETUP}" "${IPF_SETUP_URL}" +# and run it +${ACTION} bash "${IPF_SETUP}" -"${VENV}"/bin/python -m pip freeze +# validate install +${ACTION} "${VENV}"/bin/python --version +${ACTION} "${VENV}"/bin/python -m pip freeze diff --git a/utils/ipf_pkgs.sh b/utils/ipf_pkgs.sh new file mode 100755 index 0000000..b8d716e --- /dev/null +++ b/utils/ipf_pkgs.sh @@ -0,0 +1,31 @@ +#!/usr/bin/bash + +IPF_PATH=~/ipf +BIN="${IPF_PATH}"/bin +WFM="${BIN}"/wfm + + +get_json_data_file() { + "${WFM}" ls 2>&1 \ + | awk '/OUTPUT: / && /extended_modules.json/ {print $2}' \ + | head -1 +} + + + +get_clean_names() { + jq \ + '.ApplicationHandle[].Value | split("/")[0] | gsub("^\\s+|\\s+$"; "")' \ + "${JSON_PATH}" \ + | sort -u \ + | tr -d '"' +} + + +### +# MAIN +### + +JSON_PATH=$( get_json_data_file ) + +get_clean_names diff --git a/utils/mk_lmod_cache.sh b/utils/mk_lmod_cache.sh new file mode 100755 index 0000000..1daad57 --- /dev/null +++ b/utils/mk_lmod_cache.sh @@ -0,0 +1,96 @@ +#!/usr/bin/bash + +YES=0 +NO=1 + +VERBOSE=$YES +DEBUG=$YES + +# ANSI escape codes for colors +GREEN='\033[0;32m' +RED='\033[0;31m' +NC='\033[0m' # No Color + +err() { + echo -e "${RED}✗ ERROR: $*${NC}" +} + +success() { + echo -e "${GREEN}✓ $*${NC}" +} + +die() { + err "$*" + echo "from (${BASH_SOURCE[1]} [${BASH_LINENO[0]}] ${FUNCNAME[1]})" + kill 0 + exit 99 +} + +info() { + [[ $VERBOSE -eq $YES ]] && { + echo -e "${RED}INFO: ${NC}$*" 1>&2 + } +} + +debug() { + [[ $DEBUG -eq $YES ]] && { + echo -e "${RED}DEBUG: ${NC}$*" 1>&2 + } +} + + +set_jq() { + [[ $DEBUG -eq $YES ]] && set -x + JQ=$( which jq ) + [[ -x "${JQ}" ]] || die 'JQ not found' +} + + +set_lmod_vars() { + if [[ -z "${MODULESHOME}" ]] ; then + # module command prints on stderr and so does debug output + # so write module output to a file, then post-process with awk + set +x + module --config_json 2>tee module_config_json 1>/dev/null + LMOD_PREFIX=$( + awk '/^+/ {next}; /^\{/ {print};' module_config_json \ + | "${JQ}" -r '.configT.prefix' \ + | tr -d '"' + ) + [[ $DEBUG -eq $YES ]] && set -x + [[ -d "${LMOD_PREFIX}" ]] || die "lmod prefix '${LMOD_PREFIX}' not a dir" + # LMOD_PREFIX=$( module --config_json 2>&1 | "${JQ}" '.configT.prefix' | tr -d '"' ) + MODULES_HOME="${LMOD_PREFIX}"/lmod/lmod + fi + LMOD_EXEC="${MODULESHOME}"/libexec + LMOD_CACHE_UPDATER="${LMOD_EXEC}"/update_lmod_system_cache_files + [[ -f "${LMOD_CACHE_UPDATER}" ]] || die 'lmod cache update script not found' + [[ -x "${LMOD_CACHE_UPDATER}" ]] || die 'lmod cache update script not executable' +} + + +update_lmod_cache() { + [[ $DEBUG -eq $YES ]] && set -x + [[ -z "${MODULEPATH}" ]] && die 'empty MODULEPATH' + local _cache_dir _timestamp_fn + _cache_dir="${HOME}"/lmod_cache + _timestamp_fn="${HOME}"/lmod_cache.timestamp + mkdir -p "${_cache_dir}" + "${LMOD_CACHE_UPDATER}" \ + -d "${_cache_dir}" \ + -t "${_timestamp_fn}" \ + "${MODULEPATH}" +} + + +### +# MAIN +### + +[[ $DEBUG -eq $YES ]] && set -x + +set_jq + +set_lmod_vars + +update_lmod_cache diff --git a/utils/module_spider.sh b/utils/module_spider.sh new file mode 100755 index 0000000..6969913 --- /dev/null +++ b/utils/module_spider.sh @@ -0,0 +1,18 @@ +#!/usr/bin/bash + +SHOW_HIDDEN= + +ENDWHILE=0 +while [[ $# -gt 0 ]] && [[ $ENDWHILE -eq 0 ]] ; do + case $1 in + -s) SHOW_HIDDEN='--show_hidden';; + --) ENDWHILE=1;; + -*) echo "Invalid option '$1'"; exit 1;; + *) ENDWHILE=1; break;; + esac + shift +done + +module --redirect --terse ${SHOW_HIDDEN} spider \ +| awk -F'/' '/\/$/ {print $1}' \ +| sort -u diff --git a/utils/uv_installer.sh b/utils/uv_installer.sh index b6bac43..b3000ed 100644 --- a/utils/uv_installer.sh +++ b/utils/uv_installer.sh @@ -24,19 +24,19 @@ has_local 2>/dev/null || alias local=typeset set -u APP_NAME="uv" -APP_VERSION="0.11.17" +APP_VERSION="0.12.5" if [ -n "${UV_DOWNLOAD_URL:-}" ]; then ARTIFACT_DOWNLOAD_URLS="$UV_DOWNLOAD_URL" elif [ -n "${INSTALLER_DOWNLOAD_URL:-}" ]; then ARTIFACT_DOWNLOAD_URLS="$INSTALLER_DOWNLOAD_URL" elif [ -n "${UV_INSTALLER_GHE_BASE_URL:-}" ]; then INSTALLER_BASE_URL="$UV_INSTALLER_GHE_BASE_URL" - ARTIFACT_DOWNLOAD_URLS="${INSTALLER_BASE_URL}/astral-sh/uv/releases/download/0.11.17" + ARTIFACT_DOWNLOAD_URLS="${INSTALLER_BASE_URL}/astral-sh/uv/releases/download/0.12.5" elif [ -n "${UV_INSTALLER_GITHUB_BASE_URL:-}" ]; then INSTALLER_BASE_URL="$UV_INSTALLER_GITHUB_BASE_URL" - ARTIFACT_DOWNLOAD_URLS="${INSTALLER_BASE_URL}/astral-sh/uv/releases/download/0.11.17" + ARTIFACT_DOWNLOAD_URLS="${INSTALLER_BASE_URL}/astral-sh/uv/releases/download/0.12.5" else - ARTIFACT_DOWNLOAD_URLS="https://releases.astral.sh/github/uv/releases/download/0.11.17 https://github.com/astral-sh/uv/releases/download/0.11.17" + ARTIFACT_DOWNLOAD_URLS="https://releases.astral.sh/github/uv/releases/download/0.12.5 https://github.com/astral-sh/uv/releases/download/0.12.5" fi if [ -n "${UV_PRINT_VERBOSE:-}" ]; then PRINT_VERBOSE="$UV_PRINT_VERBOSE" @@ -66,7 +66,7 @@ fi AUTH_TOKEN="${UV_GITHUB_TOKEN:-}" read -r RECEIPT < Date: Mon, 17 Aug 2026 17:57:18 +0000 Subject: [PATCH 2/8] remove duplicate mk_lmod_cache --- ipf/bin/mk_lmod_cache.sh | 45 ---------------------------------------- 1 file changed, 45 deletions(-) delete mode 100644 ipf/bin/mk_lmod_cache.sh diff --git a/ipf/bin/mk_lmod_cache.sh b/ipf/bin/mk_lmod_cache.sh deleted file mode 100644 index 3672a4b..0000000 --- a/ipf/bin/mk_lmod_cache.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -INSTALL_DIR=___INSTALL_DIR___ -. ${INSTALL_DIR}/lib/utils.sh - - -SYSTEM_FILE="${HOME}"/.lmod/mData/system.txt -CACHE_DIR="${HOME}"/.lmod/mData/cacheDir -CACHE_FILE="${CACHE_DIR}"/spiderT.lua - - -assert_cache_dir() { - mkdir -p "${CACHE_DIR}" - [[ -d "${CACHE_DIR}" ]] || die "cache dir not found: '${CACHE_DIR}' " -} - - -assert_environment() { - [[ -z "${LMOD_DIR}" ]] && die "LMOD_DIR not defined" - [[ -d "${LMOD_DIR}" ]] || die "LMDO_DIR not found: '${LMOD_DIR}'" -} - - -mk_lmod_cache() { - "${LMOD_DIR}"/update_lmod_system_cache_files \ - -d "${CACHE_DIR}" \ - -t "${SYSTEM_FILE}" \ - $MODULEPATH - - [[ -f "${CACHE_FILE}" ]] || die "cache file not found: '${CACHE_FILE}'" - [[ -f "${SYSTEM_FILE}" ]] || die "system timestamp file not found: '${SYSTEM_FILE}'" -} - - -### -# MAIN -### - -[[ $DEBUG -eq $YES ]] && set -x - -assert_cache_dir - -assert_environment - -mk_lmod_cache From 2d88faa6360b65c1ed2b6ddcce82a006e36137da Mon Sep 17 00:00:00 2001 From: Andy Loftus Date: Mon, 17 Aug 2026 19:51:10 +0000 Subject: [PATCH 3/8] add credit to CHANGELOG --- docs/CHANGELOG | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CHANGELOG b/docs/CHANGELOG index 4bb9b71..69e2b8a 100644 --- a/docs/CHANGELOG +++ b/docs/CHANGELOG @@ -1,7 +1,7 @@ for 1.9.9 * CTT-947 support for python 3.12 . use importlib.metadata for version (instead of setuptool's pkg_resources) -* Renamed glue2/types.py so that in python 3.9 and greater, it doesn't conflict with functool's types.py. (Also changed IPF references from types to glue2types). +* Renamed glue2/types.py so that in python 3.9 and greater, it doesn't conflict with functool's types.py. (Also changed IPF references from types to glue2types). Credit: Eric Blau * utils to make lmod cache and compare spider to ipf * faq how to compare module spider with ipf From 4c694533b717d045cd054f8df2ddcf1d8bde5eb3 Mon Sep 17 00:00:00 2001 From: Andy Loftus Date: Mon, 17 Aug 2026 19:58:48 +0000 Subject: [PATCH 4/8] remove hardcoded paths from ipf-setup-wrapper helper utility --- utils/ipf-setup-wrapper.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/utils/ipf-setup-wrapper.sh b/utils/ipf-setup-wrapper.sh index 1afcb83..626201c 100644 --- a/utils/ipf-setup-wrapper.sh +++ b/utils/ipf-setup-wrapper.sh @@ -9,13 +9,13 @@ set -x # where to save the installer script IPF_SETUP="${HOME}"/ipf-setup.sh +# set to an absolute path (default is $HOME/ipf) +IPF_INSTALL_DIR="${HOME}"/ipf + # dont actually do anything, just print out a list of commands # that can be run manually # MANUAL=yes -# set to an absolute path or leave unset (defaults to $HOME/ipf) -# IPF_INSTALL_DIR=/scratch/user/u.al8635/ipf - # set to "yes" or leave unset (defaults to latest) # IPF_ALLOW_PRE_RELEASE=yes @@ -49,7 +49,7 @@ VENV="${HOME}"/ipf/.venv # make exports (used by setup script) export IPF_UV_URL -[[ -n "${IPF_INSTALL_DIR}" ]] && export IPF_INSTALL_DIR +export IPF_INSTALL_DIR [[ -n "${IPF_ALLOW_PRE_RELEASE}" ]] && export IPF_ALLOW_PRE_RELEASE [[ -n "${IPF_UV_PYTHON}" ]] && export IPF_UV_PYTHON [[ -n "${IPF_INSTALL_VERSION}" ]] && export IPF_INSTALL_VERSION @@ -66,8 +66,8 @@ fi [[ -x "${IPF_BIN}"/wfm ]] && ${ACTION} bash "${IPF_BIN}"/wfm stop # cleanup any existing installs -${ACTION} rm -rf ~/ipf -${ACTION} rm -rf ~/.cache/pip +${ACTION} rm -rf "${IPF_INSTALL_DIR}" +${ACTION} rm -rf "${HOME}"/.cache/pip ${ACTION} rm -f "${IPF_SETUP}" # get new setup script From 1be2ad19e4d0df7b772da75badd11badb7500326 Mon Sep 17 00:00:00 2001 From: Andy Loftus Date: Mon, 17 Aug 2026 20:01:41 +0000 Subject: [PATCH 5/8] more variable fixes for paths --- utils/ipf-setup-wrapper.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/ipf-setup-wrapper.sh b/utils/ipf-setup-wrapper.sh index 626201c..16363b8 100644 --- a/utils/ipf-setup-wrapper.sh +++ b/utils/ipf-setup-wrapper.sh @@ -40,8 +40,8 @@ IPF_INSTALL_DIR="${HOME}"/ipf URL_BASE='https://raw.githubusercontent.com/access-ci-org/ipf/refs/heads' IPF_SETUP_URL="${URL_BASE}"/"${IPF_GIT_BRANCH}"/setup.sh IPF_UV_URL="${URL_BASE}"/"${IPF_GIT_BRANCH}"/utils/uv_installer.sh -IPF_BIN="${HOME}"/ipf/bin -VENV="${HOME}"/ipf/.venv +IPF_BIN="${IPF_INSTALL_DIR}"/bin +VENV="${IPF_INSTALL_DIR}"/.venv ### # MAIN From b9bfb25070a38f9b1d2de3ee8b1e8111ced7b904 Mon Sep 17 00:00:00 2001 From: Andy Loftus Date: Mon, 17 Aug 2026 20:09:08 +0000 Subject: [PATCH 6/8] only need to export UV_PYTHON once --- setup.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.sh b/setup.sh index fffc591..bf8d1e8 100644 --- a/setup.sh +++ b/setup.sh @@ -17,7 +17,7 @@ VERBOSE=$YES # NO USER CHANGES AFTER THIS -export UV_PYTHON=3.12 #python version required by IPF +UV_PYTHON=3.12 #python version preferred by IPF UV_DIR="${INSTALL_DIR}"/uv UV="${UV_DIR}"/uv VENV="$INSTALL_DIR"/.venv @@ -78,7 +78,8 @@ mk_venv() { export UV_PYTHON_INSTALL_DIR="${UV_DIR}"/python export UV_NO_CACHE=1 export UV_MANAGED_PYTHON=1 - [[ -n "${IPF_UV_PYTHON}" ]] && export UV_PYTHON="${IPF_UV_PYTHON}" + export UV_PYTHON + [[ -n "${IPF_UV_PYTHON}" ]] && UV_PYTHON="${IPF_UV_PYTHON}" "$UV" venv "$VENV" success "Python venv created at '$VENV'" "${V_PYTHON}" -m ensurepip From 7b366b7a851aa42412c0f8962847c7442210a201 Mon Sep 17 00:00:00 2001 From: Andy Loftus Date: Mon, 17 Aug 2026 20:14:37 +0000 Subject: [PATCH 7/8] replace branch with master in absolute github URIs --- docs/faq.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 31e659e..99688aa 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -104,9 +104,10 @@ lua file. * Compare output from module spider with ipf collected data 1. `mkdir -p ~/ipf/utils` 1. `pushd ~/ipf/utils` - 1. `curl -O https://raw.githubusercontent.com/access-ci-org/ipf/refs/heads/CTT-893/lmod_cache/utils/module_spider.sh` - 1. `curl -O https://raw.githubusercontent.com/access-ci-org/ipf/refs/heads/CTT-893/lmod_cache/utils/ipf_pkgs.sh` + 1. `curl -O https://raw.githubusercontent.com/access-ci-org/ipf/refs/heads/master/utils/module_spider.sh` + 1. `curl -O https://raw.githubusercontent.com/access-ci-org/ipf/refs/heads/master/utils/ipf_pkgs.sh` 1. `comm -3 <(./module_spider.sh) <(ipf_pkgs.sh)` 1. For any unexpected differences: 1. Check lmod lua file permissions and contents 1. Check lmod cache file contents and age + 1. `pushd ~/ipf/utils` From e14b37d68b3506ea9ec59c451a3656c413e69f25 Mon Sep 17 00:00:00 2001 From: Andy Loftus Date: Mon, 17 Aug 2026 20:15:42 +0000 Subject: [PATCH 8/8] fix typo --- docs/faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faq.md b/docs/faq.md index 99688aa..15a98c1 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -110,4 +110,4 @@ lua file. 1. For any unexpected differences: 1. Check lmod lua file permissions and contents 1. Check lmod cache file contents and age - 1. `pushd ~/ipf/utils` + 1. `popd ~/ipf/utils`