access-ci-org · andylytical · Mar 23, 2026 · Aug 17, 2026 · Aug 17, 2026 · Aug 17, 2026
diff --git a/docs/CHANGELOG b/docs/CHANGELOG
@@ -1,3 +1,10 @@
+for 1.9.9
+* CTT-947 support for python 3.12
+  . use importlib.metadata for version (instead of setuptool's pkg_resources)
+* Renamed glue2/types.py so that in python 3.9 and greater, it doesn't conflict with functool's types.py. (Also changed IPF references from types to glue2types). Credit: Eric Blau
+* utils to make lmod cache and compare spider to ipf
+* faq how to compare module spider with ipf
+
 for 1.9.8
 CTT-986 include uv installer (instead of relying on cloud version)
 

diff --git a/docs/faq.md b/docs/faq.md
@@ -1,6 +1,8 @@
 # Frequently Asked Questions
 
-## How do I upgrade to the latest version?
+## INSTALLATION / UPGRADE
+
+### How do I upgrade to the latest version?
 1. Run the install script again. The `pip` command will check for a newer
    version and install it.
    * ```bash
@@ -11,8 +13,7 @@ Note: Check the pip output to verify the new version is installed. If a newer
 version is available but it wasn't installed, see the section below: "Pip won't
 install the latest version of ipf?".
 
-
-## How can I start over from scratch?
+### How can I start over from scratch?
 1. Stop any running workflows
    * ```bash
      bash ~/ipf/bin/wfm stop
@@ -44,14 +45,35 @@ install the latest version of ipf?".
    * See steps in [Configure Software Modules Publishing](03_configure-extmodules-workflow.md)
 
 
-## Can I configure multiple workflows of the same type?
+### Pip won't install the latest version of ipf?
+Two possible workarounds:
+* `rm -rf ~/ipf/`
+
+OR
+
+* `rm -rf /tmp/pip-build*`
+
+Then re-run the installer.
+
+See also:
+https://stackoverflow.com/questions/14617136/why-is-pip-installing-an-old-version-of-my-package
+
+### What version of ipf is currently installed?
+```bash
+~/ipf/.venv/bin/pip freeze
+```
+
+
+## CONFIGURATION
+
+### Can I configure multiple workflows of the same type?
 Yes!  The `configure_extmodules` script will look for config files matching the
 naming convention `configure_extmodules*.conf`. You can create multiple config
 files and a workflow definition will be created for each one. Just make sure
 that `RESOURCE_NAME` is unique in each config file.
 
 
-## How can I backup my workflow configs?
+### How can I backup my workflow configs?
 1. Backup workflow configs
    * ```bash
      bash ~/ipf/bin/save_configs.sh
@@ -64,20 +86,28 @@ On a re-install, the IPF installer will look for any backed up
 config files and re-make the symlinks.
 
 
-## Pip won't install the latest version of ipf?
-Two possible workarounds:
-* `rm -rf ~/ipf/`
 
-OR
+## TROUBLESHOOTING
 
-* `rm -rf /tmp/pip-build*`
+### Modules are missing from IPF's published list
+Check the permissions on the module lua file. Ipf must be able to read the
+module lua file in order to get the data required for publishing.
 
-Then re-run the installer.
-
-See also:
-https://stackoverflow.com/questions/14617136/why-is-pip-installing-an-old-version-of-my-package
+### Module name is spelled differently than what "module spider" reports
+Check the spelling in the module lua file. Ipf reads the module name from the
+lua file.
 
-## What version if ipf is currently installed?
-```bash
-~/ipf/.venv/bin/pip freeze
-```
+### General troubleshooting hints
+* Check the ipf log file for warnings
+  1. Log file location is given in the output of `~/ipf/bin/wfm ls`
+  1. `grep -F WARNING $(~/ipf/bin/wfm ls | awk '$1=="LOG:" {print $2}')`
+* Compare output from module spider with ipf collected data
+  1. `mkdir -p ~/ipf/utils`
+  1. `pushd ~/ipf/utils`
+  1. `curl -O https://raw.githubusercontent.com/access-ci-org/ipf/refs/heads/master/utils/module_spider.sh`
+  1. `curl -O https://raw.githubusercontent.com/access-ci-org/ipf/refs/heads/master/utils/ipf_pkgs.sh`
+  1. `comm -3 <(./module_spider.sh) <(ipf_pkgs.sh)`
+  1. For any unexpected differences:
+     1. Check lmod lua file permissions and contents
+     1. Check lmod cache file contents and age
+  1. `popd ~/ipf/utils`
diff --git a/ipf/glue2/types.py → ipf/glue2/glue2types.py b/ipf/glue2/types.py → ipf/glue2/glue2types.py
diff --git a/ipf/glue2/installedservices.py b/ipf/glue2/installedservices.py
@@ -25,7 +25,7 @@
 #from . import endpoint
 from .step import GlueStep
 #from .step import computing_service
-from .types import AppEnvState, ApplicationHandle
+from .glue2types import AppEnvState, ApplicationHandle
 from . import computing_activity
 from . import computing_manager
 from . import computing_service

diff --git a/ipf/glue2/modules.py b/ipf/glue2/modules.py
@@ -24,7 +24,7 @@
 from ipf.urnprefix import IPF_URN_PREFIX
 from . import application
 from . import service
-from .types import AppEnvState, ApplicationHandle
+from .glue2types import AppEnvState, ApplicationHandle
 
 #######################################################################################################################
 

diff --git a/ipf/glue2/openstack.py b/ipf/glue2/openstack.py
@@ -34,7 +34,7 @@
 import glue2.computing_service
 import glue2.computing_share
 import glue2.execution_environment
-import glue2.types
+import glue2.glue2types
 
 from . import accelerator_environment
 from . import computing_manager_accel_info
@@ -412,9 +412,9 @@ def _addImage(self, image, apps):
         # env.Description = ???
         # env.Repository =
         if image.status == "ACTIVE":
-            env.State = glue2.types.AppEnvState.INSTALLED_NOT_VERIFIED
+            env.State = glue2.glue2types.AppEnvState.INSTALLED_NOT_VERIFIED
         elif image.status == "SAVING":
-            env.State = glue2.types.AppEnvState.INSTALLABLE
+            env.State = glue2.glue2types.AppEnvState.INSTALLABLE
         else:
             self.warning("unknown image status: "+image.status)
         env.Extension["Updated"] = image.updated

diff --git a/ipf/glue2/valet.py b/ipf/glue2/valet.py
@@ -24,7 +24,7 @@
 from ipf.error import StepError
 from ipf.urnprefix import IPF_URN_PREFIX
 from . import application
-from .types import AppEnvState, ApplicationHandle
+from .glue2types import AppEnvState, ApplicationHandle
 
 #######################################################################################################################
 

diff --git a/ipf/ipfinfo.py b/ipf/ipfinfo.py
@@ -17,7 +17,7 @@
 
 import platform
 import socket
-import pkg_resources
+import importlib.metadata
 import os
 import json
 
@@ -37,7 +37,7 @@ class IPFVersionStep(Step):
     def __init__(self):
         Step.__init__(self)
 
-        self.description = "produces an IPF Version document using the pkg_resources version"
+        self.description = "produces an IPF Version document"
         self.time_out = 5
         self.produces = [IPFVersion]
         self._acceptParameter(
@@ -48,8 +48,8 @@ def run(self):
             ipf_version = self.params["ipf_version"]
         except KeyError:
             try:
-                ipf_version = pkg_resources.get_distribution("IPF").version
-            except:
+                ipf_version = importlib.metadata.version( "ipf" )
+            except importlib.metadata.PackageNotFoundError:
                 ipf_version = "unknown"
 
         self._output(IPFVersion(ipf_version))

diff --git a/pyproject.toml b/pyproject.toml
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta:__legacy__"
 
 [project]
 name = "ipf"
-requires-python = ">=3.6"
+requires-python = ">=3.6,<=3.12"
 dependencies = [
   "amqp >=1.4",
   "lupa",

diff --git a/setup.sh b/setup.sh
@@ -17,7 +17,7 @@ VERBOSE=$YES
 
 # NO USER CHANGES AFTER THIS
 
-export UV_PYTHON=3.9 #python version required by IPF
+UV_PYTHON=3.12 #python version preferred by IPF
 UV_DIR="${INSTALL_DIR}"/uv
 UV="${UV_DIR}"/uv
 VENV="$INSTALL_DIR"/.venv
@@ -78,6 +78,8 @@ mk_venv() {
     export UV_PYTHON_INSTALL_DIR="${UV_DIR}"/python
     export UV_NO_CACHE=1
     export UV_MANAGED_PYTHON=1
+    export UV_PYTHON
+    [[ -n "${IPF_UV_PYTHON}" ]] && UV_PYTHON="${IPF_UV_PYTHON}"
     "$UV" venv "$VENV"
     success "Python venv created at '$VENV'"
     "${V_PYTHON}" -m ensurepip
@@ -117,7 +119,7 @@ install_ipf_pre_release() {
     --pre \
     --no-deps \
     --index-url https://test.pypi.org/simple/ \
-    ipf \
+    ipf"${IPF_INSTALL_VERSION:+==$IPF_INSTALL_VERSION}" \
     && success "Installed dev version of ipf"
 }
 
@@ -126,7 +128,7 @@ install_ipf() {
   [[ $DEBUG -eq $YES ]] && set -x
   "${V_PYTHON}" -m pip install \
     --upgrade \
-    ipf \
+    ipf"${IPF_INSTALL_VERSION:+==$IPF_INSTALL_VERSION}" \
     && success "Ipf and dependencies installed"
 }
 

diff --git a/utils/ipf-setup-wrapper.sh b/utils/ipf-setup-wrapper.sh
@@ -2,30 +2,79 @@
 
 set -x
 
-IPF_BIN="${HOME}"/ipf/bin
-VENV="${HOME}"/ipf/.venv
+# =========================
+# customize these as needed
+# =========================
+
+# where to save the installer script
 IPF_SETUP="${HOME}"/ipf-setup.sh
 
-[[ -z "${IPF_GIT_BRANCH}" ]] && {
-  echo "Missing environment variable IPF_GIT_BRANCH"
-  exit 1
-}
+# set to an absolute path (default is $HOME/ipf)
+IPF_INSTALL_DIR="${HOME}"/ipf
+
+# dont actually do anything, just print out a list of commands
+# that can be run manually
+# MANUAL=yes
+
+# set to "yes" or leave unset (defaults to latest)
+# IPF_ALLOW_PRE_RELEASE=yes
+
+# specify an exact version of IPF
+# get version string from
+#    https://pypi.org/project/ipf/#history      (production releases)
+# OR https://test.pypi.org/project/ipf/#history (pre-release versions)
+# IPF_INSTALL_VERSION=1.9.9.dev1783568317
+
+# either a branch name or leave unset (defaults to master)
+# IPF_GIT_BRANCH=ATS-30367
+
+# specify a specific version of python if needed
+# IPF_UV_PYTHON=3.12
+
+# =====================================
+# shouldn't need to touch anything else
+# =====================================
+
+
+[[ -z "${IPF_GIT_BRANCH}" ]] && IPF_GIT_BRANCH=master
 URL_BASE='https://raw.githubusercontent.com/access-ci-org/ipf/refs/heads'
 IPF_SETUP_URL="${URL_BASE}"/"${IPF_GIT_BRANCH}"/setup.sh
+IPF_UV_URL="${URL_BASE}"/"${IPF_GIT_BRANCH}"/utils/uv_installer.sh
+IPF_BIN="${IPF_INSTALL_DIR}"/bin
+VENV="${IPF_INSTALL_DIR}"/.venv
 
-bash "${IPF_BIN}"/wfm stop
-
-rm -rf ~/ipf
+###
+# MAIN
+###
 
-rm -f "${IPF_SETUP}"
+# make exports (used by setup script)
+export IPF_UV_URL
+export IPF_INSTALL_DIR
+[[ -n "${IPF_ALLOW_PRE_RELEASE}" ]] && export IPF_ALLOW_PRE_RELEASE
+[[ -n "${IPF_UV_PYTHON}" ]] && export IPF_UV_PYTHON
+[[ -n "${IPF_INSTALL_VERSION}" ]] && export IPF_INSTALL_VERSION
 
-curl -o "${IPF_SETUP}" "${IPF_SETUP_URL}"
+# set ACTION if doing manual run
+if [[ -n "${MANUAL}" ]] ; then
+  ACTION=echo
+  set +x
+  echo 'COMMANDS THAT WOULD HAVE BEEN RUN'
+  echo '---------------------------------'
+fi
 
-export IPF_UV_URL="${URL_BASE}"/"${IPF_GIT_BRANCH}"/utils/uv_installer.sh
-export IPF_ALLOW_PRE_RELEASE=yes
+# stop anything running
+[[ -x "${IPF_BIN}"/wfm ]] && ${ACTION} bash "${IPF_BIN}"/wfm stop
 
-bash "${IPF_SETUP}"
+# cleanup any existing installs
+${ACTION} rm -rf "${IPF_INSTALL_DIR}"
+${ACTION} rm -rf "${HOME}"/.cache/pip
+${ACTION} rm -f "${IPF_SETUP}"
 
-"${VENV}"/bin/python --version
+# get new setup script
+${ACTION} curl -o "${IPF_SETUP}" "${IPF_SETUP_URL}"
+# and run it
+${ACTION} bash "${IPF_SETUP}"
 
-"${VENV}"/bin/python -m pip freeze
+# validate install
+${ACTION} "${VENV}"/bin/python --version
+${ACTION} "${VENV}"/bin/python -m pip freeze
diff --git a/utils/ipf_pkgs.sh b/utils/ipf_pkgs.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/bash
+
+IPF_PATH=~/ipf
+BIN="${IPF_PATH}"/bin
+WFM="${BIN}"/wfm
+
+
+get_json_data_file() {
+  "${WFM}" ls 2>&1 \
+  | awk '/OUTPUT: / && /extended_modules.json/ {print $2}' \
+  | head -1
+}
+
+
+
+get_clean_names() {
+  jq \
+    '.ApplicationHandle[].Value | split("/")[0] | gsub("^\\s+|\\s+$"; "")' \
+    "${JSON_PATH}" \
+  | sort -u \
+  | tr -d '"'
+}
+
+
+###
+# MAIN
+###
+
+JSON_PATH=$( get_json_data_file )
+
+get_clean_names