From 970c8c36cc01edf7fb3afcd6e18f1288aab42daf Mon Sep 17 00:00:00 2001 From: Matteo Chesi Date: Mon, 18 May 2026 12:34:18 +0200 Subject: [PATCH 01/10] Add lustre_pfl scripts for imagestore_create hook --- .github/workflows/release.yml | 12 ++ scripts/lustre_pfl/bin/lustre_pfl_mkdir | 148 ++++++++++++++++ scripts/lustre_pfl/bin/lustre_pfl_optimize | 158 ++++++++++++++++++ .../sbin/lustre_pfl_reference_setstripe | 137 +++++++++++++++ 4 files changed, 455 insertions(+) create mode 100755 scripts/lustre_pfl/bin/lustre_pfl_mkdir create mode 100755 scripts/lustre_pfl/bin/lustre_pfl_optimize create mode 100755 scripts/lustre_pfl/sbin/lustre_pfl_reference_setstripe diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7e58f94..f483854 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -63,6 +63,10 @@ jobs: cp target/release/mkhomedir "dist/hooks-${{ matrix.arch }}/mkhomedir-${{ github.ref_name }}" tar -C dist/hooks-${{ matrix.arch }} -czf "dist/hooks-${{ matrix.arch }}.tar.gz" . ls -lah target/release/ + mkdir -p dist/scripts + cp -a scripts/lustre_pfl/bin dist/scripts/ + cp -a scripts/lustre_pfl/sbin dist/scripts/ + tar -C dist/scripts -czf "dist/scripts.tar.gz" . ls -lah dist/ ' @@ -73,3 +77,11 @@ jobs: path: dist/hooks-${{ matrix.arch }}.tar.gz if-no-files-found: error archive: false + + - name: Upload scripts archive + uses: actions/upload-artifact@v7 + with: + name: scripts + path: dist/scripts.tar.gz + if-no-files-found: error + archive: false diff --git a/scripts/lustre_pfl/bin/lustre_pfl_mkdir b/scripts/lustre_pfl/bin/lustre_pfl_mkdir new file mode 100755 index 0000000..71773f0 --- /dev/null +++ b/scripts/lustre_pfl/bin/lustre_pfl_mkdir @@ -0,0 +1,148 @@ +#!/bin/bash +# +# Create and Check directory against Lustre progressive file layout reference. +# + +TARGET_DIR=$1 +BIN_DIR=$(readlink -f "$(dirname $0)") +ETC_DIR=$(readlink -f "${BIN_DIR}/../etc") +LFS_CMD="/usr/bin/lfs" +TIMEOUT="10" +TMP_DIR="/tmp" + +# Check dependencies +[ ! -x /usr/bin/timeout ] && echo "ERROR: missing /usr/bin/timeout" >&2 && exit 1 +[ ! -x /usr/bin/dirname ] && echo "ERROR: missing /usr/bin/dirname" >&2 && exit 1 +[ ! -x /usr/bin/findmnt ] && echo "ERROR: missing /usr/bin/findmnt" >&2 && exit 1 +[ ! -x ${LFS_CMD} ] && echo "ERROR: missing ${LFS_CMD}" >&2 && exit 1 +[ ! -x /usr/bin/yq ] && echo "ERROR: missing /usr/bin/yq" >&2 && exit 1 +[ ! -x /usr/bin/jq ] && echo "ERROR: missing /usr/bin/jq" >&2 && exit 1 +[ ! -x /usr/bin/diff ] && echo "ERROR: missing /usr/bin/diff" >&2 && exit 1 + +if [ -z "${TARGET_DIR}" ] +then + echo "ERROR: you must specify a target directory path" >&2 + exit 1 +fi + +if [ ! -e "${TARGET_DIR}" ] +then + NEED_DIR_CREATION="true" + DIR_TO_CHECK=$(/usr/bin/dirname ${TARGET_DIR}) + while [ ! -d "${DIR_TO_CHECK}" ] && [ "${DIR_TO_CHECK}" != "/" ] && [ "${DIR_TO_CHECK}" != "." ] + do + DIR_TO_CHECK=$(/usr/bin/dirname ${DIR_TO_CHECK}) + done +elif [ ! -d "${TARGET_DIR}" ] +then + echo "ERROR: \"${TARGET_DIR}\" is not a directory" >&2 + exit 1 +else + DIR_TO_CHECK=${TARGET_DIR} +fi + +# check filesystem type +FSTYPE=$(/usr/bin/timeout $TIMEOUT /usr/bin/findmnt -n -o fstype --target "${DIR_TO_CHECK}") +if [ "${FSTYPE}" != "lustre" ] +then + cat >&2 << EOF +/---------/ +WARNING: imagestore directory \"${TARGET_DIR}\" isn't on a LUSTRE filesystem. That is suboptimal. + +Please move parallax imagestore to a LUSTRE filesystem. + +Or if you are already aware, you can disable this warning adding the following annotation in the EDF: + +[annotations] +com.sarus.hooks.parallax_imagestore_create="false" +/---------/ +EOF + exit 1 +else + FSNAME=$(/usr/bin/timeout $TIMEOUT ${LFS_CMD} getname -n "${DIR_TO_CHECK}") + if [ $? -ne 0 ] + then + echo "ERROR: Cannot retrieve filesystem name for \"${DIR_TO_CHECK}\", rc=$?" >&2 + exit 1 + fi + + PFL_LAYOUT_REFERENCE="${ETC_DIR}/${FSNAME}_pfl_reference.json" + PFL_SETSTRIPE_OPTS_ENV="${ETC_DIR}/${FSNAME}_pfl_setstripe_opts.env" + + if [ ! -f "${PFL_LAYOUT_REFERENCE}" ] + then + echo "ERROR: Missing Progressive File layout reference file \"${PFL_LAYOUT_REFERENCE}\"" >&2 + exit 1 + fi + + if [ ! -f "${PFL_SETSTRIPE_OPTS_ENV}" ] + then + echo "ERROR: Missing Progressive File layout environment file \"${PFL_SETSTRIPE_OPTS_ENV}\"" >&2 + exit 1 + fi + + PFL_LAYOUT_TEMPLATE="${TMP_DIR}/pfl_layout.json.XXX" + . ${PFL_SETSTRIPE_OPTS_ENV} + + if [ -z "${LFS_SETSTRIPE_OPTS}" ] + then + echo "ERROR: Missing LFS_SETSTRIPE_OPTS environment variable " >&2 + exit 1 + fi +fi + +# create directory +if [ "$NEED_DIR_CREATION" == "true" ] +then + mkdir -p "${TARGET_DIR}" + RC=$? + if [ "${FSTYPE}" != "lustre" ] + then + exit ${RC} + else + /usr/bin/timeout ${TIMEOUT} ${LFS_CMD} setstripe ${LFS_SETSTRIPE_OPTS} ${TARGET_DIR} + RC=$? + if [ $RC -ne 0 ] + then + echo "ERROR: \"timeout $TIMEOUT lfs setstripe\" returned: $RC" >&2 + exit 1 + fi + fi +fi + +# Collect pfl.layout +PFL_LAYOUT_OUTPUT=$(mktemp ${PFL_LAYOUT_TEMPLATE}) +/usr/bin/timeout ${TIMEOUT} /usr/bin/lfs getstripe -y -d ${TARGET_DIR} | \ + /usr/bin/yq r - -j -P | \ + /usr/bin/jq 'with_entries(if (.key | test("component[0-9]$")) then ({key: .key, value: del(.value.sub_layout.lmm_objects)}) else ({key: .key, value: .value}) end)' \ + > ${PFL_LAYOUT_OUTPUT} +RC=$? +if [ $RC -ne 0 ] +then + echo "ERROR: \"timeout $TIMEOUT lfs getstripe\" returned: $RC" >&2 + exit 1 +fi + +/usr/bin/diff -q ${PFL_LAYOUT_REFERENCE} ${PFL_LAYOUT_OUTPUT} &>/dev/null +RC=$? +if [ $RC -ne 0 ] +then + cat >&2 << EOF +/---------/ +WARNING!!! +imagestore directory \"${TARGET_DIR}\" doesn't have the expected filesystem optimization + +Please run: +lustre_pfl_optimize ${TARGET_DIR} + +Or if you are already aware, you can disable this warning adding the following annotation in the EDF: + +[annotations] +com.sarus.hooks.parallax_imagestore_create="false" +/---------/ +EOF + +fi + +rm -f ${PFL_LAYOUT_OUTPUT} +exit $RC diff --git a/scripts/lustre_pfl/bin/lustre_pfl_optimize b/scripts/lustre_pfl/bin/lustre_pfl_optimize new file mode 100755 index 0000000..95bd4b5 --- /dev/null +++ b/scripts/lustre_pfl/bin/lustre_pfl_optimize @@ -0,0 +1,158 @@ +#!/bin/bash +# +# Optimize a directory and its content agains progressive file layout reference file schema. +# + +TARGET_DIR=$1 +BIN_DIR=$(readlink -f "$(dirname $0)") +ETC_DIR=$(readlink -f "${BIN_DIR}/../etc") +LFS_CMD="/usr/bin/lfs" +TIMEOUT="10" +TMP_DIR="/tmp" + +# Check dependencies +[ ! -x /usr/bin/timeout ] && echo "ERROR: missing /usr/bin/timeout" >&2 && exit 1 +[ ! -x /usr/bin/dirname ] && echo "ERROR: missing /usr/bin/dirname" >&2 && exit 1 +[ ! -x /usr/bin/findmnt ] && echo "ERROR: missing /usr/bin/findmnt" >&2 && exit 1 +[ ! -x ${LFS_CMD} ] && echo "ERROR: missing ${LFS_CMD}" >&2 && exit 1 +[ ! -x /usr/bin/yq ] && echo "ERROR: missing /usr/bin/yq" >&2 && exit 1 +[ ! -x /usr/bin/jq ] && echo "ERROR: missing /usr/bin/jq" >&2 && exit 1 +[ ! -x /usr/bin/diff ] && echo "ERROR: missing /usr/bin/diff" >&2 && exit 1 +[ ! -x /usr/bin/rsync ] && echo "ERROR: missing /usr/bin/rsync" >&2 && exit 1 + +MIGRATING_DIR="${TARGET_DIR}.optimizing" + +function rename_directory() { + mv ${MIGRATING_DIR} ${TARGET_DIR} + RC=$? + if [ $RC -ne 0 ] + then + echo "ERROR: mv ${MIGRATING_DIR} ${TARGET_DIR} returned: $RC" >&2 + echo "Optimization crashed. Target directory is in an undefined state. Please run \"${0}\" again." >&2 + return 1 + fi +} + +if [ -z "${TARGET_DIR}" ] +then + echo "ERROR: you must specify lustre directory path" >&2 + exit 1 +fi + +if [ ! -d "${TARGET_DIR}" ] +then + if [ -d "${MIGRATING_DIR}" ] + then + rename_directory || exit $? + else + echo "ERROR: target directory \"${TARGET_DIR}\" doesn't exist" >&2 + exit 1 + fi +fi + +# check filesystem type +FSTYPE=$(/usr/bin/timeout $TIMEOUT findmnt -n -o fstype --target "${TARGET_DIR}") +if [ "${FSTYPE}" != "lustre" ] +then + echo "ERROR: target directory \"${TARGET_DIR}\" isn't on a LUSTRE filesystem. Cannot optimize." >&2 + exit 1 +fi + +FSNAME=$(/usr/bin/timeout $TIMEOUT ${LFS_CMD} getname -n "${TARGET_DIR}") +if [ $? -ne 0 ] +then + echo "ERROR: Cannot retrieve filesystem name for \"${TARGET_DIR}\", rc=$?" >&2 + exit 1 +fi + +PFL_LAYOUT_REFERENCE="${ETC_DIR}/${FSNAME}_pfl_reference.json" +PFL_SETSTRIPE_OPTS_ENV="${ETC_DIR}/${FSNAME}_pfl_setstripe_opts.env" + +if [ ! -f "${PFL_LAYOUT_REFERENCE}" ] +then + echo "ERROR: Missing Progressive File layout reference file \"${PFL_LAYOUT_REFERENCE}\"" >&2 + exit 1 +fi + +if [ ! -f "${PFL_SETSTRIPE_OPTS_ENV}" ] +then + echo "ERROR: Missing Progressive File layout environment file \"${PFL_SETSTRIPE_OPTS_ENV}\"" >&2 + exit 1 +fi + +. ${PFL_SETSTRIPE_OPTS_ENV} + +if [ -z "${LFS_SETSTRIPE_OPTS}" ] +then + echo "ERROR: Missing LFS_SETSTRIPE_OPTS environment variable " >&2 + exit 1 +fi + +# Collect pfl.layout +PFL_LAYOUT_TEMPLATE="${TMP_DIR}/pfl_layout.json.XXX" +PFL_LAYOUT_OUTPUT=$(mktemp ${PFL_LAYOUT_TEMPLATE}) +/usr/bin/timeout ${TIMEOUT} ${LFS_CMD} getstripe -y -d ${TARGET_DIR} | \ + /usr/bin/yq r - -j -P | \ + /usr/bin/jq 'with_entries(if (.key | test("component[0-9]$")) then ({key: .key, value: del(.value.sub_layout.lmm_objects)}) else ({key: .key, value: .value}) end)' \ + > ${PFL_LAYOUT_OUTPUT} +RC=$? +if [ $RC -ne 0 ] +then + echo "ERROR: \"timeout $TIMEOUT lfs getstripe\" returned: $RC" >&2 + exit 1 +fi + +/usr/bin/diff -q ${PFL_LAYOUT_REFERENCE} ${PFL_LAYOUT_OUTPUT} &>/dev/null +RC=$? +if [ $RC -eq 0 ] +then + echo "ERROR: target directory \"${TARGET_DIR}\" is already optmized. Nothing to do." >&2 + exit 1 +fi +rm -f ${PFL_LAYOUT_OUTPUT} + +if [ -d "${MIGRATING_DIR}" ] +then + echo "RECOVERING Temporary Directory at ${MIGRATING_DIR}" +else + mkdir -p "${MIGRATING_DIR}" + RC=$? + if [ $RC -ne 0 ] + then + echo "ERROR: mkdir -p \"${MIGRATING_DIR}\" returned: $RC" >&2 + exit 1 + fi + echo "CREATED New Temporary Directory at ${MIGRATING_DIR}" +fi + +/usr/bin/timeout ${TIMEOUT} ${LFS_CMD} setstripe ${LFS_SETSTRIPE_OPTS} ${MIGRATING_DIR} +RC=$? +if [ $RC -ne 0 ] +then + rmdir ${MIGRATING_DIR} + echo "ERROR: \"timeout $TIMEOUT lfs setstripe\" returned: $RC" >&2 + exit 1 +fi + +echo "Moving data from ${TARGET_DIR} to ${MIGRATING_DIR}" +/usr/bin/rsync -av --remove-source-files ${TARGET_DIR}/ ${MIGRATING_DIR}/ +RC=$? +if [ $RC -ne 0 ] +then + echo "ERROR: rsync -av --delete ${TARGET_DIR}/ ${MIGRATING_DIR}/ returned: $RC" >&2 + echo "Optimization crashed. Target directory is in an undefined state. Please run \"${0}\" again." >&2 + exit 1 +fi + +find ${TARGET_DIR}/ -type d -empty -delete +RC=$? +if [ $RC -ne 0 ] +then + echo "ERROR: find ${TMP_DIR} -type d -empty -delete, returned: $RC" >&2 + echo "Optimization crashed. Target directory is currently in an undefined state. Please run \"${0}\" again." >&2 + exit 1 +fi + +rename_directory || exit $? + +exit 0 diff --git a/scripts/lustre_pfl/sbin/lustre_pfl_reference_setstripe b/scripts/lustre_pfl/sbin/lustre_pfl_reference_setstripe new file mode 100755 index 0000000..e6fbbcf --- /dev/null +++ b/scripts/lustre_pfl/sbin/lustre_pfl_reference_setstripe @@ -0,0 +1,137 @@ +#!/bin/bash +# +# Create a reference template for Progressive File Layout on a Lustre filesystem +# + +ARGS="$@" +SCRIPTNAME=$(basename $0) +LFS_CMD="/usr/bin/lfs" +TIMEOUT="10" +SBIN_DIR=$(readlink -f "$(dirname $0)") +ETC_DIR=$(readlink -f "${SBIN_DIR}/../etc") + +function help() { + cat >&2 << EOF + + This script creates a reference template for lustre progressive file layout + on a specific lustre filesystem and saves it at: + ${ETC_DIR} + + Usage: ${SCRIPTNAME} + +EOF +} + + +# Check dependencies +[ ! -x /usr/bin/timeout ] && echo "ERROR: missing /usr/bin/timeout" >&2 && exit 1 +[ ! -x /usr/bin/dirname ] && echo "ERROR: missing /usr/bin/dirname" >&2 && exit 1 +[ ! -x /usr/bin/findmnt ] && echo "ERROR: missing /usr/bin/findmnt" >&2 && exit 1 +[ ! -x ${LFS_CMD} ] && echo "ERROR: missing ${LFS_CMD}" >&2 && exit 1 +[ ! -x /usr/bin/yq ] && echo "ERROR: missing /usr/bin/yq" >&2 && exit 1 +[ ! -x /usr/bin/jq ] && echo "ERROR: missing /usr/bin/jq" >&2 && exit 1 +[ ! -x /usr/bin/diff ] && echo "ERROR: missing /usr/bin/diff" >&2 && exit 1 +[ ! -x /usr/bin/rmdir ] && echo "ERROR: missing /usr/bin/rmdir" >&2 && exit 1 +[ ! -x /usr/bin/mktemp ] && echo "ERROR: missing /usr/bin/mktemp" >&2 && exit 1 + +if [ ! -d "${ETC_DIR}" ] +then + mkdir -p "${ETC_DIR}" + [ ! -d "${ETC_DIR}" ] && echo "ERROR: cannot create \"${ETC_DIR}\"" >&2 && exit 1 +fi + +PARENT_DIR="${ARGS##* }" +LFS_SETSTRIPE_OPTS="${ARGS% *}" + +if [ -z "${PARENT_DIR}" ] || [ -z "${LFS_SETSTRIPE_OPTS}" ] +then + echo "ERROR: you must specify \"lfs setstripe\" options and a directory path on a target Lustre filesystem" >&2 + help + exit 1 +fi + +if [ ! -d "${PARENT_DIR}" ] +then + echo "ERROR: \"${PARENT_DIR}\" is not a directory" >&2 + exit 1 +fi + +FSTYPE=$(/usr/bin/timeout $TIMEOUT /usr/bin/findmnt -n -o fstype --target "${PARENT_DIR}") +if [ "${FSTYPE}" != "lustre" ] +then + echo "ERROR: \"${PARENT_DIR}\" is not a on a lustre filesystem" >&2 + exit 1 +fi + +FSNAME=$(/usr/bin/timeout $TIMEOUT ${LFS_CMD} getname -n "${PARENT_DIR}") +if [ $? -ne 0 ] +then + echo "ERROR: Cannot retrieve filesystem name for \"${PARENT_DIR}\", rc=$?" >&2 + exit 1 +fi + +# Create directory +TARGET_DIR=$(/usr/bin/mktemp -d -p "${PARENT_DIR}" pfl_template_XXX) +RC=$? +if [ ! -d "${TARGET_DIR}" ] +then + echo "ERROR: Cannot create directory at \"${TARGET_DIR}\", rc=$RC" >&2 + exit 1 +fi +PFL_REFERENCE="${ETC_DIR}/${FSNAME}_pfl_reference.json" +PFL_SETSTRIPE_OPTS_ENV="${ETC_DIR}/${FSNAME}_pfl_setstripe_opts.env" + +# Set Striping +/usr/bin/timeout ${TIMEOUT} ${LFS_CMD} setstripe ${LFS_SETSTRIPE_OPTS} ${TARGET_DIR} +RC=$? +if [ $RC -ne 0 ] +then + echo "ERROR: \"timeout $TIMEOUT lfs setstripe\" returned: $RC" >&2 + exit 1 +fi + +# Collect pfl.layout +/usr/bin/timeout ${TIMEOUT} ${LFS_CMD} getstripe -y -d ${TARGET_DIR} | \ + /usr/bin/yq r - -j -P | \ + /usr/bin/jq 'with_entries(if (.key | test("component[0-9]$")) then ({key: .key, value: del(.value.sub_layout.lmm_objects)}) else ({key: .key, value: .value}) end)' \ + > ${PFL_REFERENCE} +RC=$? +if [ $RC -ne 0 ] +then + echo "ERROR: \"timeout $TIMEOUT lfs getstripe\" returned: $RC" >&2 + exit 1 +fi +if [ ! -f "${PFL_REFERENCE}" ] +then + echo "ERROR: \"${PFL_REFERENCE}\" was not created." >&2 + exit 1 +else + echo "Created \"${PFL_REFERENCE}\"" +fi + +# Save lfs setstripe opts +echo "export LFS_SETSTRIPE_OPTS=\"${LFS_SETSTRIPE_OPTS}\"" > ${PFL_SETSTRIPE_OPTS_ENV} +RC=$? +if [ $RC -ne 0 ] +then + echo "ERROR: Cannot create \"${PFL_SETSTRIPE_OPTS_ENV}\", rc: $RC" >&2 + exit 1 +fi +if [ ! -f "${PFL_SETSTRIPE_OPTS_ENV}" ] +then + echo "ERROR: \"${PFL_SETSTRIPE_OPTS_ENV}\" was not created." >&2 + exit 1 +else + echo "Created \"${PFL_SETSTRIPE_OPTS_ENV}\"" +fi + +# Remove TARGET_DIR +/usr/bin/rmdir ${TARGET_DIR} +RC=$? +if [ $RC -ne 0 ] +then + echo "ERROR: Cannot remove \"${TARGET_DIR}\", rc: $RC" >&2 + exit 1 +fi + +exit 0 From 854f5d886a5228ea8f9e7d9c0b3a8203d3f2b7fa Mon Sep 17 00:00:00 2001 From: matteo-chesi Date: Mon, 18 May 2026 14:17:59 +0200 Subject: [PATCH 02/10] Update release.yml --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f483854..4b79a39 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: # for manual trigger ref: ${{ inputs.ref != '' && inputs.ref || github.ref_name }} From e9fb284e9859aeeafaee7349b5d3ee4c90c8c79e Mon Sep 17 00:00:00 2001 From: matteo-chesi Date: Mon, 18 May 2026 14:25:15 +0200 Subject: [PATCH 03/10] Update login-action on release.yml --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4b79a39..233d79c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,7 +33,7 @@ jobs: ref: ${{ inputs.ref != '' && inputs.ref || github.ref_name }} - name: Login to GHCR - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} From faadc8a34e81bede3534d56154c9ad30ac4a7191 Mon Sep 17 00:00:00 2001 From: Matteo Chesi Date: Mon, 18 May 2026 14:37:55 +0200 Subject: [PATCH 04/10] upload scripts only on amd64 --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 233d79c..1a19e04 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -79,6 +79,7 @@ jobs: archive: false - name: Upload scripts archive + if: ${{ matrix.arch == 'amd64' }} uses: actions/upload-artifact@v7 with: name: scripts From 197b0c0570b2a984497e406958296bd656c49e1d Mon Sep 17 00:00:00 2001 From: Matteo Chesi Date: Mon, 18 May 2026 16:15:05 +0200 Subject: [PATCH 05/10] Remove branch from names --- .github/workflows/release.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7e58f94..9c78085 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -56,11 +56,11 @@ jobs: bash -lc ' cargo build --release mkdir -p dist/hooks-${{ matrix.arch }} - cp target/release/ldcache_hook "dist/hooks-${{ matrix.arch }}/ldcache_hook-${{ github.ref_name }}" - cp target/release/pce_hook "dist/hooks-${{ matrix.arch }}/pce_hook-${{ github.ref_name }}" - cp target/release/mps_hook "dist/hooks-${{ matrix.arch }}/mps_hook-${{ github.ref_name }}" - cp target/release/sethomevar "dist/hooks-${{ matrix.arch }}/sethomevar-${{ github.ref_name }}" - cp target/release/mkhomedir "dist/hooks-${{ matrix.arch }}/mkhomedir-${{ github.ref_name }}" + cp target/release/ldcache_hook "dist/hooks-${{ matrix.arch }}/ldcache_hook + cp target/release/pce_hook "dist/hooks-${{ matrix.arch }}/pce_hook + cp target/release/mps_hook "dist/hooks-${{ matrix.arch }}/mps_hook + cp target/release/sethomevar "dist/hooks-${{ matrix.arch }}/sethomevar + cp target/release/mkhomedir "dist/hooks-${{ matrix.arch }}/mkhomedir tar -C dist/hooks-${{ matrix.arch }} -czf "dist/hooks-${{ matrix.arch }}.tar.gz" . ls -lah target/release/ ls -lah dist/ From d247191f264cb18b3cb90902db74cae2801ec710 Mon Sep 17 00:00:00 2001 From: matteo-chesi Date: Tue, 19 May 2026 11:34:51 +0200 Subject: [PATCH 06/10] Fix #9 - alternative way to collect bundle from OciState (#13) --- crates/mkhomedir/Cargo.toml | 1 + crates/mkhomedir/src/main.rs | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/crates/mkhomedir/Cargo.toml b/crates/mkhomedir/Cargo.toml index 5fed5b4..d497365 100644 --- a/crates/mkhomedir/Cargo.toml +++ b/crates/mkhomedir/Cargo.toml @@ -10,3 +10,4 @@ libc = "0.2.185" procfs = "0.18.0" serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.149" +sysinfo = "0.38.4" diff --git a/crates/mkhomedir/src/main.rs b/crates/mkhomedir/src/main.rs index ac701a0..4ba1f35 100644 --- a/crates/mkhomedir/src/main.rs +++ b/crates/mkhomedir/src/main.rs @@ -13,6 +13,7 @@ use std::{ #[derive(Debug, Deserialize)] struct OciState { bundle: String, + id: String, status: String, root: String, } @@ -202,6 +203,28 @@ fn get_home_from_etc_passwd(root: &Path, uid: uid_t) -> Result )) } +fn get_graphroot_from_root(root: &PathBuf) -> Result { + + let mut graphroot = root.clone(); + let mut uplevels = 3; + while uplevels > 0 { + if graphroot.pop() { + uplevels -= 1; + } else { + return Err(format!("failed get graphroot from root: {:#?}", root)); + } + } + Ok(graphroot) +} + +fn get_bundle_from_graphroot_and_id(graphroot: &PathBuf, id: &String) -> String { + let rel_path_str = format!("overlay-containers/{id}/userdata"); + let rel_path = Path::new(&rel_path_str); + let bundle = graphroot.join(rel_path); + let bundle_string = bundle.to_string_lossy().to_string(); + bundle_string +} + fn run() -> Result { let oci_state: OciState = read_stdin_json().map_err(|e| format!("failed to parse OCI State: {e}"))?; @@ -210,7 +233,16 @@ fn run() -> Result { return Ok(0); } - let config = get_config_from_bundle(Path::new(&oci_state.bundle))?; + let bundle; + if oci_state.bundle != "/" { + bundle = oci_state.bundle; + } else { + let root_path = PathBuf::from(&oci_state.root); + let graphroot = get_graphroot_from_root(&root_path)?; + bundle = get_bundle_from_graphroot_and_id(&graphroot, &oci_state.id); + } + + let config = get_config_from_bundle(Path::new(&bundle))?; let home = match config.home { Some(h) => { From 587f6cb62fb6442a97f3b75ebe9875c98973ac01 Mon Sep 17 00:00:00 2001 From: Matteo Chesi Date: Tue, 19 May 2026 12:03:49 +0200 Subject: [PATCH 07/10] mkhomedir - remove unnecessary dependency --- crates/mkhomedir/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/mkhomedir/Cargo.toml b/crates/mkhomedir/Cargo.toml index d497365..5fed5b4 100644 --- a/crates/mkhomedir/Cargo.toml +++ b/crates/mkhomedir/Cargo.toml @@ -10,4 +10,3 @@ libc = "0.2.185" procfs = "0.18.0" serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.149" -sysinfo = "0.38.4" From 7b25a4ced682f34dab1c5904743c3315d1c98692 Mon Sep 17 00:00:00 2001 From: Matteo Chesi Date: Tue, 19 May 2026 14:53:59 +0200 Subject: [PATCH 08/10] Fix quotes --- .github/workflows/release.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9c78085..8d88afc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -56,11 +56,11 @@ jobs: bash -lc ' cargo build --release mkdir -p dist/hooks-${{ matrix.arch }} - cp target/release/ldcache_hook "dist/hooks-${{ matrix.arch }}/ldcache_hook - cp target/release/pce_hook "dist/hooks-${{ matrix.arch }}/pce_hook - cp target/release/mps_hook "dist/hooks-${{ matrix.arch }}/mps_hook - cp target/release/sethomevar "dist/hooks-${{ matrix.arch }}/sethomevar - cp target/release/mkhomedir "dist/hooks-${{ matrix.arch }}/mkhomedir + cp target/release/ldcache_hook "dist/hooks-${{ matrix.arch }}/ldcache_hook" + cp target/release/pce_hook "dist/hooks-${{ matrix.arch }}/pce_hook" + cp target/release/mps_hook "dist/hooks-${{ matrix.arch }}/mps_hook" + cp target/release/sethomevar "dist/hooks-${{ matrix.arch }}/sethomevar" + cp target/release/mkhomedir "dist/hooks-${{ matrix.arch }}/mkhomedir" tar -C dist/hooks-${{ matrix.arch }} -czf "dist/hooks-${{ matrix.arch }}.tar.gz" . ls -lah target/release/ ls -lah dist/ From b7b7640b35c37f0e82e193feca55e337b1265e76 Mon Sep 17 00:00:00 2001 From: Matteo Chesi Date: Tue, 19 May 2026 16:50:16 +0200 Subject: [PATCH 09/10] lustre_pfl_mkdir - removed unrequired escapes from warnings --- scripts/lustre_pfl/bin/lustre_pfl_mkdir | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/lustre_pfl/bin/lustre_pfl_mkdir b/scripts/lustre_pfl/bin/lustre_pfl_mkdir index 71773f0..4f7d4b5 100755 --- a/scripts/lustre_pfl/bin/lustre_pfl_mkdir +++ b/scripts/lustre_pfl/bin/lustre_pfl_mkdir @@ -47,7 +47,7 @@ if [ "${FSTYPE}" != "lustre" ] then cat >&2 << EOF /---------/ -WARNING: imagestore directory \"${TARGET_DIR}\" isn't on a LUSTRE filesystem. That is suboptimal. +WARNING: imagestore directory "${TARGET_DIR}" isn't on a LUSTRE filesystem. That is suboptimal. Please move parallax imagestore to a LUSTRE filesystem. @@ -130,7 +130,7 @@ then cat >&2 << EOF /---------/ WARNING!!! -imagestore directory \"${TARGET_DIR}\" doesn't have the expected filesystem optimization +imagestore directory "${TARGET_DIR}" doesn't have the expected filesystem optimization Please run: lustre_pfl_optimize ${TARGET_DIR} From 08508e4ffdf0a94ab76c5dc6da1fc381f96cd29d Mon Sep 17 00:00:00 2001 From: Matteo Chesi Date: Thu, 21 May 2026 16:07:41 +0200 Subject: [PATCH 10/10] Moved lustre_pfl_mkdir output from stderr to stdout --- scripts/lustre_pfl/bin/lustre_pfl_mkdir | 34 ++++++++++++------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/scripts/lustre_pfl/bin/lustre_pfl_mkdir b/scripts/lustre_pfl/bin/lustre_pfl_mkdir index 4f7d4b5..0f19b80 100755 --- a/scripts/lustre_pfl/bin/lustre_pfl_mkdir +++ b/scripts/lustre_pfl/bin/lustre_pfl_mkdir @@ -11,17 +11,17 @@ TIMEOUT="10" TMP_DIR="/tmp" # Check dependencies -[ ! -x /usr/bin/timeout ] && echo "ERROR: missing /usr/bin/timeout" >&2 && exit 1 -[ ! -x /usr/bin/dirname ] && echo "ERROR: missing /usr/bin/dirname" >&2 && exit 1 -[ ! -x /usr/bin/findmnt ] && echo "ERROR: missing /usr/bin/findmnt" >&2 && exit 1 -[ ! -x ${LFS_CMD} ] && echo "ERROR: missing ${LFS_CMD}" >&2 && exit 1 -[ ! -x /usr/bin/yq ] && echo "ERROR: missing /usr/bin/yq" >&2 && exit 1 -[ ! -x /usr/bin/jq ] && echo "ERROR: missing /usr/bin/jq" >&2 && exit 1 -[ ! -x /usr/bin/diff ] && echo "ERROR: missing /usr/bin/diff" >&2 && exit 1 +[ ! -x /usr/bin/timeout ] && echo "ERROR: missing /usr/bin/timeout" >&1 && exit 1 +[ ! -x /usr/bin/dirname ] && echo "ERROR: missing /usr/bin/dirname" >&1 && exit 1 +[ ! -x /usr/bin/findmnt ] && echo "ERROR: missing /usr/bin/findmnt" >&1 && exit 1 +[ ! -x ${LFS_CMD} ] && echo "ERROR: missing ${LFS_CMD}" >&1 && exit 1 +[ ! -x /usr/bin/yq ] && echo "ERROR: missing /usr/bin/yq" >&1 && exit 1 +[ ! -x /usr/bin/jq ] && echo "ERROR: missing /usr/bin/jq" >&1 && exit 1 +[ ! -x /usr/bin/diff ] && echo "ERROR: missing /usr/bin/diff" >&1 && exit 1 if [ -z "${TARGET_DIR}" ] then - echo "ERROR: you must specify a target directory path" >&2 + echo "ERROR: you must specify a target directory path" >&1 exit 1 fi @@ -35,7 +35,7 @@ then done elif [ ! -d "${TARGET_DIR}" ] then - echo "ERROR: \"${TARGET_DIR}\" is not a directory" >&2 + echo "ERROR: \"${TARGET_DIR}\" is not a directory" >&1 exit 1 else DIR_TO_CHECK=${TARGET_DIR} @@ -45,7 +45,7 @@ fi FSTYPE=$(/usr/bin/timeout $TIMEOUT /usr/bin/findmnt -n -o fstype --target "${DIR_TO_CHECK}") if [ "${FSTYPE}" != "lustre" ] then - cat >&2 << EOF + cat >&1 << EOF /---------/ WARNING: imagestore directory "${TARGET_DIR}" isn't on a LUSTRE filesystem. That is suboptimal. @@ -62,7 +62,7 @@ else FSNAME=$(/usr/bin/timeout $TIMEOUT ${LFS_CMD} getname -n "${DIR_TO_CHECK}") if [ $? -ne 0 ] then - echo "ERROR: Cannot retrieve filesystem name for \"${DIR_TO_CHECK}\", rc=$?" >&2 + echo "ERROR: Cannot retrieve filesystem name for \"${DIR_TO_CHECK}\", rc=$?" >&1 exit 1 fi @@ -71,13 +71,13 @@ else if [ ! -f "${PFL_LAYOUT_REFERENCE}" ] then - echo "ERROR: Missing Progressive File layout reference file \"${PFL_LAYOUT_REFERENCE}\"" >&2 + echo "ERROR: Missing Progressive File layout reference file \"${PFL_LAYOUT_REFERENCE}\"" >&1 exit 1 fi if [ ! -f "${PFL_SETSTRIPE_OPTS_ENV}" ] then - echo "ERROR: Missing Progressive File layout environment file \"${PFL_SETSTRIPE_OPTS_ENV}\"" >&2 + echo "ERROR: Missing Progressive File layout environment file \"${PFL_SETSTRIPE_OPTS_ENV}\"" >&1 exit 1 fi @@ -86,7 +86,7 @@ else if [ -z "${LFS_SETSTRIPE_OPTS}" ] then - echo "ERROR: Missing LFS_SETSTRIPE_OPTS environment variable " >&2 + echo "ERROR: Missing LFS_SETSTRIPE_OPTS environment variable " >&1 exit 1 fi fi @@ -104,7 +104,7 @@ then RC=$? if [ $RC -ne 0 ] then - echo "ERROR: \"timeout $TIMEOUT lfs setstripe\" returned: $RC" >&2 + echo "ERROR: \"timeout $TIMEOUT lfs setstripe\" returned: $RC" >&1 exit 1 fi fi @@ -119,7 +119,7 @@ PFL_LAYOUT_OUTPUT=$(mktemp ${PFL_LAYOUT_TEMPLATE}) RC=$? if [ $RC -ne 0 ] then - echo "ERROR: \"timeout $TIMEOUT lfs getstripe\" returned: $RC" >&2 + echo "ERROR: \"timeout $TIMEOUT lfs getstripe\" returned: $RC" >&1 exit 1 fi @@ -127,7 +127,7 @@ fi RC=$? if [ $RC -ne 0 ] then - cat >&2 << EOF + cat >&1 << EOF /---------/ WARNING!!! imagestore directory "${TARGET_DIR}" doesn't have the expected filesystem optimization