diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7e58f94..c7a419d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,13 +27,13 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: # for manual trigger ref: ${{ inputs.ref != '' && inputs.ref || github.ref_name }} - name: Login to GHCR - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} @@ -56,13 +56,17 @@ jobs: bash -lc ' cargo build --release mkdir -p dist/hooks-${{ matrix.arch }} - cp target/release/ldcache_hook "dist/hooks-${{ matrix.arch }}/ldcache_hook-${{ github.ref_name }}" - cp target/release/pce_hook "dist/hooks-${{ matrix.arch }}/pce_hook-${{ github.ref_name }}" - cp target/release/mps_hook "dist/hooks-${{ matrix.arch }}/mps_hook-${{ github.ref_name }}" - cp target/release/sethomevar "dist/hooks-${{ matrix.arch }}/sethomevar-${{ github.ref_name }}" - cp target/release/mkhomedir "dist/hooks-${{ matrix.arch }}/mkhomedir-${{ github.ref_name }}" + cp target/release/ldcache_hook "dist/hooks-${{ matrix.arch }}/ldcache_hook" + cp target/release/pce_hook "dist/hooks-${{ matrix.arch }}/pce_hook" + cp target/release/mps_hook "dist/hooks-${{ matrix.arch }}/mps_hook" + cp target/release/sethomevar "dist/hooks-${{ matrix.arch }}/sethomevar" + cp target/release/mkhomedir "dist/hooks-${{ matrix.arch }}/mkhomedir" tar -C dist/hooks-${{ matrix.arch }} -czf "dist/hooks-${{ matrix.arch }}.tar.gz" . ls -lah target/release/ + mkdir -p dist/scripts + cp -a scripts/lustre_pfl/bin dist/scripts/ + cp -a scripts/lustre_pfl/sbin dist/scripts/ + tar -C dist/scripts -czf "dist/scripts.tar.gz" . ls -lah dist/ ' @@ -73,3 +77,12 @@ jobs: path: dist/hooks-${{ matrix.arch }}.tar.gz if-no-files-found: error archive: false + + - name: Upload scripts archive + if: ${{ matrix.arch == 'amd64' }} + uses: actions/upload-artifact@v7 + with: + name: scripts + path: dist/scripts.tar.gz + if-no-files-found: error + archive: false diff --git a/scripts/lustre_pfl/bin/lustre_pfl_mkdir b/scripts/lustre_pfl/bin/lustre_pfl_mkdir new file mode 100755 index 0000000..0f19b80 --- /dev/null +++ b/scripts/lustre_pfl/bin/lustre_pfl_mkdir @@ -0,0 +1,148 @@ +#!/bin/bash +# +# Create and Check directory against Lustre progressive file layout reference. +# + +TARGET_DIR=$1 +BIN_DIR=$(readlink -f "$(dirname $0)") +ETC_DIR=$(readlink -f "${BIN_DIR}/../etc") +LFS_CMD="/usr/bin/lfs" +TIMEOUT="10" +TMP_DIR="/tmp" + +# Check dependencies +[ ! -x /usr/bin/timeout ] && echo "ERROR: missing /usr/bin/timeout" >&1 && exit 1 +[ ! -x /usr/bin/dirname ] && echo "ERROR: missing /usr/bin/dirname" >&1 && exit 1 +[ ! -x /usr/bin/findmnt ] && echo "ERROR: missing /usr/bin/findmnt" >&1 && exit 1 +[ ! -x ${LFS_CMD} ] && echo "ERROR: missing ${LFS_CMD}" >&1 && exit 1 +[ ! -x /usr/bin/yq ] && echo "ERROR: missing /usr/bin/yq" >&1 && exit 1 +[ ! -x /usr/bin/jq ] && echo "ERROR: missing /usr/bin/jq" >&1 && exit 1 +[ ! -x /usr/bin/diff ] && echo "ERROR: missing /usr/bin/diff" >&1 && exit 1 + +if [ -z "${TARGET_DIR}" ] +then + echo "ERROR: you must specify a target directory path" >&1 + exit 1 +fi + +if [ ! -e "${TARGET_DIR}" ] +then + NEED_DIR_CREATION="true" + DIR_TO_CHECK=$(/usr/bin/dirname ${TARGET_DIR}) + while [ ! -d "${DIR_TO_CHECK}" ] && [ "${DIR_TO_CHECK}" != "/" ] && [ "${DIR_TO_CHECK}" != "." ] + do + DIR_TO_CHECK=$(/usr/bin/dirname ${DIR_TO_CHECK}) + done +elif [ ! -d "${TARGET_DIR}" ] +then + echo "ERROR: \"${TARGET_DIR}\" is not a directory" >&1 + exit 1 +else + DIR_TO_CHECK=${TARGET_DIR} +fi + +# check filesystem type +FSTYPE=$(/usr/bin/timeout $TIMEOUT /usr/bin/findmnt -n -o fstype --target "${DIR_TO_CHECK}") +if [ "${FSTYPE}" != "lustre" ] +then + cat >&1 << EOF +/---------/ +WARNING: imagestore directory "${TARGET_DIR}" isn't on a LUSTRE filesystem. That is suboptimal. + +Please move parallax imagestore to a LUSTRE filesystem. + +Or if you are already aware, you can disable this warning adding the following annotation in the EDF: + +[annotations] +com.sarus.hooks.parallax_imagestore_create="false" +/---------/ +EOF + exit 1 +else + FSNAME=$(/usr/bin/timeout $TIMEOUT ${LFS_CMD} getname -n "${DIR_TO_CHECK}") + if [ $? -ne 0 ] + then + echo "ERROR: Cannot retrieve filesystem name for \"${DIR_TO_CHECK}\", rc=$?" >&1 + exit 1 + fi + + PFL_LAYOUT_REFERENCE="${ETC_DIR}/${FSNAME}_pfl_reference.json" + PFL_SETSTRIPE_OPTS_ENV="${ETC_DIR}/${FSNAME}_pfl_setstripe_opts.env" + + if [ ! -f "${PFL_LAYOUT_REFERENCE}" ] + then + echo "ERROR: Missing Progressive File layout reference file \"${PFL_LAYOUT_REFERENCE}\"" >&1 + exit 1 + fi + + if [ ! -f "${PFL_SETSTRIPE_OPTS_ENV}" ] + then + echo "ERROR: Missing Progressive File layout environment file \"${PFL_SETSTRIPE_OPTS_ENV}\"" >&1 + exit 1 + fi + + PFL_LAYOUT_TEMPLATE="${TMP_DIR}/pfl_layout.json.XXX" + . ${PFL_SETSTRIPE_OPTS_ENV} + + if [ -z "${LFS_SETSTRIPE_OPTS}" ] + then + echo "ERROR: Missing LFS_SETSTRIPE_OPTS environment variable " >&1 + exit 1 + fi +fi + +# create directory +if [ "$NEED_DIR_CREATION" == "true" ] +then + mkdir -p "${TARGET_DIR}" + RC=$? + if [ "${FSTYPE}" != "lustre" ] + then + exit ${RC} + else + /usr/bin/timeout ${TIMEOUT} ${LFS_CMD} setstripe ${LFS_SETSTRIPE_OPTS} ${TARGET_DIR} + RC=$? + if [ $RC -ne 0 ] + then + echo "ERROR: \"timeout $TIMEOUT lfs setstripe\" returned: $RC" >&1 + exit 1 + fi + fi +fi + +# Collect pfl.layout +PFL_LAYOUT_OUTPUT=$(mktemp ${PFL_LAYOUT_TEMPLATE}) +/usr/bin/timeout ${TIMEOUT} /usr/bin/lfs getstripe -y -d ${TARGET_DIR} | \ + /usr/bin/yq r - -j -P | \ + /usr/bin/jq 'with_entries(if (.key | test("component[0-9]$")) then ({key: .key, value: del(.value.sub_layout.lmm_objects)}) else ({key: .key, value: .value}) end)' \ + > ${PFL_LAYOUT_OUTPUT} +RC=$? +if [ $RC -ne 0 ] +then + echo "ERROR: \"timeout $TIMEOUT lfs getstripe\" returned: $RC" >&1 + exit 1 +fi + +/usr/bin/diff -q ${PFL_LAYOUT_REFERENCE} ${PFL_LAYOUT_OUTPUT} &>/dev/null +RC=$? +if [ $RC -ne 0 ] +then + cat >&1 << EOF +/---------/ +WARNING!!! +imagestore directory "${TARGET_DIR}" doesn't have the expected filesystem optimization + +Please run: +lustre_pfl_optimize ${TARGET_DIR} + +Or if you are already aware, you can disable this warning adding the following annotation in the EDF: + +[annotations] +com.sarus.hooks.parallax_imagestore_create="false" +/---------/ +EOF + +fi + +rm -f ${PFL_LAYOUT_OUTPUT} +exit $RC diff --git a/scripts/lustre_pfl/bin/lustre_pfl_optimize b/scripts/lustre_pfl/bin/lustre_pfl_optimize new file mode 100755 index 0000000..95bd4b5 --- /dev/null +++ b/scripts/lustre_pfl/bin/lustre_pfl_optimize @@ -0,0 +1,158 @@ +#!/bin/bash +# +# Optimize a directory and its content agains progressive file layout reference file schema. +# + +TARGET_DIR=$1 +BIN_DIR=$(readlink -f "$(dirname $0)") +ETC_DIR=$(readlink -f "${BIN_DIR}/../etc") +LFS_CMD="/usr/bin/lfs" +TIMEOUT="10" +TMP_DIR="/tmp" + +# Check dependencies +[ ! -x /usr/bin/timeout ] && echo "ERROR: missing /usr/bin/timeout" >&2 && exit 1 +[ ! -x /usr/bin/dirname ] && echo "ERROR: missing /usr/bin/dirname" >&2 && exit 1 +[ ! -x /usr/bin/findmnt ] && echo "ERROR: missing /usr/bin/findmnt" >&2 && exit 1 +[ ! -x ${LFS_CMD} ] && echo "ERROR: missing ${LFS_CMD}" >&2 && exit 1 +[ ! -x /usr/bin/yq ] && echo "ERROR: missing /usr/bin/yq" >&2 && exit 1 +[ ! -x /usr/bin/jq ] && echo "ERROR: missing /usr/bin/jq" >&2 && exit 1 +[ ! -x /usr/bin/diff ] && echo "ERROR: missing /usr/bin/diff" >&2 && exit 1 +[ ! -x /usr/bin/rsync ] && echo "ERROR: missing /usr/bin/rsync" >&2 && exit 1 + +MIGRATING_DIR="${TARGET_DIR}.optimizing" + +function rename_directory() { + mv ${MIGRATING_DIR} ${TARGET_DIR} + RC=$? + if [ $RC -ne 0 ] + then + echo "ERROR: mv ${MIGRATING_DIR} ${TARGET_DIR} returned: $RC" >&2 + echo "Optimization crashed. Target directory is in an undefined state. Please run \"${0}\" again." >&2 + return 1 + fi +} + +if [ -z "${TARGET_DIR}" ] +then + echo "ERROR: you must specify lustre directory path" >&2 + exit 1 +fi + +if [ ! -d "${TARGET_DIR}" ] +then + if [ -d "${MIGRATING_DIR}" ] + then + rename_directory || exit $? + else + echo "ERROR: target directory \"${TARGET_DIR}\" doesn't exist" >&2 + exit 1 + fi +fi + +# check filesystem type +FSTYPE=$(/usr/bin/timeout $TIMEOUT findmnt -n -o fstype --target "${TARGET_DIR}") +if [ "${FSTYPE}" != "lustre" ] +then + echo "ERROR: target directory \"${TARGET_DIR}\" isn't on a LUSTRE filesystem. Cannot optimize." >&2 + exit 1 +fi + +FSNAME=$(/usr/bin/timeout $TIMEOUT ${LFS_CMD} getname -n "${TARGET_DIR}") +if [ $? -ne 0 ] +then + echo "ERROR: Cannot retrieve filesystem name for \"${TARGET_DIR}\", rc=$?" >&2 + exit 1 +fi + +PFL_LAYOUT_REFERENCE="${ETC_DIR}/${FSNAME}_pfl_reference.json" +PFL_SETSTRIPE_OPTS_ENV="${ETC_DIR}/${FSNAME}_pfl_setstripe_opts.env" + +if [ ! -f "${PFL_LAYOUT_REFERENCE}" ] +then + echo "ERROR: Missing Progressive File layout reference file \"${PFL_LAYOUT_REFERENCE}\"" >&2 + exit 1 +fi + +if [ ! -f "${PFL_SETSTRIPE_OPTS_ENV}" ] +then + echo "ERROR: Missing Progressive File layout environment file \"${PFL_SETSTRIPE_OPTS_ENV}\"" >&2 + exit 1 +fi + +. ${PFL_SETSTRIPE_OPTS_ENV} + +if [ -z "${LFS_SETSTRIPE_OPTS}" ] +then + echo "ERROR: Missing LFS_SETSTRIPE_OPTS environment variable " >&2 + exit 1 +fi + +# Collect pfl.layout +PFL_LAYOUT_TEMPLATE="${TMP_DIR}/pfl_layout.json.XXX" +PFL_LAYOUT_OUTPUT=$(mktemp ${PFL_LAYOUT_TEMPLATE}) +/usr/bin/timeout ${TIMEOUT} ${LFS_CMD} getstripe -y -d ${TARGET_DIR} | \ + /usr/bin/yq r - -j -P | \ + /usr/bin/jq 'with_entries(if (.key | test("component[0-9]$")) then ({key: .key, value: del(.value.sub_layout.lmm_objects)}) else ({key: .key, value: .value}) end)' \ + > ${PFL_LAYOUT_OUTPUT} +RC=$? +if [ $RC -ne 0 ] +then + echo "ERROR: \"timeout $TIMEOUT lfs getstripe\" returned: $RC" >&2 + exit 1 +fi + +/usr/bin/diff -q ${PFL_LAYOUT_REFERENCE} ${PFL_LAYOUT_OUTPUT} &>/dev/null +RC=$? +if [ $RC -eq 0 ] +then + echo "ERROR: target directory \"${TARGET_DIR}\" is already optmized. Nothing to do." >&2 + exit 1 +fi +rm -f ${PFL_LAYOUT_OUTPUT} + +if [ -d "${MIGRATING_DIR}" ] +then + echo "RECOVERING Temporary Directory at ${MIGRATING_DIR}" +else + mkdir -p "${MIGRATING_DIR}" + RC=$? + if [ $RC -ne 0 ] + then + echo "ERROR: mkdir -p \"${MIGRATING_DIR}\" returned: $RC" >&2 + exit 1 + fi + echo "CREATED New Temporary Directory at ${MIGRATING_DIR}" +fi + +/usr/bin/timeout ${TIMEOUT} ${LFS_CMD} setstripe ${LFS_SETSTRIPE_OPTS} ${MIGRATING_DIR} +RC=$? +if [ $RC -ne 0 ] +then + rmdir ${MIGRATING_DIR} + echo "ERROR: \"timeout $TIMEOUT lfs setstripe\" returned: $RC" >&2 + exit 1 +fi + +echo "Moving data from ${TARGET_DIR} to ${MIGRATING_DIR}" +/usr/bin/rsync -av --remove-source-files ${TARGET_DIR}/ ${MIGRATING_DIR}/ +RC=$? +if [ $RC -ne 0 ] +then + echo "ERROR: rsync -av --delete ${TARGET_DIR}/ ${MIGRATING_DIR}/ returned: $RC" >&2 + echo "Optimization crashed. Target directory is in an undefined state. Please run \"${0}\" again." >&2 + exit 1 +fi + +find ${TARGET_DIR}/ -type d -empty -delete +RC=$? +if [ $RC -ne 0 ] +then + echo "ERROR: find ${TMP_DIR} -type d -empty -delete, returned: $RC" >&2 + echo "Optimization crashed. Target directory is currently in an undefined state. Please run \"${0}\" again." >&2 + exit 1 +fi + +rename_directory || exit $? + +exit 0 diff --git a/scripts/lustre_pfl/sbin/lustre_pfl_reference_setstripe b/scripts/lustre_pfl/sbin/lustre_pfl_reference_setstripe new file mode 100755 index 0000000..e6fbbcf --- /dev/null +++ b/scripts/lustre_pfl/sbin/lustre_pfl_reference_setstripe @@ -0,0 +1,137 @@ +#!/bin/bash +# +# Create a reference template for Progressive File Layout on a Lustre filesystem +# + +ARGS="$@" +SCRIPTNAME=$(basename $0) +LFS_CMD="/usr/bin/lfs" +TIMEOUT="10" +SBIN_DIR=$(readlink -f "$(dirname $0)") +ETC_DIR=$(readlink -f "${SBIN_DIR}/../etc") + +function help() { + cat >&2 << EOF + + This script creates a reference template for lustre progressive file layout + on a specific lustre filesystem and saves it at: + ${ETC_DIR} + + Usage: ${SCRIPTNAME} + +EOF +} + + +# Check dependencies +[ ! -x /usr/bin/timeout ] && echo "ERROR: missing /usr/bin/timeout" >&2 && exit 1 +[ ! -x /usr/bin/dirname ] && echo "ERROR: missing /usr/bin/dirname" >&2 && exit 1 +[ ! -x /usr/bin/findmnt ] && echo "ERROR: missing /usr/bin/findmnt" >&2 && exit 1 +[ ! -x ${LFS_CMD} ] && echo "ERROR: missing ${LFS_CMD}" >&2 && exit 1 +[ ! -x /usr/bin/yq ] && echo "ERROR: missing /usr/bin/yq" >&2 && exit 1 +[ ! -x /usr/bin/jq ] && echo "ERROR: missing /usr/bin/jq" >&2 && exit 1 +[ ! -x /usr/bin/diff ] && echo "ERROR: missing /usr/bin/diff" >&2 && exit 1 +[ ! -x /usr/bin/rmdir ] && echo "ERROR: missing /usr/bin/rmdir" >&2 && exit 1 +[ ! -x /usr/bin/mktemp ] && echo "ERROR: missing /usr/bin/mktemp" >&2 && exit 1 + +if [ ! -d "${ETC_DIR}" ] +then + mkdir -p "${ETC_DIR}" + [ ! -d "${ETC_DIR}" ] && echo "ERROR: cannot create \"${ETC_DIR}\"" >&2 && exit 1 +fi + +PARENT_DIR="${ARGS##* }" +LFS_SETSTRIPE_OPTS="${ARGS% *}" + +if [ -z "${PARENT_DIR}" ] || [ -z "${LFS_SETSTRIPE_OPTS}" ] +then + echo "ERROR: you must specify \"lfs setstripe\" options and a directory path on a target Lustre filesystem" >&2 + help + exit 1 +fi + +if [ ! -d "${PARENT_DIR}" ] +then + echo "ERROR: \"${PARENT_DIR}\" is not a directory" >&2 + exit 1 +fi + +FSTYPE=$(/usr/bin/timeout $TIMEOUT /usr/bin/findmnt -n -o fstype --target "${PARENT_DIR}") +if [ "${FSTYPE}" != "lustre" ] +then + echo "ERROR: \"${PARENT_DIR}\" is not a on a lustre filesystem" >&2 + exit 1 +fi + +FSNAME=$(/usr/bin/timeout $TIMEOUT ${LFS_CMD} getname -n "${PARENT_DIR}") +if [ $? -ne 0 ] +then + echo "ERROR: Cannot retrieve filesystem name for \"${PARENT_DIR}\", rc=$?" >&2 + exit 1 +fi + +# Create directory +TARGET_DIR=$(/usr/bin/mktemp -d -p "${PARENT_DIR}" pfl_template_XXX) +RC=$? +if [ ! -d "${TARGET_DIR}" ] +then + echo "ERROR: Cannot create directory at \"${TARGET_DIR}\", rc=$RC" >&2 + exit 1 +fi +PFL_REFERENCE="${ETC_DIR}/${FSNAME}_pfl_reference.json" +PFL_SETSTRIPE_OPTS_ENV="${ETC_DIR}/${FSNAME}_pfl_setstripe_opts.env" + +# Set Striping +/usr/bin/timeout ${TIMEOUT} ${LFS_CMD} setstripe ${LFS_SETSTRIPE_OPTS} ${TARGET_DIR} +RC=$? +if [ $RC -ne 0 ] +then + echo "ERROR: \"timeout $TIMEOUT lfs setstripe\" returned: $RC" >&2 + exit 1 +fi + +# Collect pfl.layout +/usr/bin/timeout ${TIMEOUT} ${LFS_CMD} getstripe -y -d ${TARGET_DIR} | \ + /usr/bin/yq r - -j -P | \ + /usr/bin/jq 'with_entries(if (.key | test("component[0-9]$")) then ({key: .key, value: del(.value.sub_layout.lmm_objects)}) else ({key: .key, value: .value}) end)' \ + > ${PFL_REFERENCE} +RC=$? +if [ $RC -ne 0 ] +then + echo "ERROR: \"timeout $TIMEOUT lfs getstripe\" returned: $RC" >&2 + exit 1 +fi +if [ ! -f "${PFL_REFERENCE}" ] +then + echo "ERROR: \"${PFL_REFERENCE}\" was not created." >&2 + exit 1 +else + echo "Created \"${PFL_REFERENCE}\"" +fi + +# Save lfs setstripe opts +echo "export LFS_SETSTRIPE_OPTS=\"${LFS_SETSTRIPE_OPTS}\"" > ${PFL_SETSTRIPE_OPTS_ENV} +RC=$? +if [ $RC -ne 0 ] +then + echo "ERROR: Cannot create \"${PFL_SETSTRIPE_OPTS_ENV}\", rc: $RC" >&2 + exit 1 +fi +if [ ! -f "${PFL_SETSTRIPE_OPTS_ENV}" ] +then + echo "ERROR: \"${PFL_SETSTRIPE_OPTS_ENV}\" was not created." >&2 + exit 1 +else + echo "Created \"${PFL_SETSTRIPE_OPTS_ENV}\"" +fi + +# Remove TARGET_DIR +/usr/bin/rmdir ${TARGET_DIR} +RC=$? +if [ $RC -ne 0 ] +then + echo "ERROR: Cannot remove \"${TARGET_DIR}\", rc: $RC" >&2 + exit 1 +fi + +exit 0