diff --git a/.gitignore b/.gitignore index f20af3db..fd672be3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ build/ dist/ tests/utils/test_follow_symlinks tests/utils/test_follow_symlinks_non_archived +tests/utils/globus_tar_deletion zstash.egg-info/ *.pyc *~ diff --git a/tests/README.md b/tests/README.md index d8dead64..c4552471 100644 --- a/tests/README.md +++ b/tests/README.md @@ -6,6 +6,7 @@ tests/ integration/ # Tests that call zstash from the command line bash_tests/ # Test zstash commands via bash scripts + run_from_any/ # Run these from any machine run_from_chrysalis/ # Run these from Chrysalis (these use Globus and/or require the Chrysalis file system) run_from_perlmutter/ # Run these from Perlmutter (these use `hsi` directly and/or require the Perlmutter file system) python_tests/ # Test zstash commands via Python unittest wrappers @@ -39,6 +40,7 @@ python -m unittest tests/integration/python_tests/group_by_workflow/test_*.py cd tests/integration/bash_tests/run_from_any/ # Review the directions at the bottom of globus_auth.bash # Run `./globus_auth.bash` with the appropriate parameters. +# Run `./test_globus_tar_deletion.bash` with the appropriate parameters. ``` ### Perlmutter-specific diff --git a/tests/integration/bash_tests/run_from_any/test_globus_tar_deletion.bash b/tests/integration/bash_tests/run_from_any/test_globus_tar_deletion.bash new file mode 100755 index 00000000..4b76f4c5 --- /dev/null +++ b/tests/integration/bash_tests/run_from_any/test_globus_tar_deletion.bash @@ -0,0 +1,286 @@ +# Assertions ################################################################## +check_log_has() +{ + local expected_grep="${1}" + local log_file="${2}" + grep -q "${expected_grep}" ${log_file} + if [ $? != 0 ]; then + echo "Expected grep '${expected_grep}' not found in $(realpath ${log_file}). Test failed." + return 2 + fi + return 0 +} + +check_log_does_not_have() +{ + local not_expected_grep="${1}" + local log_file="${2}" + grep -q "${not_expected_grep}" ${log_file} + if [ $? == 0 ]; then + echo "Not-expected grep '${not_expected_grep}' was found in $(realpath ${log_file}). Test failed." + return 2 + fi + return 0 +} + +# Helper functions ############################################################ +setup() +{ + echo "##########################################################################################################" + local case_name="${1}" + local src_dir="${2}" + echo "Testing: ${case_name}" + full_dir="${src_dir}/${case_name}" + rm -rf ${full_dir} + mkdir -p ${full_dir} + cd ${full_dir} + + mkdir zstash_demo + mkdir zstash_demo/empty_dir + mkdir zstash_demo/dir + echo -n '' > zstash_demo/file_empty.txt + echo 'file0 stuff' > zstash_demo/dir/file0.txt +} + +get_endpoint() +{ + # Usage example: + # uuid=$(get_endpoint NERSC_PERLMUTTER_ENDPOINT) + + local endpoint_name=$1 + # Define endpoints; see https://app.globus.org/collections + LCRC_IMPROV_DTN_ENDPOINT=15288284-7006-4041-ba1a-6b52501e49f1 + NERSC_PERLMUTTER_ENDPOINT=6bdc7956-fc0f-4ad2-989c-7aa5ee643a79 + NERSC_HPSS_ENDPOINT=9cd89cfd-6d04-11e5-ba46-22000b92c6ec + PIC_COMPY_DTN_ENDPOINT=68fbd2fa-83d7-11e9-8e63-029d279f7e24 + GLOBUS_TUTORIAL_COLLECTION_1_ENDPOINT=6c54cade-bde5-45c1-bdea-f4bd71dba2cc + case ${endpoint_name} in + LCRC_IMPROV_DTN_ENDPOINT) + echo ${LCRC_IMPROV_DTN_ENDPOINT} + ;; + NERSC_PERLMUTTER_ENDPOINT) + echo ${NERSC_PERLMUTTER_ENDPOINT} + ;; + NERSC_HPSS_ENDPOINT) + echo ${NERSC_HPSS_ENDPOINT} + ;; + PIC_COMPY_DTN_ENDPOINT) + echo ${PIC_COMPY_DTN_ENDPOINT} + ;; + GLOBUS_TUTORIAL_COLLECTION_1_ENDPOINT) + echo ${GLOBUS_TUTORIAL_COLLECTION_1_ENDPOINT} + ;; + *) + echo "Unknown endpoint name: ${endpoint_name}" >&2 + exit 1 + ;; + esac +} + +confirm() { + read -p "$1 (y/n): " -n 1 -r + echo + [[ $REPLY =~ ^[Yy]$ ]] +} + +wait_for_directory() { + local dir_path="$1" + local max_wait=300 # 5 minutes + local waited=0 + + echo "Waiting for directory ${dir_path} to be created by Globus transfer..." + while [ ! -d "${dir_path}" ] && [ ${waited} -lt ${max_wait} ]; do + sleep 5 + waited=$((waited + 5)) + echo " Waited ${waited}s..." + done + + if [ -d "${dir_path}" ]; then + echo "Directory appeared after ${waited}s" + return 0 + else + echo "Directory did not appear after ${max_wait}s" + return 1 + fi +} + +# Tests ####################################################################### +test_globus_tar_deletion() +{ + local path_to_repo=$1 + local dst_endpoint=$2 + local dst_dir=$3 + local blocking_str=$4 + local keep_str=$5 + + src_dir=${path_to_repo}/tests/utils/globus_tar_deletion + rm -rf ${src_dir} # Start fresh + mkdir -p ${src_dir} + dst_endpoint_uuid=$(get_endpoint ${dst_endpoint}) + globus_path=globus://${dst_endpoint_uuid}/${dst_dir} + + case_name=${blocking_str}_${keep_str} + echo "Running test_globus_tar_deletion on case=${case_name}" + echo "Exit codes: 0 -- success, 1 -- zstash failed, 2 -- grep check failed" + + setup ${case_name} "${src_dir}" + + if [ "$blocking_str" == "non-blocking" ]; then + blocking_flag="--non-blocking" + else + blocking_flag="" + fi + + if [ "$keep_str" == "keep" ]; then + keep_flag="--keep" + else + keep_flag="" + fi + + zstash create ${blocking_flag} ${keep_flag} --hpss=${globus_path}/${case_name} --maxsize 128 zstash_demo 2>&1 | tee ${case_name}.log + if [ $? != 0 ]; then + echo "${case_name} failed. Check ${case_name}_create.log for details. Cannot continue." + return 1 + fi + echo "${case_name} completed successfully. Checking ${case_name}.log now." + check_log_has "Creating new tar archive 000000.tar" ${case_name}.log || return 2 + + echo "" + echo "Checking directory status after 'zstash create' has completed. src should only have index.db. dst should have tar and index.db." + echo "Checking logs in current directory: ${PWD}" + + echo "" + echo "Checking src" + ls ${src_dir}/${case_name}/ 2>&1 | tee ls_${case_name}_src_output.log + check_log_has "${case_name}.log" ls_${case_name}_src_output.log || return 2 + check_log_has "zstash_demo" ls_${case_name}_src_output.log || return 2 + echo "" + ls ${src_dir}/${case_name}/zstash_demo 2>&1 | tee ls_${case_name}_src2_output.log + check_log_has "zstash" ls_${case_name}_src2_output.log || return 2 + echo "" + ls ${src_dir}/${case_name}/zstash_demo/zstash 2>&1 | tee ls_${case_name}_src3_output.log + check_log_has "index.db" ls_${case_name}_src3_output.log || return 2 + if [ "$keep_str" == "keep" ]; then + check_log_has "000000.tar" ls_${case_name}_src3_output.log || return 2 + else + check_log_does_not_have "000000.tar" ls_${case_name}_src3_output.log || return 2 + fi + + echo "" + echo "Checking dst" + if [ "$blocking_str" == "non-blocking" ]; then + wait_for_directory "${dst_dir}/${case_name}" || return 1 + fi + ls ${dst_dir}/ 2>&1 | tee ls_${case_name}_dst_output.log + check_log_has "${case_name}" ls_${case_name}_dst_output.log || return 2 + echo "" + ls ${dst_dir}/${case_name}/ 2>&1 | tee ls_${case_name}_dst2_output.log + check_log_has "index.db" ls_${case_name}_dst2_output.log || return 2 + check_log_has "000000.tar" ls_${case_name}_dst2_output.log || return 2 + + return 0 # Success +} + +# Follow these directions ##################################################### + +# Example usage: +# ./test_globus_tar_deletion.bash run1 /home/ac.forsyth2/ez/zstash /home/ac.forsyth2/zstash_tests LCRC_IMPROV_DTN_ENDPOINT + +# Command line parameters: +unique_id="$1" +path_to_repo="$2" # /home/ac.forsyth2/ez/zstash +dst_basedir="$3" # /home/ac.forsyth2/zstash_tests +endpoint_str="$4" # LCRC_IMPROV_DTN_ENDPOINT +fresh_globus="${5:-no}" # Default to "no" if not provided +machine_dst_dir=${dst_basedir}/test_globus_tar_deletion_${unique_id} + + +echo "You may wish to clear your dst directories for a fresh start:" +echo "rm -rf ${dst_basedir}/test_globus_tar_deletion*" +echo "It is advisable to just set a unique_id to avoid directory conflicts." +echo "Currently, unique_id=${unique_id}" +if ! confirm "Is the unique_id correct?"; then + exit 1 +fi + +echo "Go to https://app.globus.org/file-manager?two_pane=true > For "Collection", choose the endpoint you're testing, and authenticate if needed:" +echo "LCRC Improv DTN, NERSC Perlmutter, NERSC HPSS, pic#compy-dtn" +if ! confirm "Have you authenticated into the endpoint you're testing?"; then + exit 1 +fi + +if [ "$fresh_globus" == "yes" ]; then + INI_PATH=${HOME}/.zstash.ini + TOKEN_FILE=${HOME}/.zstash_globus_tokens.json + rm -rf ${INI_PATH} + rm -rf ${TOKEN_FILE} + echo "Reset Globus consents:" + echo "https://auth.globus.org/v2/web/consents > Globus Endpoint Performance Monitoring > rescind all" + if ! confirm "Have you revoked Globus consents?"; then + exit 1 + fi +fi + +# Test execution with independent runs and pass/fail tracking +run_test_with_tracking() { + local test_name="$1" + shift + local args=("$@") + + echo "" + echo "==========================================" + echo "Running: ${test_name}" + echo "==========================================" + + if test_globus_tar_deletion "${args[@]}"; then + # Print test result in the output block AND at the end + echo "✓ ${test_name} PASSED" + test_results+=("✓ ${test_name} PASSED") # Uses Global variable + ((tests_passed++)) + return 0 + else + # Print test result in the output block AND at the end + echo "✗ ${test_name} FAILED" + test_results+=("✗ ${test_name} FAILED") # Uses Global variable + ((tests_failed++)) + return 1 + fi +} + +# Initialize counters +tests_passed=0 +tests_failed=0 +test_results=() # Global variable to hold test results + +echo "Primary tests: single authentication code tests for each endpoint" +echo "If a test hangs, check if https://app.globus.org/activity reports any errors on your transfers." + +# Run all tests independently +run_test_with_tracking "blocking_non-keep" ${path_to_repo} ${endpoint_str} ${machine_dst_dir} "blocking" "non-keep" || true +run_test_with_tracking "non-blocking_non-keep" ${path_to_repo} ${endpoint_str} ${machine_dst_dir} "non-blocking" "non-keep" || true +run_test_with_tracking "blocking_keep" ${path_to_repo} ${endpoint_str} ${machine_dst_dir} "blocking" "keep" || true +run_test_with_tracking "non-blocking_keep" ${path_to_repo} ${endpoint_str} ${machine_dst_dir} "non-blocking" "keep" || true + +# Print summary +echo "" +echo "==========================================" +echo "TEST RESULTS" +echo "==========================================" +for result in "${test_results[@]}"; do + echo "${result}" +done +echo "==========================================" +echo "TEST SUMMARY" +echo "==========================================" +echo "Total tests: $((tests_passed + tests_failed))" +echo "Passed: ${tests_passed}" +echo "Failed: ${tests_failed}" +echo "==========================================" + +if [ ${tests_failed} -eq 0 ]; then + echo "All globus tar deletion tests completed successfully." + exit 0 +else + echo "Some tests failed. Please review the logs above." + exit 1 +fi