diff --git a/README.md b/README.md index 08278a7..d30ab37 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ pip install -r requirements.txt ``` ### B. Data Download -Download the GAUSS data from Globus (requires around 140GB of disk space): +Download the GAUSS data from Globus (requires up to 140GB of disk space if all data is downloaded): 1. First install the [Globus CLI](https://docs.globus.org/cli/): ```bash pip install globus-cli @@ -39,10 +39,16 @@ Download the GAUSS data from Globus (requires around 140GB of disk space): ```bash globus login ``` -3. Download the data: +3. Download the data. You can download all data (~140GB) using: ```bash sh scripts/download_gauss.sh 2e01e83a-5180-47f7-a6ab-c98b626ad9e4 data/gauss/ ``` + + Or you can select a subset of the data to download (e.g. `tas` and `monthly`) using: + ```bash + sh scripts/download_gauss.sh 2e01e83a-5180-47f7-a6ab-c98b626ad9e4 data/gauss/ --tas --monthly + ``` + If only a subset of the data is downloaded, it will affect which of the following steps can successfully run. To fit the regression models, the monthly temperature (--tas --monthly) data must be downloaded at a minimum. At this time, it is only possible to subselect temperature (--tas) or precipitation (--pr) data. ### C. Data Processing 1. Process the daily data to create monthly values. You can process all the daily data (both `tas` and `pr`) using: diff --git a/scripts/download_gauss.sh b/scripts/download_gauss.sh old mode 100644 new mode 100755 index 688e926..92a61bb --- a/scripts/download_gauss.sh +++ b/scripts/download_gauss.sh @@ -1,6 +1,6 @@ # Argument validation check -if [ "$#" -ne 3 ]; then - echo "Usage: $0 " +if [ "$#" -lt 3 ] || [ "$#" -gt 5 ]; then + echo "Usage: $0 [--tas | --pr] [--daily | --monthly]" exit 1 fi @@ -9,6 +9,51 @@ source_endpoint_id=$1 destination_endpoint_id=$2 local_destination=$3 +# Initialize optional variables +download_tas=false +download_pr=false +download_daily=false +download_monthly=false + +# Check for variable flags +for arg in "${@:4}" +do + case $arg in + --tas) + if [ "$download_pr" = true ]; then + echo "Error: You cannot select both ---tas and --pr." + exit 1 + fi + download_tas=true + ;; + --pr) + if [ "$download_tas" = true ]; then + echo "Error: You cannot select both --tas and --pr." + exit 1 + fi + download_pr=true + ;; + --daily) + if [ "$download_monthly" = true ]; then + echo "Error: You cannot select both --daily and --monthly." + exit 1 + fi + download_daily=true + ;; + --monthly) + if [ "$download_daily" = true ]; then + echo "Error: You cannot select both --daily and --monthly." + exit 1 + fi + download_monthly=true + ;; + *) + echo "Invalid option: $arg" + exit 1 + ;; + esac +done + # Create the local destination directory if it does not exist mkdir -p $local_destination @@ -23,85 +68,121 @@ do base_dirs+=("/SSP245-MA-GAUSS-LOWER-1.0.00$i/atm/proc/tseries/month_1") done -# Loop through each base directory and download the files -for dir in "${base_dirs[@]}" -do - echo "Listing files in $dir..." - # List files in the directory and filter for specific patterns - files=$(globus ls $source_endpoint_id:$dir | grep -E "\.TREFHT\.|\.\PRECT\.|\.TREFHTMN\.|\.TREFHTMX\.|\.QFLX\." | grep -E "\.h0\.") +# Download monthly files as long as user disn't request just daily +if [ "$download_daily" = false ]; then + # Define the file pattern based on the user's selection + file_pattern="" + if [ "$download_tas" = true ]; then + file_pattern="\.TREFHT\." + fi + if [ "$download_pr" = true ]; then + file_pattern="\.\PRECT\." + fi + if [ "$download_tas" = false ] && [ "$download_pr" = false ]; then + file_pattern="\.TREFHT\.|\.\PRECT\.|\.TREFHTMN\.|\.TREFHTMX\.|\.QFLX\." + fi - # Loop through filtered files and initiate transfer - for file in $files + # Loop through each base directory and download the files + for dir in "${base_dirs[@]}" do - # Check if file already exists at destination - if globus ls $destination_endpoint_id:$local_destination | grep -q "^$file$"; then - echo "Skipping $file, already exists at destination." - else - echo "Transferring $file from $dir..." - globus transfer $source_endpoint_id:$dir/$file $destination_endpoint_id:$local_destination$file - fi - done -done + echo "Listing files in $dir... with file pattern $file_pattern" + # List files in the directory and filter for specific patterns + files=$(globus ls $source_endpoint_id:$dir | grep -E "$file_pattern" | grep -E "\.h0\.") -# For MA-BASELINE.002 and MA-BASELINE.003, have to download PRECC and PRECL instead of PRECT -for i in {2..3} -do - echo "Listing files in /MA-BASELINE.00$i/atm/proc/tseries/month_1..." - files=$(globus ls $source_endpoint_id:/MA-BASELINE.00$i/atm/proc/tseries/month_1 | grep -E "\.PRECL\.|\.\PRECC\." | grep -E "\.h0\.") - for file in $files - do - if globus ls $destination_endpoint_id:$local_destination | grep -q "^$file$"; then - echo "Skipping $file, already exists at destination." - else - echo "Transferring $file from /MA-BASELINE.00$i/atm/proc/tseries/month_1..." - globus transfer $source_endpoint_id:/MA-BASELINE.00$i/atm/proc/tseries/month_1/$file $destination_endpoint_id:$local_destination$file - fi + # Loop through filtered files and initiate transfer + for file in $files + do + # Check if file already exists at destination + if globus ls $destination_endpoint_id:$local_destination | grep -q "^$file$"; then + echo "Skipping $file, already exists at destination." + else + echo "Transferring $file from $dir..." + globus transfer $source_endpoint_id:$dir/$file $destination_endpoint_id:$local_destination$file + fi + done done -done -# For MA-HISTORICAL, have to download PRECC and PRECL insrtead of PRECT -for i in {1..3} -do - echo "Listing files in /MA-HISTORICAL.00$i/atm/proc/tseries/month_1..." - files=$(globus ls $source_endpoint_id:/MA-HISTORICAL.00$i/atm/proc/tseries/month_1 | grep -E "\.PRECL\.|\.\PRECC\." | grep -E "\.h0\.") - for file in $files - do - if globus ls $destination_endpoint_id:$local_destination | grep -q "^$file$"; then - echo "Skipping $file, already exists at destination." - else - echo "Transferring $file from /MA-HISTORICAL.00$i/atm/proc/tseries/month_1..." - globus transfer $source_endpoint_id:/MA-HISTORICAL.00$i/atm/proc/tseries/month_1/$file $destination_endpoint_id:$local_destination$file - fi - done -done + # Only download additional variables if not only downloading tas + if [ "$download_tas" = false ]; then + # For MA-BASELINE.002 and MA-BASELINE.003, have to download PRECC and PRECL instead of PRECT + for i in {2..3} + do + echo "Listing files in /MA-BASELINE.00$i/atm/proc/tseries/month_1... with file pattern \.PRECL\.|\.\PRECC\." + files=$(globus ls $source_endpoint_id:/MA-BASELINE.00$i/atm/proc/tseries/month_1 | grep -E "\.PRECL\.|\.\PRECC\." | grep -E "\.h0\.") + for file in $files + do + if globus ls $destination_endpoint_id:$local_destination | grep -q "^$file$"; then + echo "Skipping $file, already exists at destination." + else + echo "Transferring $file from /MA-BASELINE.00$i/atm/proc/tseries/month_1..." + globus transfer $source_endpoint_id:/MA-BASELINE.00$i/atm/proc/tseries/month_1/$file $destination_endpoint_id:$local_destination$file + fi + done + done -# Download daily TREFHT and PRECT files for base_dirs -for dir in "${base_dirs[@]}" -do - # Replace month with day - dir=$(echo $dir | sed 's/month_1/day_1/g') - echo "Listing files in $dir..." - files=$(globus ls $source_endpoint_id:$dir | grep -E "\.TREFHT\.|\.PRECT" | grep -E "\.h1\.") - for file in $files + # For MA-HISTORICAL, have to download PRECC and PRECL insrtead of PRECT + for i in {1..3} + do + echo "Listing files in /MA-HISTORICAL.00$i/atm/proc/tseries/month_1... with file pattern \.PRECL\.|\.\PRECC\." + files=$(globus ls $source_endpoint_id:/MA-HISTORICAL.00$i/atm/proc/tseries/month_1 | grep -E "\.PRECL\.|\.\PRECC\." | grep -E "\.h0\.") + for file in $files + do + if globus ls $destination_endpoint_id:$local_destination | grep -q "^$file$"; then + echo "Skipping $file, already exists at destination." + else + echo "Transferring $file from /MA-HISTORICAL.00$i/atm/proc/tseries/month_1..." + globus transfer $source_endpoint_id:/MA-HISTORICAL.00$i/atm/proc/tseries/month_1/$file $destination_endpoint_id:$local_destination$file + fi + done + done + fi +fi + +# Download daily data as long as user didn't only request monthly data +if [ "$download_monthly" = false ]; then + # Define the file pattern based on the user's selection + file_pattern="" + if [ "$download_tas" = true ]; then + file_pattern="\.TREFHT\." + fi + if [ "$download_pr" = true ]; then + file_pattern="\.PRECT" + fi + if [ "$download_tas" = false ] && [ "$download_pr" = false ]; then + file_pattern="\.TREFHT\.|\.PRECT" + fi + + # Download daily TREFHT and/or PRECT files for base_dirs + for dir in "${base_dirs[@]}" do - if globus ls $destination_endpoint_id:$local_destination | grep -q "^$file$"; then - echo "Skipping $file, already exists at destination." - else - echo "Transferring $file from $dir..." - globus transfer $source_endpoint_id:$dir/$file $destination_endpoint_id:$local_destination$file - fi + # Replace month with day + dir=$(echo $dir | sed 's/month_1/day_1/g') + echo "Listing files in $dir... with file pattern $file_pattern" + files=$(globus ls $source_endpoint_id:$dir | grep -E "$file_pattern" | grep -E "\.h1\.") + for file in $files + do + if globus ls $destination_endpoint_id:$local_destination | grep -q "^$file$"; then + echo "Skipping $file, already exists at destination." + else + echo "Transferring $file from $dir..." + globus transfer $source_endpoint_id:$dir/$file $destination_endpoint_id:$local_destination$file + fi + done done -done -# Download daily PRECT from ARISE-HISTORICAL -echo "Listing files in /ARISE-HISTORICAL" -files=$(globus ls $source_endpoint_id:/ARISE-HISTORICAL | grep -E "\.PRECT\." | grep -E "\.h1\.") -for file in $files -do - if globus ls $destination_endpoint_id:$local_destination | grep -q "^$file$"; then - echo "Skipping $file, already exists at destination." - else - echo "Transferring $file from /ARISE-HISTORICAL..." - globus transfer $source_endpoint_id:/ARISE-HISTORICAL/$file $destination_endpoint_id:$local_destination$file + # Only download if not just downloading tas + if [ "$download_tas" = false ]; then + # Download daily PRECT from ARISE-HISTORICAL + echo "Listing files in /ARISE-HISTORICAL" + files=$(globus ls $source_endpoint_id:/ARISE-HISTORICAL | grep -E "\.PRECT\." | grep -E "\.h1\.") + for file in $files + do + if globus ls $destination_endpoint_id:$local_destination | grep -q "^$file$"; then + echo "Skipping $file, already exists at destination." + else + echo "Transferring $file from /ARISE-HISTORICAL..." + globus transfer $source_endpoint_id:/ARISE-HISTORICAL/$file $destination_endpoint_id:$local_destination$file + fi + done fi -done +fi \ No newline at end of file diff --git a/scripts/test_download_gauss.sh b/scripts/test_download_gauss.sh new file mode 100644 index 0000000..9c046e4 --- /dev/null +++ b/scripts/test_download_gauss.sh @@ -0,0 +1,166 @@ +# Configuration +SCRIPT="./scripts/download_gauss.sh" +SOURCE_ENDPOINT="test_source_endpoint" +DEST_ENDPOINT="test_dest_endpoint" +TEST_DIR="tmp/test" + +# Test mutually exclusive flags +test_mutually_exclusive_flags() { + echo "Testing mutually exclusive flags..." + + output=$($SCRIPT $SOURCE_ENDPOINT $DEST_ENDPOINT $TEST_DIR --tas --pr 2>&1) + if ! echo "$output" | grep -q "cannot select both.*tas.*pr"; then + echo "✗ Failed to catch mutually exclusive --tas and --pr" + return 1 + fi + + output=$($SCRIPT $SOURCE_ENDPOINT $DEST_ENDPOINT $TEST_DIR --daily --monthly 2>&1) + if ! echo "$output" | grep -q "cannot select both.*daily.*monthly"; then + echo "✗ Failed to catch mutually exclusive --daily and --monthly" + return 1 + fi + + echo "✓ Mutually exclusive flags tests passed" +} + +test_file_patterns() { + echo "Testing file pattern generation..." + + # Test TAS + MONTHLY + output=$($SCRIPT $SOURCE_ENDPOINT $DEST_ENDPOINT $TEST_DIR --tas --monthly 2>&1) + if echo "$output" | grep -q "TREFHT" && echo "$output" | grep -q "month_1" && ! echo "$output" | grep -q "PRECT\|PRECL\|PRECC\|day_1"; then + echo "✓ Test passed for --tas --monthly pattern." + else + echo "✗ Test failed for --tas --monthly pattern." + return 1 + fi + + # Test PR + MONTHLY + output=$($SCRIPT $SOURCE_ENDPOINT $DEST_ENDPOINT $TEST_DIR --pr --monthly 2>&1) + if echo "$output" | grep -q "PRECT" \ + && echo "$output" | grep -q "PRECL\|PRECC" \ + && echo "$output$" | grep -q "month_1" \ + && ! echo "$output" | grep -q "TREFHT\|day_1"; then + echo "✓ Test passed for --pr --monthly pattern." + else + echo "✗ Test failed for --pr --monthly pattern." + return 1 + fi + + # Test TAS + DAILY + output=$($SCRIPT $SOURCE_ENDPOINT $DEST_ENDPOINT $TEST_DIR --tas --daily 2>&1) + if echo "$output" | grep -q "TREFHT" \ + && echo "$output" | grep -q "day_1" \ + && ! echo "$output" | grep -q "PRECT\|ARISE\|month_1"; then + echo "✓ Test passed for --tas --daily pattern." + else + echo "✗ Test failed for --tas --daily pattern." + return 1 + fi + + # Test PR + DAILY + output=$($SCRIPT $SOURCE_ENDPOINT $DEST_ENDPOINT $TEST_DIR --pr --daily 2>&1) + if echo "$output" | grep -q "PRECT" \ + && echo "$output" | grep -q "day_1" \ + && echo "$output" | grep -q "ARISE" \ + && ! echo "$output" | grep -q "TREFHT\|month_1"; then + echo "✓ Test passed for --pr --daily pattern." + else + echo "✗ Test failed for --pr --daily pattern." + return 1 + fi + + # Test TAS only + output=$($SCRIPT $SOURCE_ENDPOINT $DEST_ENDPOINT $TEST_DIR --tas 2>&1) + if echo "$output" | grep -q "TREFHT" \ + && echo "$output" | grep -q "day_1" \ + && echo "$output" | grep -q "month_1" \ + && ! echo "$output" | grep -q "PRECT\|ARISE"; then + echo "✓ Test passed for --tas pattern." + else + echo "✗ Test failed for --tas pattern." + return 1 + fi + + # Test PR only + output=$($SCRIPT $SOURCE_ENDPOINT $DEST_ENDPOINT $TEST_DIR --pr 2>&1) + if echo "$output" | grep -q "PRECT" \ + && echo "$output" | grep -q "day_1" \ + && echo "$output" | grep -q "month_1" \ + && echo "$output" | grep -q "ARISE" \ + && echo "$output" | grep -q "PRECL\|PRECC" \ + && ! echo "$output" | grep -q "TREFHT"; then + echo "✓ Test passed for --pr pattern." + else + echo "✗ Test failed for --pr pattern." + return 1 + fi + + # Test MONTHLY only + output=$($SCRIPT $SOURCE_ENDPOINT $DEST_ENDPOINT $TEST_DIR --monthly 2>&1) + if echo "$output" | grep -q "PRECT" \ + && echo "$output" | grep -q "PRECL\|PRECC" \ + && echo "$output" | grep -q "TREFHT" \ + && echo "$output$" | grep -q "month_1" \ + && ! echo "$output" | grep -q "day_1\|ARISE"; then + echo "✓ Test passed for --monthly pattern." + else + echo "✗ Test failed for --monthly pattern." + return 1 + fi + + # Test DAILY only + output=$($SCRIPT $SOURCE_ENDPOINT $DEST_ENDPOINT $TEST_DIR --daily 2>&1) + if echo "$output" | grep -q "PRECT" \ + && echo "$output" | grep -q "TREFHT" \ + && echo "$output$" | grep -q "ARISE" \ + && echo "$output$" | grep -q "day_1" \ + && ! echo "$output" | grep -q "PRECL\|PRECC\|month_1"; then + echo "✓ Test passed for --daily pattern." + else + echo "✗ Test failed for --daily pattern." + return 1 + fi + + # Test with no patterns + output=$($SCRIPT $SOURCE_ENDPOINT $DEST_ENDPOINT $TEST_DIR 2>&1) + if echo "$output" | grep -q "PRECT" \ + && echo "$output" | grep -q "TREFHT" \ + && echo "$output" | grep -q "PRECL\|PRECC" \ + && echo "$output$" | grep -q "ARISE" \ + && echo "$output$" | grep -q "day_1" \ + && echo "$output$" | grep -q "month_1"; then + echo "✓ Test passed for no pattern (original function)." + else + echo "✗ Test failed for no pattern (original function)." + return 1 + fi +} + + +# Main test runner +main() { + + # Run all tests + tests=( + test_mutually_exclusive_flags + test_file_patterns + ) + + failed=0 + for test in "${tests[@]}"; do + if ! $test; then + failed=$((failed + 1)) + fi + done + + # Report results + total=${#tests[@]} + passed=$((total - failed)) + echo "Test Results: $passed/$total tests passed" + + return $failed +} + +# Run the test suite +main "$@" \ No newline at end of file