v492 dev16

docs/changes/143.maintenance.md

@@ -0,0 +1 @@
+Standardize directories and remove legacy support for curved atmospheres.

scripts/IRF.combine_lookup_table_parts.sh

@@ -80,7 +80,7 @@ if [[ -f $OFILE ]]; then
 fi
 
 # run scripts and output are written into this directory
-LOGDIR="$VERITAS_USER_LOG_DIR/$DATE/MSCW.MAKETABLES/${ANALYSIS_TYPE}/submit-COMBINETABLES-${UUID}/"
+LOGDIR="${ODIR}/submit-COMBINETABLES-${UUID}/"
 echo -e "Log files will be written to:\n $LOGDIR"
 mkdir -p "$LOGDIR"
 

scripts/IRF.generate_effective_area_parts.sh

@@ -84,7 +84,7 @@ if [[ ${DISPBDT} == "1" ]]; then
 fi
 # output and log directories
 ODIR="$VERITAS_IRFPRODUCTION_DIR/$IRFVERSION/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma"
-LOGDIR="$VERITAS_IRFPRODUCTION_DIR/$IRFVERSION/${ANALYSIS_TYPE}/${SIMTYPE}/${EPOCH}_ATM${ATM}_gamma/submit-EFFAREA-RECID${RECID}-${UUID}"
+LOGDIR="${ODIR}/submit-EFFAREA-RECID${RECID}-${UUID}"
 mkdir -p "$LOGDIR"
 echo "Input: $INDIR"
 echo "Output: $ODIR"

scripts/IRF.generate_lookup_table_parts.sh

@@ -64,19 +64,20 @@ if [[ -z "$VERITAS_IRFPRODUCTION_DIR" ]]; then
     echo "Error: IRF production directory not found: $VERITAS_IRFPRODUCTION_DIR"
     exit 1
 fi
-# input directory containing evndisp products (first check for curved, than normal atmosphere)
-INDIR="$VERITAS_IRFPRODUCTION_DIR/${EVNIRFVERSION}/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/ze${ZA}deg_curved_offset${WOBBLE}deg_NSB${NOISE}MHz"
+# input directory containing evndisp products
+# CURVED_ATMOSPHERE_MC INDIR="$VERITAS_IRFPRODUCTION_DIR/${EVNIRFVERSION}/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/ze${ZA}deg_curved_offset${WOBBLE}deg_NSB${NOISE}MHz"
+INDIR="$VERITAS_IRFPRODUCTION_DIR/${EVNIRFVERSION}/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/ze${ZA}deg_offset${WOBBLE}deg_NSB${NOISE}MHz"
 if [[ ! -d $INDIR ]]; then
     INDIR="$VERITAS_IRFPRODUCTION_DIR/${EVNIRFVERSION}/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/ze${ZA}deg_offset${WOBBLE}deg_NSB${NOISE}MHz"
     if [[ ! -d $INDIR ]]; then
-        echo "Error, could not locate input directory. Locations searched (plus curved):"
+        echo "Error, could not locate input directory. Locations searched (minus curved):"
         echo "$INDIR"
         exit 1
     fi
 fi
 # output and log directories
 ODIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/Tables"
-LOGDIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANALYSIS_TYPE}/${SIMTYPE}/${EPOCH}_ATM${ATM}_gamma/submit-MAKETABLES-${UUID}/"
+LOGDIR="${ODIR}/submit-MAKETABLES-${UUID}/"
 mkdir -p "$ODIR"
 chmod g+w "$ODIR"
 mkdir -p "$LOGDIR"

scripts/IRF.mscw_energy_MC.sh

@@ -85,18 +85,21 @@ if [[ -z "$VERITAS_IRFPRODUCTION_DIR" ]]; then
     exit 1
 fi
 # input directory containing evndisp products (first check for curved, than normal atmosphere)
-INDIR="$VERITAS_IRFPRODUCTION_DIR/${EVNIRFVERSION}/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/ze${ZA}deg_curved_offset${WOBBLE}deg_NSB${NOISE}MHz"
+# CURVED_ATMOSPHERE_MC INDIR="$VERITAS_IRFPRODUCTION_DIR/${EVNIRFVERSION}/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/ze${ZA}deg_curved_offset${WOBBLE}deg_NSB${NOISE}MHz"
+INDIR="$VERITAS_IRFPRODUCTION_DIR/${EVNIRFVERSION}/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/ze${ZA}deg_offset${WOBBLE}deg_NSB${NOISE}MHz"
 if [[ ! -d $INDIR ]]; then
     INDIR="$VERITAS_IRFPRODUCTION_DIR/${EVNIRFVERSION}/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/ze${ZA}deg_offset${WOBBLE}deg_NSB${NOISE}MHz"
     if [[ ! -d $INDIR ]]; then
-        echo "Error, could not locate input directory. Locations searched (plus curved):"
+        echo "Error, could not locate input directory. Locations searched (minus curved):"
         echo "$INDIR"
         exit 1
     fi
 fi
 # output and log directories
 ODIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma"
-LOGDIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANALYSIS_TYPE}/${SIMTYPE}/${EPOCH}_ATM${ATM}_gamma/submit-MSCW-RECID${RECID}-${UUID}"
+LOGDIR="${ODIR}/submit-MSCW-RECID${RECID}-${UUID}"
+mkdir -p "$ODIR"
+chmod g+w "$ODIR"
 mkdir -p "$LOGDIR"
 echo "Input: $INDIR"
 echo "Output: $ODIR"

scripts/IRF.production.sh

@@ -210,9 +210,9 @@ elif [[ "${SIMTYPE}" == "CARE_202404" ]] || [[ "${SIMTYPE}" == "CARE_24_20" ]];
     ######################################
     # TEST
     # ZENITH_ANGLES=( 00 20 30 35 40 45 )
-    # ZENITH_ANGLES=( 60 65 )
+    # ZENITH_ANGLES=( 50 55 60 65 )
     # WOBBLE_OFFSETS=( 0.5 )
-    # NSB_LEVELS=( 160 )
+    # NSB_LEVELS=( 200 )
     # IRF comparison
     # ZENITH_ANGLES=( 20 40 50 60 65 )
     # WOBBLE_OFFSETS=( 0.5 1.0 1.5 )
@@ -322,7 +322,7 @@ for VX in $EPOCH; do
        #############################################
        # XGB Classification Training
        if [[ $IRFTYPE == "TRAINXGBGH" ]]; then
-           BCKDIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANATYPE}/BDTtraining/mscw_${VX:0:2}_DISP"
+           BCKDIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANATYPE}/BDTtraining/mscw_${VX:0:2}_XGB"
            RUNPAR="$VERITAS_EVNDISP_AUX_DIR/ParameterFiles/XGB-classify-parameter.json"
            ODIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANATYPE}/${SIMTYPE}/${VX}_ATM${ATM}_gamma/TrainXGBGammaHadron"
            echo "XGB Classification Training"
@@ -436,10 +436,11 @@ for VX in $EPOCH; do
                     # run simulations through evndisp
                     if [[ $IRFTYPE == "EVNDISP" ]] || [[ $IRFTYPE == "MVAEVNDISP" ]] || [[ $IRFTYPE == "EVNDISPCOMPRESS" ]]; then
                        SIMDIRZA="$SIMDIR"
-                       if [[ -e "$SIMDIR/Zd${ZA}_curved/" ]]; then
-                          SIMDIRZA="$SIMDIR/Zd${ZA}_curved/"
-                          echo "Using curved atmosphere simulations from $SIMDIRZA"
-                      elif [[ -e "$SIMDIR/Zd${ZA}/" ]]; then
+                       # CURVED_ATMOSPHERE_MC
+                       # if [[ -e "$SIMDIR/Zd${ZA}_curved/" ]]; then
+                       #   SIMDIRZA="$SIMDIR/Zd${ZA}_curved/"
+                       #    echo "Using curved atmosphere simulations from $SIMDIRZA"
+                       if [[ -e "$SIMDIR/Zd${ZA}/" ]]; then
                           SIMDIRZA="$SIMDIR/Zd${ZA}/"
                           echo "Using flat atmosphere simulations from $SIMDIRZA"
                        fi

scripts/IRF.trainTMVAforAngularReconstruction.sh

@@ -63,7 +63,7 @@ if [[ -z "$VERITAS_IRFPRODUCTION_DIR" ]]; then
 fi
 # output and log directories
 ODIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/TMVA_AngularReconstruction/ze${ZA}deg/"
-LOGDIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANALYSIS_TYPE}/${SIMTYPE}/${EPOCH}_ATM${ATM}_gamma/submit-TMVAAngRes-RECID${RECID}-${UUID}"
+LOGDIR="${ODIR}/submit-TMVAAngRes-RECID${RECID}-${UUID}"
 mkdir -p "$ODIR"
 chmod g+w "$ODIR"
 mkdir -p "$LOGDIR"
@@ -85,15 +85,16 @@ check_evndisp_directory()
 {
     W=${1}
     N=${2}
-    # input directory containing evndisp products (first check for curved, than normal atmosphere)
+    # input directory containing evndisp products
     INDIRBASE="$VERITAS_IRFPRODUCTION_DIR/${EVNIRFVERSION}/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma"
     if [[ -n "$VERITAS_IRFPRODUCTION_DIR" ]]; then
-        INDIR="${INDIRBASE}/ze${ZA}deg_curved_offset${W}deg_NSB${N}MHz"
+        # CURVED_ATMOSPHERE_MC INDIR="${INDIRBASE}/ze${ZA}deg_curved_offset${W}deg_NSB${N}MHz"
+        INDIR="${INDIRBASE}/ze${ZA}deg_offset${W}deg_NSB${N}MHz"
     fi
     if [[ ! -d $INDIR ]]; then
         INDIR="${INDIRBASE}/ze${ZA}deg_offset${W}deg_NSB${N}MHz"
         if [[ ! -d $INDIR ]]; then
-            echo "Error, could not locate input directory. Locations searched (plus curved):"
+            echo "Error, could not locate input directory. Locations searched (minus curved):"
             echo "$INDIR"
             exit 1
         fi

scripts/IRF.trainXGBforGammaHadronSeparationTraining.sh

@@ -1,11 +1,7 @@
 #!/bin/bash
 # train XGB for gamma/hadron separation
 #
-# note the large amount of hardwired parameters in this scripts:
-# - zenith angles to be trained
 # - training at wobble offsets 0.5 deg only
-# - fixed of NSB levels (adapted to stdHV settings)
-#
 
 h_cpu=11:59:59; h_vmem=16000M; tmpdir_size=24G
 EDVERSION=$(cat $VERITAS_EVNDISP_AUX_DIR/IRFVERSION)
@@ -95,15 +91,16 @@ mkdir -p $LOGDIR
 mkdir -p $ODIR
 
 #####################################
-# energy bins
+# energy / zenith bins
 NENE=$(jq '.energy_bins_log10_tev | length' "$RUNPAR")
+NEZE=$(jq '.zenith_bins_deg | length' "$RUNPAR")
 RUNPAR_CONTENT=$(cat "$RUNPAR")
-echo "Number of energy bins: $NENE"
+echo "Number of energy / zenith bins: $NENE $NEZE"
 
 #####################################
-# zenith angle bins of MC simulation files
-ZENITH_ANGLES=( 20 30 35 40 45 50 55 60 65)
-NOISE_VALUES=(100 160 200 250 350 450)
+# zenith angle / NSB bins of MC simulation files
+ZENITH_ANGLES=($(jq -r '.input_zenith_angles[]' "$RUNPAR"))
+NOISE_VALUES=($(jq -r '.input_noise_values[]' "$RUNPAR"))
 
 ####################################
 # Run prefix
@@ -149,13 +146,16 @@ BCKLIST="${ODIR}/bck_files.list"
 echo "Background file list: $BCKLIST"
 rm -f "${BCKLIST}"
 touch "${BCKLIST}"
-for ((i=0; i<=2; i++)); do
+tmpfile=$(mktemp)
+for ((i=0; i<${NEZE}; i++)); do
   if [[ ! -d "${BDIR}/Ze_${i}" ]]; then
       echo "Error, directory with background files ${BDIR}/Ze_${i} not found, exiting..."
       exit 1
   fi
-  find ${BDIR}/Ze_${i} -name "*.root" | sort -R | head -n 1000 >> "${BCKLIST}"
+  find ${BDIR}/Ze_${i} -name "*.root" | shuf -n 1000 >> "${tmpfile}"
 done
+shuf "$tmpfile" > "${BCKLIST}"
+rm "$tmpfile"
 
 ###############################################################
 # loop over energy bins and submit a job for each bin

scripts/helper_scripts/IRF.dispXGB_sub.sh

@@ -4,7 +4,6 @@
 # Don't do set -e.
 # set -e
 
-# parameters replaced by parent script using sed
 MSCW_FILE=FFILE
 ODIR=OODIR
 env_name="eventdisplay_ml"

scripts/helper_scripts/IRF.mscw_energy_MC_sub.sh

@@ -96,14 +96,23 @@ if [ $DISPBDT -eq 1 ]; then
     else
         DISPDIR="${VERITAS_EVNDISP_AUX_DIR}/DispBDTs/${ANATYPE}/${EPOCH}_ATM${ATM}/"
     fi
+    # zenith angle dependent disp training; important at large zenith
     if [[ "${ZA}" -lt "38" ]]; then
-        DISPDIR="${DISPDIR}/SZE/"
+        DISPDIR="${DISPDIR}/20deg/"
     elif [[ "${ZA}" -lt "48" ]]; then
-        DISPDIR="${DISPDIR}/MZE/"
+        DISPDIR="${DISPDIR}/45deg/"
     elif [[ "${ZA}" -lt "58" ]]; then
-        DISPDIR="${DISPDIR}/LZE/"
+        DISPDIR="${DISPDIR}/55deg/"
+    elif [[ "${ZA}" -lt "62" ]]; then
+        DISPDIR="${DISPDIR}/60deg/"
+        if [[ ${SIMTYPE} == "CARE_RedHV" ]]; then  # fix for incomplete MC set
+            DISPDIR="${DISPDIR}/55deg/"
+        fi
     else
-        DISPDIR="${DISPDIR}/XZE/"
+        DISPDIR="${DISPDIR}/65deg/"
+        if [[ ${SIMTYPE} == "CARE_RedHV" ]]; then  # fix for incomplete MC set
+            DISPDIR="${DISPDIR}/55deg/"
+        fi
     fi
     # unzip XML files into DDIR
     cp -v -f ${DISPDIR}/*.xml.gz ${DDIR}/