From 3931fdc3fdbb85e4d3c45250f82d1578c1e5411e Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Wed, 1 Jul 2026 14:14:36 -0700 Subject: [PATCH 01/29] remove APOE ref in spreadsheet unless it exists --- population_report.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/population_report.py b/population_report.py index 696f1e6..6dcf18c 100644 --- a/population_report.py +++ b/population_report.py @@ -2297,9 +2297,12 @@ def get_case_stats(subject_id, timepoint): print('Report generated in ' + dir + '/reports/' + run_folder + '/population_report' + output_dir + "_" + date_filename + '.html') print('Excluded report generated in ' + dir + '/reports/' + run_folder + '/population_report_exclude' + output_dir + "_" + date_filename + '.html') -if os.path.exists(os.path.join(dir, '../dce_available_3524_ac.xlsx')): +apoe_file = os.path.join(dir, '../dce_available_3524_ac.xlsx') +apoe_exists = os.path.exists(apoe_file) + +if apoe_exists: # add apoe and cdr fields to population_data - df = pd.read_excel(os.path.join(dir, '../dce_available_3524_ac.xlsx'), sheet_name="main") + df = pd.read_excel(apoe_file, sheet_name="main") # get apoe and cdr values for each subject for subject in population_data.keys(): @@ -2509,7 +2512,11 @@ def get_case_stats(subject_id, timepoint): df_success = pd.DataFrame(population_data) # df_exclude = pd.DataFrame(population_data_exclude) -order = ["Date", "APOE", "Sex", "Age", "Machine", "Institution", "Coil", "TR", "Time_resolution", "TE", "Flip_angle", "n_reps", +order = ["Date"] +if apoe_exists: + order.append("APOE") +order.extend([ + "Sex", "Age", "Machine", "Institution", "Coil", "TR", "Time_resolution", "TE", "Flip_angle", "n_reps", "Approximate SNR", "AIFitness", "aif_fitted_r2", "manual_aif_status", "max_disp", "T1_blood", "T1_wm_median", "T1_gm_median", "Ktrans_wm_median", "Ktrans_gm_median", "Ktrans_Hippo_median", "Ktrans_PhG_median", "Ktrans_Putamen_median", "Ktrans_Pallidum_median", "Ktrans_Thalamus_median", "Ktrans_Caudate_median", "Ktrans_Amygdala_median", "Ktrans_Entorhinal_cortex_median", @@ -2539,10 +2546,10 @@ def get_case_stats(subject_id, timepoint): "superiorparietal_thickness_avg", "superiorparietal_thickness_std", "superiortemporal_thickness_avg", "superiortemporal_thickness_std", "supramarginal_thickness_avg", "supramarginal_thickness_std", "frontalpole_thickness_avg", "frontalpole_thickness_std", "temporalpole_thickness_avg", "temporalpole_thickness_std", "transversetemporal_thickness_avg", "transversetemporal_thickness_std", - "insula_thickness_avg", "insula_thickness_std"] + "insula_thickness_avg", "insula_thickness_std"]) df_success = df_success.T -df_success = df_success[order] +df_success = df_success.reindex(columns=order) order_exclude = order.copy() From 488a33b307c3e06694243b666cf7b3e9bb9d4e25 Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Wed, 1 Jul 2026 14:18:21 -0700 Subject: [PATCH 02/29] update refs to rawdata to fit new BIDS standard (v1.11.1) rawdata is now sourcedata/raw, sometimes it's in the root but they don't make it clear when --- .github/workflows/DCE.yml | 4 ++-- DCE_all.sh | 2 +- README.md | 4 ++-- preprocess_all.sh | 5 +++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/DCE.yml b/.github/workflows/DCE.yml index 3120896..eab42ab 100644 --- a/.github/workflows/DCE.yml +++ b/.github/workflows/DCE.yml @@ -53,10 +53,10 @@ jobs: # uses: matlab-actions/setup-matlab@v1 - name: Preprocess the data - run: ./preprocess_all.sh -d /media/network_mriphysics/USC-PPG/RUNNER_DATA/rawdata -b -Z -m -c + run: ./preprocess_all.sh -d /media/network_mriphysics/USC-PPG/RUNNER_DATA/sourcedata/raw -b -Z -m -c - name: Run DCE - run: ./DCE_all.sh -d /media/network_mriphysics/USC-PPG/RUNNER_DATA/rawdata -f + run: ./DCE_all.sh -d /media/network_mriphysics/USC-PPG/RUNNER_DATA/sourcedata/raw -f - name: Output logs run: | diff --git a/DCE_all.sh b/DCE_all.sh index aa540bf..a502201 100755 --- a/DCE_all.sh +++ b/DCE_all.sh @@ -35,7 +35,7 @@ while getopts ":d:C:fhl:sST:" options; do DATA_DIR=${DATA_DIR::-1} fi date=$(date +%Y-%m-%d) - DERIV_DIR=$(dirname $DATA_DIR)/derivatives + DERIV_DIR=$(dirname $(dirname $DATA_DIR))/derivatives LOG_FILE=$DERIV_DIR/logs/dce_log_$date.txt if [ ! -d "$DATA_DIR/logs" ] then diff --git a/README.md b/README.md index 1cb4bb8..60d3628 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ All data is assumed to be [BIDS](https://bids-specification.readthedocs.io/) com | Flag | Description | |---|---| -| `-d [rawdata_path]` | **Required.** Path to your BIDS raw data folder. | +| `-d [rawdata_path]` | **Required.** Path to your BIDS raw data (NIFTIs) folder. | | `-a [suffix]` | AIF suffix (default: `desc-AIF_mask`). `.nii.gz` is appended automatically. | | `-A [mode]` | Enable AutoAIF: `A` (fully automatic), `M` (manual if available), or `T` (manual + training if available). Requires the [vascular_function repo and weights](https://github.com/petmri/vascular_function). | | `-b` | Enable first round of bias field correction. | @@ -216,7 +216,7 @@ All data is assumed to be [BIDS](https://bids-specification.readthedocs.io/) com | Flag | Description | |---|---| -| `-d [path]` | **Required.** Path to raw BIDS data directory. | +| `-d [path]` | **Required.** Path to raw BIDS data directory. Should be something like bids_root/sourcedata/raw | | `-C [name]` | Enable comparison mode. Copies essential files from a standard run if a preprocessed run of the same name does not exist. | | `-f` | Enable FreeSurfer WM parcellation for subregion analysis. | | `-s` | Skip cases already processed. | diff --git a/preprocess_all.sh b/preprocess_all.sh index 0195e70..c901fc1 100755 --- a/preprocess_all.sh +++ b/preprocess_all.sh @@ -71,8 +71,9 @@ while getopts ":d:bBa:A:ZfhcC:mMsS:tl:T:w:" options; do DATA_DIR=${DATA_DIR::-1} fi DATE=$(date +%Y-%m-%d) - # derivatives dir is up 1 level from data dir - DERIV_DIR=$(dirname $DATA_DIR)/derivatives + # derivatives dir is up 2 levels from data dir + DERIV_DIR=$(dirname $(dirname $DATA_DIR))/derivatives + echo $DERIV_DIR if [ ! -d "$DERIV_DIR" ] then mkdir -p "$DERIV_DIR" From 17dbf22f0a59ad2902ae17f0bec26e67a0ca1d70 Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Wed, 1 Jul 2026 15:12:11 -0700 Subject: [PATCH 03/29] do not double gaussian normalize a slice if there are too few WM voxels on that slice --- VFA_norm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VFA_norm.py b/VFA_norm.py index 6896816..fab9b64 100644 --- a/VFA_norm.py +++ b/VFA_norm.py @@ -89,7 +89,7 @@ def double_gaussian(x, A1, mu1, sigma1, A2, mu2, sigma2): amp_guess = 0 params = 0 # check if slice is empty - if area == 0 or pstdev(wm_data[:,:,i][a]) == 0: + if area < 1000 or pstdev(wm_data[:,:,i][a]) == 0: # just get from next slice # if i == slice_num - 1: # a = np.where(wm_data[:,:,i-1] > 0) From 3c343bb00112545c51ea86b53ca709a9c6549ab2 Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Wed, 1 Jul 2026 15:22:12 -0700 Subject: [PATCH 04/29] update skip logic to look for gzipped T1map --- preprocess_all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/preprocess_all.sh b/preprocess_all.sh index c901fc1..0a700ea 100755 --- a/preprocess_all.sh +++ b/preprocess_all.sh @@ -283,7 +283,7 @@ for source_dir in $DATA_DIR/$SCRIPT_LOOP_DIRS; do if [ $SKIP_IF_SUCCESS -eq 1 ] then if [ -f "dce/${PREFIX}_desc-bfcz_DCE.nii.gz" ] && [ -f "anat/${PREFIX}_space-DCEref_desc-brain_mask.nii.gz" ] && \ - [ -f "dce/${PREFIX}_desc-AIF_T1map.nii.gz" ] && [ -f "anat/${PREFIX}_space-DCEref_T1map.nii" ] #&& [ -f "reports/${PREFIX}_desc-casereport.html" ] + [ -f "dce/${PREFIX}_desc-AIF_T1map.nii.gz" ] && [ -f "anat/${PREFIX}_space-DCEref_T1map.nii.gz" ] #&& [ -f "reports/${PREFIX}_desc-casereport.html" ] then echo "Skipping ${source_dir} because it has already been processed." >> $LOG_FILE let successes++ From d3c8293bcb8bd1a39f22cfce718af386882ae470 Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Fri, 21 Aug 2026 19:03:35 -0700 Subject: [PATCH 05/29] Refactor DCE and VFA processing scripts to use updated mask naming conventions - Updated mask file naming from 'seg-WM_DCE.nii' to 'label-WM_DCE.nii' in DCE_norm.py. - Adjusted file patterns and processing logic in VFA_norm.py to accommodate new mask naming. - Modified case_report.py to reflect changes in mask file names and added a function for space-RAS path conversion. - Updated ktrans_analysis.py to use 'label' prefixes for Ktrans files. - Changed max_disp.py to dynamically display the total number of frames. - Revised preprocess_all.sh to use 'label' suffixes for AIF masks and updated related file paths. - Ensured consistency in mask handling across scripts, including brain and tissue masks. - Updated documentation to reflect changes in file naming conventions for FreeSurfer parcellation. --- DCE_all.sh | 47 ++++++++------ DCE_norm.py | 2 +- VFA_norm.py | 6 +- case_report.py | 72 +++++++++++++-------- docs/how-to/freesurfer-parcellation.md | 2 +- ktrans_analysis.py | 6 +- max_disp.py | 2 +- preprocess_all.sh | 89 +++++++++++++++----------- 8 files changed, 136 insertions(+), 90 deletions(-) diff --git a/DCE_all.sh b/DCE_all.sh index a502201..bf6a897 100755 --- a/DCE_all.sh +++ b/DCE_all.sh @@ -16,9 +16,10 @@ PURGE_INTERMEDIATES=0 GIGA_PURGE=0 TARGET_FLAG=0 SCRIPT_LOOP_DIR=dceprep/sub-*/ses-* +USE_PYTHON=0 # options -while getopts ":d:C:fhl:sST:" options; do +while getopts ":d:C:fhl:sST:p" options; do case "${options}" in b) EN_BIAS1=1 @@ -56,6 +57,7 @@ while getopts ":d:C:fhl:sST:" options; do echo "-d: specify main data directory containing all subject folders" echo "-f: enable Freesurfer wm parcellation for subregion analysis" echo "-h: display this message" + echo "-p: use Python for ROCKETSHIP calls" # echo "-l: specify a list of subjects to process (filename only, place in code folder)" echo "-s: skip subjects that have already been processed" echo "-S: enable smoothing of DCE input" @@ -65,6 +67,9 @@ while getopts ":d:C:fhl:sST:" options; do l) INPUT_LIST=$DATA_DIR/../code/${OPTARG} ;; + p) + USE_PYTHON=1 + ;; s) SKIP_IF_SUCCESS=1 ;; @@ -194,9 +199,9 @@ for der_dir in $SCRIPT_LOOP_DIR; do cd $der_dir || echo "ERROR: $der_dir does not exist. Preprocess first or check the name and try again." >> $LOG_FILE fi SUBJECT_TP_PATH=$(pwd) - if [ ! -f anat/${PREFIX}_desc-brain_T1w.nii.gz ] + if [ ! -f anat/${PREFIX}_label-brain_T1w.nii.gz ] then - cp -r $DERIV_DIR/dceprep-multihance_fix/$SUBJECT/$SESSION/anat/${PREFIX}_desc-brain_T1w.nii.gz $DERIV_DIR/dceprep-$OUTPUT_DIR/$SUBJECT/$SESSION/anat + cp -r $DERIV_DIR/dceprep-multihance_fix/$SUBJECT/$SESSION/anat/${PREFIX}_label-brain_T1w.nii.gz $DERIV_DIR/dceprep-$OUTPUT_DIR/$SUBJECT/$SESSION/anat fi # Locking mechanism to prevent concurrent processing from other machines LOCKFILE="lock.txt" @@ -249,8 +254,14 @@ for der_dir in $SCRIPT_LOOP_DIR; do # DCE # ------------------------------ - echo Begin DCE processing... - matlab -nodisplay -r "cd('$ROCKETSHIP_PATH'); addpath '$GPUFIT_PATH'; addpath '$GPUFIT_M_PATH'; run_dce_cli('$DATA_DIR/$SUBJECT/$SESSION/', '$SUBJECT_TP_PATH/'); exit;" + # echo Begin DCE processing... + rm -f dce/dce_patlak_fit_rois.xls + if [ $USE_PYTHON -eq 1 ] + then + ${ROCKETSHIP_PATH}/.venv/bin/python ${ROCKETSHIP_PATH}/run_dce_cli_python_case.py --subject-source $SUBJECT_TP_PATH --subject-tp $SUBJECT_TP_PATH --output-dir $SUBJECT_TP_PATH --events off + else + matlab -nodisplay -r "cd('$ROCKETSHIP_PATH'); addpath '$GPUFIT_PATH'; addpath '$GPUFIT_M_PATH'; run_dce_cli('$DATA_DIR/$SUBJECT/$SESSION/', '$SUBJECT_TP_PATH/'); exit;" + fi mv dce/dce_*_fit_Ktrans.nii dce/${PREFIX}_Ktrans.nii mv dce/dce_*_fit_ktrans_ci_low.nii dce/${PREFIX}_Ktrans_ci_low.nii mv dce/dce_*_fit_ktrans_ci_high.nii dce/${PREFIX}_Ktrans_ci_high.nii @@ -278,25 +289,25 @@ for der_dir in $SCRIPT_LOOP_DIR; do antsApplyTransforms -i anat/${PREFIX}_label-GM_mask.nii.gz -r $DCEREF_FILE -t anat/${PREFIX}_from-T1w_to-DCEref.mat -o anat/${PREFIX}_space-DCEref_label-GM_mask_pv.nii.gz &> /dev/null fslmaths anat/${PREFIX}_space-DCEref_label-GM_mask_pv.nii.gz -thr 0.9 -bin anat/${PREFIX}_space-DCEref_label-GM_mask.nii.gz rm anat/${PREFIX}_space-DCEref_label-GM_mask_pv.nii.gz - + # Align CSF mask antsApplyTransforms -i anat/${PREFIX}_label-CSF_mask.nii.gz -r $DCEREF_FILE -t anat/${PREFIX}_from-T1w_to-DCEref.mat -o anat/${PREFIX}_space-DCEref_label-CSF_mask_pv.nii.gz &> /dev/null fslmaths anat/${PREFIX}_space-DCEref_label-CSF_mask_pv.nii.gz -thr 0.9 -bin anat/${PREFIX}_space-DCEref_label-CSF_mask.nii.gz rm anat/${PREFIX}_space-DCEref_label-CSF_mask_pv.nii.gz - + # Apply masks to T1 map fslmaths anat/${PREFIX}_space-DCEref_T1map.nii -mas anat/${PREFIX}_space-DCEref_label-WM_mask.nii.gz anat/${PREFIX}_space-DCEref_label-WM_T1map.nii fslmaths anat/${PREFIX}_space-DCEref_T1map.nii -mas anat/${PREFIX}_space-DCEref_label-GM_mask.nii.gz anat/${PREFIX}_space-DCEref_label-GM_T1map.nii fslmaths anat/${PREFIX}_space-DCEref_T1map.nii -mas anat/${PREFIX}_space-DCEref_label-CSF_mask.nii.gz anat/${PREFIX}_space-DCEref_label-CSF_T1map.nii - + # Apply masks to Ktrans map - fslmaths dce/${PREFIX}_Ktrans.nii -mas anat/${PREFIX}_space-DCEref_label-WM_mask.nii.gz dce/${PREFIX}_seg-WM_Ktrans.nii - fslmaths dce/${PREFIX}_Ktrans.nii -mas anat/${PREFIX}_space-DCEref_label-GM_mask.nii.gz dce/${PREFIX}_seg-GM_Ktrans.nii - fslmaths dce/${PREFIX}_Ktrans.nii -mas anat/${PREFIX}_space-DCEref_label-CSF_mask.nii.gz dce/${PREFIX}_seg-CSF_Ktrans.nii - + fslmaths dce/${PREFIX}_Ktrans.nii -mas anat/${PREFIX}_space-DCEref_label-WM_mask.nii.gz dce/${PREFIX}_label-WM_Ktrans.nii + fslmaths dce/${PREFIX}_Ktrans.nii -mas anat/${PREFIX}_space-DCEref_label-GM_mask.nii.gz dce/${PREFIX}_label-GM_Ktrans.nii + fslmaths dce/${PREFIX}_Ktrans.nii -mas anat/${PREFIX}_space-DCEref_label-CSF_mask.nii.gz dce/${PREFIX}_label-CSF_Ktrans.nii + # registration QC python3 $SCRIPT_PATH/ktrans_analysis.py $dir $PREFIX - + fslmaths anat/${PREFIX}_space-DCEref_label-WM_mask.nii.gz -add 2000 huh.nii fslmaths huh.nii.gz -thr 2001 huh.nii fslmaths $DCEREF_FILE -sub huh.nii.gz bozo.nii @@ -307,7 +318,6 @@ for der_dir in $SCRIPT_LOOP_DIR; do fslmaths $DCEREF_FILE -sub huh2.nii.gz bozo2.nii fslmaths bozo2.nii -thr 0 anat/${PREFIX}_space-DCEref_label-GMQC.nii.gz rm huh.nii.gz huh2.nii.gz bozo.nii.gz bozo2.nii.gz - # wm parcellation with Freesurfer if [ $USE_FREESURFER -eq 1 ] then @@ -327,8 +337,8 @@ for der_dir in $SCRIPT_LOOP_DIR; do --o $DERIV_DIR/freesurfer/$SUBJECT/$SESSION/mri/wmparc-in-rawavg.mgz \ --regheader $DERIV_DIR/freesurfer/$SUBJECT/$SESSION/mri/wmparc.mgz &> /dev/null mri_convert $DERIV_DIR/freesurfer/$SUBJECT/$SESSION/mri/wmparc-in-rawavg.mgz wmparc.nii.gz &> /dev/null - antsApplyTransforms -i wmparc.nii.gz -r dce/${PREFIX}_desc-hmc_DCEref.nii.gz -t anat/${PREFIX}_from-t1w_to-DCEref.mat -n NearestNeighbor -o anat/${PREFIX}_space-DCEref_desc-wmparc.nii &> /dev/null - gzip -f anat/${PREFIX}_space-DCEref_desc-wmparc.nii &> /dev/null + antsApplyTransforms -i wmparc.nii.gz -r dce/${PREFIX}_desc-hmc_DCEref.nii.gz -t anat/${PREFIX}_from-t1w_to-DCEref.mat -n NearestNeighbor -o anat/${PREFIX}_space-DCEref_seg-wmparc_dseg.nii &> /dev/null + gzip -f anat/${PREFIX}_space-DCEref_seg-wmparc_dseg.nii &> /dev/null rm wmparc.nii.gz elif [ ! -f dce/${PREFIX}_space-MNI_Ktrans.nii.gz ] then @@ -339,8 +349,8 @@ for der_dir in $SCRIPT_LOOP_DIR; do # --winsorize-image-intensities [ 0.005,0.995 ] --transform Affine[ 0.1 ] \ # --metric MI[ T1_bet.nii.gz,DCE_mc.nii.gz,1,32,Regular,0.25 ] \ # --convergence [ 1000x500x250x100,1e-6,10 ] --shrink-factors 12x8x4x2 --smoothing-sigmas 4x3x2x1vox - # antsRegistrationSyNQuick.sh -d 3 -f anat/${PREFIX}_desc-brain_T1w.nii.gz -m dce/${PREFIX}_desc-hmc_DCEref.nii.gz -o dce/${PREFIX}_space-MNI_DCEref -n 8 - antsRegistrationSyNQuick.sh -d 3 -f $FSLDIR/data/standard/MNI152_T1_1mm_brain.nii.gz -m anat/${PREFIX}_desc-brain_T1w.nii.gz -o anat/${PREFIX}_space-MNI_T1w -n 8 + # antsRegistrationSyNQuick.sh -d 3 -f anat/${PREFIX}_label-brain_T1w.nii.gz -m dce/${PREFIX}_desc-hmc_DCEref.nii.gz -o dce/${PREFIX}_space-MNI_DCEref -n 8 + antsRegistrationSyNQuick.sh -d 3 -f $FSLDIR/data/standard/MNI152_T1_1mm_brain.nii.gz -m anat/${PREFIX}_label-brain_T1w.nii.gz -o anat/${PREFIX}_space-MNI_T1w -n 8 # antsApplyTransforms -i dce/${PREFIX}_desc-hmc_DCEref.nii.gz -r $FSLDIR/data/standard/MNI152_T1_1mm_brain.nii.gz -t dce/${PREFIX}_space-MNI_DCEref1Warp.nii.gz -t anat/${PREFIX}_space-MNI_T1w0GenericAffine.mat -t [ anat/${PREFIX}_from-T1w_to-DCEref.mat, 1] -o DCE_mc_MNI.nii.gz antsApplyTransforms -i dce/${PREFIX}_desc-hmc_DCEref.nii.gz -r $FSLDIR/data/standard/MNI152_T1_1mm_brain.nii.gz -t anat/${PREFIX}_space-MNI_T1w0GenericAffine.mat -t [ anat/${PREFIX}_from-T1w_to-DCEref.mat, 1] -o dce/${PREFIX}_space-MNI_DCEref.nii.gz # antsRegistration --verbose 1 --dimensionality 3 --float 0 --collapse-output-transforms 1 \ @@ -358,6 +368,7 @@ for der_dir in $SCRIPT_LOOP_DIR; do mkdir reports &> /dev/null python3 $SCRIPT_PATH/case_report.py $DATA_DIR/$SUBJECT/$SESSION $PREFIX $USE_FREESURFER python3 $SCRIPT_PATH/ktrans_report.py $DATA_DIR/$SUBJECT/$SESSION $PREFIX + if [ $PURGE_INTERMEDIATES -eq 1 ] then # rm -f !(Ktrans_*|T1_gm*|T1_wm*|T1_csf*|*_patlak_fit*.nii|case_report.html|*_MNI.nii.gz|*fit_VFA.nii|figures|dce*.png|*.log) diff --git a/DCE_norm.py b/DCE_norm.py index 163689e..14df0ae 100644 --- a/DCE_norm.py +++ b/DCE_norm.py @@ -202,7 +202,7 @@ def normalize(mri_file1, wm_masked, file_dir): # THE FUNCTION PERFORMING THE N if str(file).endswith('desc-bfc_DCE.nii') or str(file).endswith('desc-bfc_DCE.nii.gz'): file1 = str(file) mask_file = file1.split('desc-bfc_DCE', 1)[0] - mask_file = mask_file + 'seg-WM_DCE.nii' + mask_file = mask_file + 'label-WM_DCE.nii' if Path(mask_file + ".gz").exists(): mask_file += ".gz" diff --git a/VFA_norm.py b/VFA_norm.py index fab9b64..ef99b07 100644 --- a/VFA_norm.py +++ b/VFA_norm.py @@ -267,10 +267,10 @@ def double_gaussian(x, A1, mu1, sigma1, A2, mu2, sigma2): dir = Path(sys.argv[1]) # takes timepoint directory as argument prefix = sys.argv[2] BFC = sys.argv[3] - file_pattern = r'.*flip-\d+_space-DCEref_desc-bfc_VFA.nii.*' if BFC else r'.*flip-\d+_space-DCEref_desc-brain_VFA.nii.*' + file_pattern = r'.*flip-\d+_space-DCEref_desc-bfc_VFA.nii.*' if BFC else r'.*flip-\d+_space-DCEref_label-brain_VFA.nii.*' files_in_dir = dir.iterdir() - file_list = [file for file in files_in_dir if re.search(r'.*flip-\d+_space-DCEref_desc-brain_VFA.nii.*', str(file))] + file_list = [file for file in files_in_dir if re.search(r'.*flip-\d+_space-DCEref_label-brain_VFA.nii.*', str(file))] num_processes = len(file_list) with multiprocessing.Pool(processes=num_processes) as pool: - pool.starmap(normalize, [(str(file), str(file).split('_desc-brain', 1)[0] + '_seg-WM_VFA.nii.gz', str(dir)) for file in file_list]) + pool.starmap(normalize, [(str(file), str(file).split('_label-brain', 1)[0] + '_label-WM_VFA.nii.gz', str(dir)) for file in file_list]) diff --git a/case_report.py b/case_report.py index 62ef85f..6eb06d7 100644 --- a/case_report.py +++ b/case_report.py @@ -27,9 +27,32 @@ files_to_reorient = [f'anat/{prefix}_flip-01_space-DCEref_VFA.nii.gz', f'dce/{prefix}_Ktrans.nii', f'anat/{prefix}_space-DCEref_T1w.nii.gz', f'anat/{prefix}_space-DCEref_label-WM_mask.nii.gz', - f'anat/{prefix}_space-DCEref_T1map.nii', f'anat/{prefix}_space-DCEref_desc-brain_mask.nii.gz', - f'anat/{prefix}_space-DCEref_label-GM_mask.nii.gz', f'anat/{prefix}_space-DCEref_desc-wmparc.nii.gz', + f'anat/{prefix}_space-DCEref_T1map.nii', f'anat/{prefix}_space-DCEref_label-brain_mask.nii.gz', + f'anat/{prefix}_space-DCEref_label-GM_mask.nii.gz', f'anat/{prefix}_space-DCEref_seg-wmparc_dseg.nii.gz', first_existing_nifti_path(f'dce/{prefix}_desc-hmc_DCEref.nii.gz', f'dce/{prefix}_DCEref.nii.gz')] + + +def space_ras_path(path): + if path.endswith('.nii.gz'): + extension = '.nii.gz' + stem = path[:-7] + elif path.endswith('.nii'): + extension = '.nii' + stem = path[:-4] + else: + raise ValueError(f'Unsupported NIfTI path: {path}') + + directory, filename = os.path.split(stem) + if 'space-DCEref_' in filename: + filename = filename.replace('space-DCEref_', 'space-RAS_', 1) + else: + filename, replacements = re.subn(r'(ses-[^_]+_)', r'\1space-RAS_', filename, count=1) + if replacements == 0: + raise ValueError(f'Could not derive space-RAS path from {path}') + + return os.path.join(directory, filename + extension) + + # if c3d exists, reorient files to RAS dimensions = 0 voxel_size = 0 @@ -42,12 +65,12 @@ if not os.path.exists(file): print(f"File does not exist, skipping: {file}") continue - file_no_extension = nifti_stem(file) - command = ['c3d', file, '-orient', 'RAS', '-o', file_no_extension + '_RAS.nii.gz'] + reoriented_file = space_ras_path(file) + command = ['c3d', file, '-orient', 'RAS', '-o', reoriented_file] try: subprocess.run(command, check=True) if nifti_stem(file) == f'dce/{prefix}_Ktrans': - ktrans = nib.load(resolve_nifti_path(f'dce/{prefix}_Ktrans_RAS.nii.gz')) + ktrans = nib.load(resolve_nifti_path(space_ras_path(f'dce/{prefix}_Ktrans.nii'))) ktrans_data = ktrans.get_fdata() ktrans_flipped = np.flip(ktrans_data, axis=1) ktrans_flipped = nib.Nifti1Image(ktrans_flipped, ktrans.affine, ktrans.header) @@ -75,12 +98,12 @@ if not os.path.exists(file): print(f"File does not exist, skipping: {file}") continue - file_no_extension = nifti_stem(file) - command = ['mri_convert', '--in_orientation', 'LPI', file, file_no_extension + '_RAS.nii.gz'] + reoriented_file = space_ras_path(file) + command = ['mri_convert', '--in_orientation', 'LPI', file, reoriented_file] try: subprocess.run(command, check=True) if nifti_stem(file) == f'dce/{prefix}_Ktrans': - ktrans = nib.load(resolve_nifti_path(f'dce/{prefix}_Ktrans_RAS.nii.gz')) + ktrans = nib.load(resolve_nifti_path(space_ras_path(f'dce/{prefix}_Ktrans.nii'))) ktrans_data = ktrans.get_fdata() ktrans_flipped = np.flip(ktrans_data, axis=1) ktrans_flipped = nib.Nifti1Image(ktrans_flipped, ktrans.affine, ktrans.header) @@ -158,7 +181,7 @@ try: t1w_native = load_nifti_float32(f'{source_dir}/anat/{prefix}_T1w.nii.gz') - brain_mask_native = load_nifti_float32(f'anat/{prefix}_desc-brain_mask.nii.gz') + brain_mask_native = load_nifti_float32(f'anat/{prefix}_label-brain_mask.nii.gz') wm_mask_native = load_nifti_float32(f'anat/{prefix}_label-WM_mask.nii.gz') gm_mask_native = load_nifti_float32(f'anat/{prefix}_label-GM_mask.nii.gz') @@ -175,33 +198,33 @@ try: # T1w to VFA - # plotting.plot_anat(f'anat/{prefix}_label-WM_mask_RAS.nii.gz', cmap='gray', output_file='figures/t1w_to_dceref.svg', cut_coords=7, display_mode='z', annotate=False, colorbar=False, draw_cross=False, title='T1w to DCEref') + # plotting.plot_anat(space_ras_path(f'anat/{prefix}_label-WM_mask.nii.gz'), cmap='gray', output_file='figures/t1w_to_dceref.svg', cut_coords=7, display_mode='z', annotate=False, colorbar=False, draw_cross=False, title='T1w to DCEref') # T1w to dyn - t1w_dceref = nib.load(resolve_nifti_path(f'anat/{prefix}_space-DCEref_T1w_RAS.nii.gz')) + t1w_dceref = nib.load(resolve_nifti_path(space_ras_path(f'anat/{prefix}_space-DCEref_T1w.nii.gz'))) t1w_dceref_data = t1w_dceref.get_fdata() t1w_dceref_flipped = np.flip(t1w_dceref_data, axis=1) t1w_dceref_flipped = nib.Nifti1Image(t1w_dceref_flipped, t1w_dceref.affine, t1w_dceref.header) plotting.plot_anat(t1w_dceref_flipped, cmap='gray', output_file=f'figures/{prefix}_space-DCEref_T1w.svg', cut_coords=7, display_mode='z', annotate=False, colorbar=False, draw_cross=False, title='T1w to dyn') # flip T1w masks - t1w_mask = nib.load(resolve_nifti_path(f'anat/{prefix}_space-DCEref_desc-brain_mask_RAS.nii.gz')) + t1w_mask = nib.load(resolve_nifti_path(space_ras_path(f'anat/{prefix}_space-DCEref_label-brain_mask.nii.gz'))) t1w_mask_data = t1w_mask.get_fdata() t1w_mask_flipped = np.flip(t1w_mask_data, axis=1) t1w_mask_flipped = nib.Nifti1Image(t1w_mask_flipped, t1w_mask.affine, t1w_mask.header) - # nib.save(t1w_mask_flipped, str(tp_dir) + '/T1_bet_mask_RAS.nii') + # nib.save(t1w_mask_flipped, space_ras_path(str(tp_dir) + '/T1_bet_mask.nii')) - t1w_wm_mask = nib.load(resolve_nifti_path(f'anat/{prefix}_space-DCEref_label-WM_mask_RAS.nii.gz')) + t1w_wm_mask = nib.load(resolve_nifti_path(space_ras_path(f'anat/{prefix}_space-DCEref_label-WM_mask.nii.gz'))) t1w_wm_mask_data = t1w_wm_mask.get_fdata() t1w_wm_mask_flipped = np.flip(t1w_wm_mask_data, axis=1) t1w_wm_mask_flipped = nib.Nifti1Image(t1w_wm_mask_flipped, t1w_wm_mask.affine, t1w_wm_mask.header) - # nib.save(t1w_wm_mask_flipped, str(tp_dir) + '/T1_wm_mask_RAS.nii') + # nib.save(t1w_wm_mask_flipped, space_ras_path(str(tp_dir) + '/T1_wm_mask.nii')) - t1w_gm_mask = nib.load(resolve_nifti_path(f'anat/{prefix}_space-DCEref_label-GM_mask_RAS.nii.gz')) + t1w_gm_mask = nib.load(resolve_nifti_path(space_ras_path(f'anat/{prefix}_space-DCEref_label-GM_mask.nii.gz'))) t1w_gm_mask_data = t1w_gm_mask.get_fdata() t1w_gm_mask_flipped = np.flip(t1w_gm_mask_data, axis=1) t1w_gm_mask_flipped = nib.Nifti1Image(t1w_gm_mask_flipped, t1w_gm_mask.affine, t1w_gm_mask.header) - # nib.save(t1w_gm_mask_flipped, str(tp_dir) + '/T1_gm_mask_RAS.nii') + # nib.save(t1w_gm_mask_flipped, space_ras_path(str(tp_dir) + '/T1_gm_mask.nii')) plotting.plot_roi(t1w_mask_flipped, cmap='gray', bg_img=t1w_dceref_flipped, output_file='figures/t1bet_to_dyn.svg', display_mode='z', vmin=0, vmax=1, dim=0, annotate=True, colorbar=False, draw_cross=False, title='T1w brain mask to DCEref') plotting.plot_roi(t1w_wm_mask_flipped, cmap='gray', bg_img=t1w_dceref_flipped, output_file='figures/t1wm_to_dyn.svg', display_mode='z', vmin=0, vmax=1, dim=0, annotate=True, colorbar=False, draw_cross=False, title='T1w wm to DCEref') @@ -277,7 +300,7 @@ try: # T1 map # flip T1 map - img = nib.load(resolve_nifti_path(f'anat/{prefix}_space-DCEref_T1map_RAS.nii.gz')) + img = nib.load(resolve_nifti_path(space_ras_path(f'anat/{prefix}_space-DCEref_T1map.nii.gz'))) img_data = img.get_fdata() img_data = np.flip(img_data, axis=0) img_data = np.flip(img_data, axis=1) @@ -289,8 +312,7 @@ # try: # AIF -# plot graph of AIF region -aif = nib.load(resolve_nifti_path(f'dce/{prefix}_desc-AIFpos_T1map.nii.gz')) +aif = nib.load(resolve_nifti_path(f'dce/{prefix}_label-AIF_T1map.nii.gz')) aif_data = aif.get_fdata() # img = nib.load(str(tp_dir) + '/DCE_mc.nii.gz') try: @@ -353,15 +375,15 @@ if freesurfer: # wmparc overlay on DCE - wmparc = nib.load(resolve_nifti_path(f'anat/{prefix}_space-DCEref_desc-wmparc_RAS.nii.gz')) + wmparc = nib.load(resolve_nifti_path(space_ras_path(f'anat/{prefix}_space-DCEref_seg-wmparc_dseg.nii.gz'))) wmparc_data = wmparc.get_fdata() wmparc_flipped = np.flip(wmparc_data, axis=1) wmparc_flipped = nib.Nifti1Image(wmparc_flipped, wmparc.affine, wmparc.header) try: - dce = nib.load(resolve_nifti_path(f'dce/{prefix}_desc-hmc_DCEref_RAS.nii.gz')) + dce = nib.load(resolve_nifti_path(space_ras_path(f'dce/{prefix}_desc-hmc_DCEref.nii.gz'))) except FileNotFoundError: - dce = nib.load(resolve_nifti_path(f'dce/{prefix}_DCEref_RAS.nii.gz')) + dce = nib.load(resolve_nifti_path(space_ras_path(f'dce/{prefix}_DCEref.nii.gz'))) dce_data = dce.get_fdata() dce_flipped = np.flip(dce_data, axis=1) dce_flipped = nib.Nifti1Image(dce_flipped, dce.affine, dce.header) @@ -579,7 +601,7 @@ def extract_elapsed_time(log_text): # Ktrans # get Ktrans mean wm and gm -ktrans_wm = nib.load(resolve_nifti_path(f'dce/{prefix}_seg-WM_Ktrans.nii.gz')) +ktrans_wm = nib.load(resolve_nifti_path(f'dce/{prefix}_label-WM_Ktrans.nii.gz')) ktrans_wm_data = ktrans_wm.get_fdata() ktrans_wm_mask = nib.load(resolve_nifti_path(f'anat/{prefix}_space-DCEref_label-WM_mask.nii.gz')) ktrans_wm_mask_data = ktrans_wm_mask.get_fdata() @@ -587,7 +609,7 @@ def extract_elapsed_time(log_text): ktrans_median_wm = np.nanmedian(ktrans_wm_data[np.logical_and(ktrans_wm_mask_data > 0, ktrans_wm_data > KTRANS_MIN_THRESHOLD)])*1000 ktrans_std_wm = np.nanstd(ktrans_wm_data[np.logical_and(ktrans_wm_mask_data > 0, ktrans_wm_data > KTRANS_MIN_THRESHOLD)])*1000 -ktrans_gm = nib.load(resolve_nifti_path(f'dce/{prefix}_seg-GM_Ktrans.nii.gz')) +ktrans_gm = nib.load(resolve_nifti_path(f'dce/{prefix}_label-GM_Ktrans.nii.gz')) ktrans_gm_data = ktrans_gm.get_fdata() ktrans_gm_mask = nib.load(resolve_nifti_path(f'anat/{prefix}_space-DCEref_label-GM_mask.nii.gz')) ktrans_gm_mask_data = ktrans_gm_mask.get_fdata() diff --git a/docs/how-to/freesurfer-parcellation.md b/docs/how-to/freesurfer-parcellation.md index e956730..89b01d3 100644 --- a/docs/how-to/freesurfer-parcellation.md +++ b/docs/how-to/freesurfer-parcellation.md @@ -38,7 +38,7 @@ The registration pipeline: 1. `mri_label2vol` converts `wmparc.mgz` from FreeSurfer conformed space to the subject's native T1w space (`rawavg`) 2. `mri_convert` converts the result to NIfTI format 3. `antsApplyTransforms` applies the T1w → DCE rigid transform (with nearest-neighbor interpolation) to bring the parcellation into DCE space -4. The result is saved as `anat/sub-##_ses-##_space-DCEref_desc-wmparc.nii.gz` +4. The result is saved as `anat/sub-##_ses-##_space-DCEref_seg-wmparc_dseg.nii.gz` --- diff --git a/ktrans_analysis.py b/ktrans_analysis.py index e388181..cc9abe7 100644 --- a/ktrans_analysis.py +++ b/ktrans_analysis.py @@ -19,9 +19,9 @@ def analyze(tp_dir): files = ['anat/' + prefix + '_space-DCEref_label-WM_T1map.nii.gz', 'anat/' + prefix + '_space-DCEref_label-GM_T1map.nii.gz', 'anat/' + prefix + '_space-DCEref_label-CSF_T1map.nii.gz', - 'dce/' + prefix + '_seg-WM_Ktrans.nii.gz', - 'dce/' + prefix + '_seg-GM_Ktrans.nii.gz', - 'dce/' + prefix + '_seg-CSF_Ktrans.nii.gz', + 'dce/' + prefix + '_label-WM_Ktrans.nii.gz', + 'dce/' + prefix + '_label-GM_Ktrans.nii.gz', + 'dce/' + prefix + '_label-CSF_Ktrans.nii.gz', 'anat/' + prefix + '_space-DCEref_label-WM_mask.nii.gz', 'anat/' + prefix + '_space-DCEref_label-GM_mask.nii.gz'] # for i, file in enumerate(files): diff --git a/max_disp.py b/max_disp.py index c261928..6fe7be5 100644 --- a/max_disp.py +++ b/max_disp.py @@ -53,7 +53,7 @@ plt.xlabel("") plt.ylim([-2.5, 2.5]) plt.text(len(mc_params[:,0])/3, 2, "Max vector displacement: " + str(round(vector_max_disp,4)) + "mm") -plt.text(len(mc_params[:,0])/3, 1.8, "Frame: " + str(vector_max_disp_i) + "/64") +plt.text(len(mc_params[:,0])/3, 1.8, "Frame: " + str(vector_max_disp_i) + "/" + str(len(mc_params[:,0]))) # plt.text(len(mc_params[:,0])/2, 1.8, "param: " + param_type) path = dir + '/../figures/displacements.svg' plt.savefig(path, bbox_inches='tight') diff --git a/preprocess_all.sh b/preprocess_all.sh index 0a700ea..e3d8901 100755 --- a/preprocess_all.sh +++ b/preprocess_all.sh @@ -9,8 +9,9 @@ EN_BIAS2=0 EN_MOTION_CORR=0 T1_ONLY=0 USE_AUTO_AIF=0 -AIF_SUFFIX="desc-AIF_mask" -AIF_TRAINING_SUFFIX="desc-trainingAIF_mask" +USE_PYTHON=0 +AIF_SUFFIX="label-AIF_mask" +AIF_TRAINING_SUFFIX="label-AIF_desc-training_mask" SKIP_IF_SUCCESS=0 SCRIPT_LOOP_DIRS=sub-*/ses-* AUTOAIF_WEIGHT_PATH="docker/files/model_weight_huber1.h5" @@ -90,7 +91,7 @@ while getopts ":d:bBa:A:ZfhcC:mMsS:tl:T:w:" options; do h) echo "This script runs through all subject folders of a specified main data directory, preprocessing every folder ending in '_timepoint'." echo "The output is the DCE input, which are the corrected dynamic images, brain mask, T1 maps." - echo "-a: specify AIF suffix (default is 'desc-AIF_mask'). .nii.gz will be appended to the suffix." + echo "-a: specify AIF suffix (default is 'label-AIF_mask'). .nii.gz will be appended to the suffix." echo "-A: enable AutoAIF with argument A (All automatic), M (Manual if available), or T (Manual + Training if available)" echo "-b: enable first round of bias field corrections" echo "-B: enable second round of bias field corrections, post-Z-norm if enabled" @@ -103,6 +104,7 @@ while getopts ":d:bBa:A:ZfhcC:mMsS:tl:T:w:" options; do echo "-T [dir_path]: target the subject(s)/session(s) to run (default is 'sub-*/ses-*/')" echo "-t: only run up to T1 mapping" echo "-w [path]: specify the path to the AutoAIF weights file" + echo "-p: Use Python for ROCKETSHIP calls" echo "-Z: enable Z-slice normalization" exit 0 ;; @@ -115,6 +117,9 @@ while getopts ":d:bBa:A:ZfhcC:mMsS:tl:T:w:" options; do M) AUTOAIF_MODEL=${OPTARG} ;; + p) + USE_PYTHON=1 + ;; s) SKIP_IF_SUCCESS=1 ;; @@ -228,11 +233,11 @@ for source_dir in $DATA_DIR/$SCRIPT_LOOP_DIRS; do # If the session string is not empty, copy the masks if [ $mask_copied -eq 0 ] && [[ -n $session_str ]]; then echo "Copying masks for $SUBJECT $SESSION..." >> $LOG_FILE - cp /media/network_mriphysics/USC-PPG/AI_training/loos_model/test/masks/sub-${pat}_ses-${session_num}_desc-AIF_mask.nii.gz $DERIV_DIR/dceprep-$OUTPUT_DIR/$SUBJECT/$SESSION/dce/${SUBJECT}_${SESSION}_${AIF_TRAINING_SUFFIX}.nii.gz && mask_copied=1 + cp /media/network_mriphysics/USC-PPG/AI_training/loos_model/test/masks/sub-${pat}_ses-${session_num}_label-AIF_mask.nii.gz $DERIV_DIR/dceprep-$OUTPUT_DIR/$SUBJECT/$SESSION/dce/${SUBJECT}_${SESSION}_${AIF_TRAINING_SUFFIX}.nii.gz && mask_copied=1 cp /media/network_mriphysics/USC-PPG/AI_training/loos_model/test/masks/${pat}_${session_str}_timepoint.nii.gz $DERIV_DIR/dceprep-$OUTPUT_DIR/$SUBJECT/$SESSION/dce/${SUBJECT}_${SESSION}_${AIF_TRAINING_SUFFIX}.nii.gz && mask_copied=1 - cp /media/network_mriphysics/USC-PPG/AI_training/loos_model/train/masks/sub-${pat}_ses-${session_num}_desc-AIF_mask.nii.gz $DERIV_DIR/dceprep-$OUTPUT_DIR/$SUBJECT/$SESSION/dce/${SUBJECT}_${SESSION}_${AIF_TRAINING_SUFFIX}.nii.gz && mask_copied=1 + cp /media/network_mriphysics/USC-PPG/AI_training/loos_model/train/masks/sub-${pat}_ses-${session_num}_label-AIF_mask.nii.gz $DERIV_DIR/dceprep-$OUTPUT_DIR/$SUBJECT/$SESSION/dce/${SUBJECT}_${SESSION}_${AIF_TRAINING_SUFFIX}.nii.gz && mask_copied=1 cp /media/network_mriphysics/USC-PPG/AI_training/loos_model/train/masks/${pat}_${session_str}_timepoint.nii.gz $DERIV_DIR/dceprep-$OUTPUT_DIR/$SUBJECT/$SESSION/dce/${SUBJECT}_${SESSION}_${AIF_TRAINING_SUFFIX}.nii.gz && mask_copied=1 - cp /media/network_mriphysics/USC-PPG/AI_training/loos_model/val/masks/sub-${pat}_ses-${session_num}_desc-AIF_mask.nii.gz $DERIV_DIR/dceprep-$OUTPUT_DIR/$SUBJECT/$SESSION/dce/${SUBJECT}_${SESSION}_${AIF_TRAINING_SUFFIX}.nii.gz && mask_copied=1 + cp /media/network_mriphysics/USC-PPG/AI_training/loos_model/val/masks/sub-${pat}_ses-${session_num}_label-AIF_mask.nii.gz $DERIV_DIR/dceprep-$OUTPUT_DIR/$SUBJECT/$SESSION/dce/${SUBJECT}_${SESSION}_${AIF_TRAINING_SUFFIX}.nii.gz && mask_copied=1 cp /media/network_mriphysics/USC-PPG/AI_training/loos_model/val/masks/${pat}_${session_str}_timepoint.nii.gz $DERIV_DIR/dceprep-$OUTPUT_DIR/$SUBJECT/$SESSION/dce/${SUBJECT}_${SESSION}_${AIF_TRAINING_SUFFIX}.nii.gz && mask_copied=1 fi if [ $mask_copied -eq 0 ] && [ ! -f $DERIV_DIR/dceprep-$OUTPUT_DIR/$SUBJECT/$SESSION/dce/${PREFIX}_${AIF_SUFFIX}.nii.gz ] && [ ! -f $DERIV_DIR/dceprep-$OUTPUT_DIR/$SUBJECT/$SESSION/dce/${PREFIX}_${AIF_TRAINING_SUFFIX}.nii.gz ] && [ $USE_AUTO_AIF -eq 2 ] @@ -268,7 +273,7 @@ for source_dir in $DATA_DIR/$SCRIPT_LOOP_DIRS; do echo "$LOCKPATH" >> "$LOCKLIST" trap 'for f in $(cat "$LOCKLIST" 2>/dev/null); do rm -f "$f"; done; rm -f "$LOCKLIST"; exit $?' INT TERM EXIT else - echo "Skipping $dir because it is currently being processed by $(cat $LOCKPATH)." >> $LOG_FILE + echo "Skipping $source_dir because it is currently being processed by $(cat $LOCKPATH)." >> $LOG_FILE cd $DERIV_DIR continue fi @@ -282,8 +287,8 @@ for source_dir in $DATA_DIR/$SCRIPT_LOOP_DIRS; do if [ $SKIP_IF_SUCCESS -eq 1 ] then - if [ -f "dce/${PREFIX}_desc-bfcz_DCE.nii.gz" ] && [ -f "anat/${PREFIX}_space-DCEref_desc-brain_mask.nii.gz" ] && \ - [ -f "dce/${PREFIX}_desc-AIF_T1map.nii.gz" ] && [ -f "anat/${PREFIX}_space-DCEref_T1map.nii.gz" ] #&& [ -f "reports/${PREFIX}_desc-casereport.html" ] + if [ -f "dce/${PREFIX}_desc-bfcz_DCE.nii.gz" ] && [ -f "anat/${PREFIX}_space-DCEref_label-brain_mask.nii.gz" ] && \ + [ -f "dce/${PREFIX}_label-AIF_T1map.nii.gz" ] && [ -f "anat/${PREFIX}_space-DCEref_T1map.nii.gz" ] #&& [ -f "reports/${PREFIX}_desc-casereport.html" ] then echo "Skipping ${source_dir} because it has already been processed." >> $LOG_FILE let successes++ @@ -480,7 +485,7 @@ for source_dir in $DATA_DIR/$SCRIPT_LOOP_DIRS; do if [ ! -f anat/${PREFIX}_label-WM_mask.nii.gz ] then echo -ne "T1 SEG w/ FAST [=> ] $prog% ($current/$count) ~$mETA min remaining \r" - fast -t 1 -n 3 -H 0.1 -I 4 -l 20.0 -b --nopve -g -o anat/${PREFIX}_label- anat/${PREFIX}_desc-brain_T1w.nii.gz + fast -t 1 -n 3 -H 0.1 -I 4 -l 20.0 -b --nopve -g -o anat/${PREFIX}_label- anat/${PREFIX}_label-brain_T1w.nii.gz # rename segmented files mv anat/${PREFIX}_label-_bias.nii.gz anat/${PREFIX}_desc-bias_T1w.nii.gz mv anat/${PREFIX}_label-_seg_0.nii.gz anat/${PREFIX}_label-CSF_mask.nii.gz @@ -548,7 +553,7 @@ for source_dir in $DATA_DIR/$SCRIPT_LOOP_DIRS; do # apply MP-RAGE wm mask for VFA in "${VFA_LIST[@]}"; do # VFA_NUM=$(echo $VFA | grep -o '[0-9]*') - fslmaths anat/${PREFIX}_${VFA}_${REF_SPACE}_desc-brain_VFA.nii.gz -mas anat/${PREFIX}_${REF_SPACE}_label-WM_mask.nii.gz anat/${PREFIX}_${VFA}_${REF_SPACE}_seg-WM_VFA.nii.gz &> /dev/null + fslmaths anat/${PREFIX}_${VFA}_${REF_SPACE}_label-brain_VFA.nii.gz -mas anat/${PREFIX}_${REF_SPACE}_label-WM_mask.nii.gz anat/${PREFIX}_${VFA}_${REF_SPACE}_label-WM_VFA.nii.gz &> /dev/null done fi @@ -616,7 +621,7 @@ for source_dir in $DATA_DIR/$SCRIPT_LOOP_DIRS; do else echo Concatenating raw images for VFA in "${VFA_DYN_LIST[@]}"; do - VFA_INPUT+="anat/${PREFIX}_${VFA}_${REF_SPACE}_desc-brain_VFA.nii.gz " + VFA_INPUT+="anat/${PREFIX}_${VFA}_${REF_SPACE}_label-brain_VFA.nii.gz " done fslmerge -t anat/${PREFIX}_${REF_SPACE}_desc-unified_VFA.nii.gz $VFA_INPUT VFA_INPUT="desc-unified_VFA" @@ -671,9 +676,9 @@ for source_dir in $DATA_DIR/$SCRIPT_LOOP_DIRS; do # ----------------------------- prog=$(echo "scale=2; $prog + 2.77 / $count" | bc -l) echo -ne "REG BET MASK [========================> ] $prog% ($current/$count) ~$ETA min remaining \r" - antsApplyTransforms -i anat/${PREFIX}_desc-brain_mask.nii.gz -r $DCE_REF_VOL -t $structural_to_DCEref -o anat/${PREFIX}_${REF_SPACE}_desc-brain_mask_pv.nii.gz &> /dev/null - fslmaths anat/${PREFIX}_${REF_SPACE}_desc-brain_mask_pv.nii.gz -thr 1 -bin anat/${PREFIX}_${REF_SPACE}_desc-brain_mask.nii.gz &> /dev/null - rm anat/${PREFIX}_${REF_SPACE}_desc-brain_mask_pv.nii.gz + antsApplyTransforms -i anat/${PREFIX}_label-brain_mask.nii.gz -r $DCE_REF_VOL -t $structural_to_DCEref -o anat/${PREFIX}_${REF_SPACE}_label-brain_desc-pv_mask.nii.gz &> /dev/null + fslmaths anat/${PREFIX}_${REF_SPACE}_label-brain_desc-pv_mask.nii.gz -thr 1 -bin anat/${PREFIX}_${REF_SPACE}_label-brain_mask.nii.gz &> /dev/null + rm anat/${PREFIX}_${REF_SPACE}_label-brain_desc-pv_mask.nii.gz prog=$(echo "scale=2; $prog + 0.55 / $count" | bc -l) ETA=$(echo "scale=0; $mETA - ($SECONDS)/60" | bc -l) echo -ne "FAST DCE REP 1 [========================> ] $prog% ($current/$count) ~$ETA min remaining \r" @@ -689,22 +694,29 @@ for source_dir in $DATA_DIR/$SCRIPT_LOOP_DIRS; do --model_name $AUTOAIF_MODEL \ --save_image 1 &> /dev/null # rename output - mv dce/${PREFIX}_desc-hmc_DCE_float_mask.nii dce/${PREFIX}_desc-AIFfloat_mask.nii - mv dce/${PREFIX}_desc-hmc_DCE_mask.nii dce/${PREFIX}_desc-AIFtopvoxels_mask.nii - mv dce/${PREFIX}_desc-hmc_DCE_curve.svg figures/${PREFIX}_desc-AIF_resampledcurve.svg - mv dce/${PREFIX}_desc-hmc_DCE_mask.svg figures/${PREFIX}_desc-AIF_mask.svg + mv dce/${PREFIX}_desc-hmc_DCE_float_mask.nii dce/${PREFIX}_label-AIF_desc-float_mask.nii + mv dce/${PREFIX}_desc-hmc_DCE_mask.nii dce/${PREFIX}_label-AIF_desc-topvoxels_mask.nii + mv dce/${PREFIX}_desc-hmc_DCE_curve.svg figures/${PREFIX}_label-AIF_desc-resampled_mask.svg + mv dce/${PREFIX}_desc-hmc_DCE_mask.svg figures/${PREFIX}_label-AIF_mask.svg # fslmaths aif_floats.nii -thr 0.95 aif_mask.nii - fslmaths anat/${PREFIX}_${REF_SPACE}_T1map.nii.gz -mas dce/${PREFIX}_desc-AIFtopvoxels_mask.nii dce/${PREFIX}_desc-AIF_T1map.nii + fslmaths anat/${PREFIX}_${REF_SPACE}_T1map.nii.gz -mas dce/${PREFIX}_label-AIF_desc-topvoxels_mask.nii dce/${PREFIX}_label-AIF_T1map.nii else python3 $AUTO_AIF_PATH/main_vif.py --mode inference --input_path $source_dir/dce/${PREFIX}_DCE.nii.gz --save_output_path $PWD/dce \ --model_weight_path $SCRIPT_PATH/$AUTOAIF_WEIGHT_PATH \ --model_name $AUTOAIF_MODEL \ - --save_image 1 &> /dev/null - mv dce/${PREFIX}_DCE_float_mask.nii dce/${PREFIX}_AIFfloat_mask.nii - mv dce/${PREFIX}_DCE_mask.nii dce/${PREFIX}_AIFtopvoxels_mask.nii - mv dce/${PREFIX}_DCE_curve.svg figures/${PREFIX}_AIF_resampledcurve.svg - mv dce/${PREFIX}_DCE_mask.svg figures/${PREFIX}_AIF_mask.svg - fslmaths anat/${PREFIX}_${REF_SPACE}_T1map.nii.gz -mas dce/${PREFIX}_AIFtopvoxels_mask.nii dce/${PREFIX}_desc-AIF_T1map.nii + --save_image 1 &> dce/${PREFIX}_desc-autoaif.log + if [ ! -f "dce/${PREFIX}_DCE_float_mask.nii" ] || [ ! -f "dce/${PREFIX}_DCE_mask.nii" ]; then + echo "$source_dir AutoAIF failed. See dce/${PREFIX}_desc-autoaif.log. Skipping timepoint..." >> "$LOG_FILE" + cd "$DATA_DIR" + fail=1 + continue + fi + # rename output + mv dce/${PREFIX}_DCE_float_mask.nii dce/${PREFIX}_label-AIF_desc-float_mask.nii + mv dce/${PREFIX}_DCE_mask.nii dce/${PREFIX}_label-AIF_desc-topvoxels_mask.nii + mv dce/${PREFIX}_DCE_curve.svg figures/${PREFIX}_label-AIF_desc-resampled_mask.svg + mv dce/${PREFIX}_DCE_mask.svg figures/${PREFIX}_label-AIF_mask.svg + fslmaths anat/${PREFIX}_${REF_SPACE}_T1map.nii.gz -mas dce/${PREFIX}_label-AIF_desc-topvoxels_mask.nii dce/${PREFIX}_label-AIF_T1map.nii fi elif [ $USE_AUTO_AIF -eq 2 ] then @@ -712,29 +724,30 @@ for source_dir in $DATA_DIR/$SCRIPT_LOOP_DIRS; do if [ -f "dce/${PREFIX}_${AIF_SUFFIX}.nii.gz" ] then # use manual AIF - fslmaths anat/${PREFIX}_${REF_SPACE}_T1map.nii.gz -mas dce/${PREFIX}_${AIF_SUFFIX}.nii.gz dce/${PREFIX}_desc-AIF_T1map.nii.gz + fslmaths anat/${PREFIX}_${REF_SPACE}_T1map.nii.gz -mas dce/${PREFIX}_${AIF_SUFFIX}.nii.gz dce/${PREFIX}_label-AIF_T1map.nii.gz elif [ -f "dce/${PREFIX}_${AIF_TRAINING_SUFFIX}.nii.gz" ] then # use training manual AIF - fslmaths anat/${PREFIX}_${REF_SPACE}_T1map.nii.gz -mas dce/${PREFIX}_${AIF_TRAINING_SUFFIX}.nii.gz dce/${PREFIX}_desc-AIF_T1map.nii.gz + fslmaths anat/${PREFIX}_${REF_SPACE}_T1map.nii.gz -mas dce/${PREFIX}_${AIF_TRAINING_SUFFIX}.nii.gz dce/${PREFIX}_label-AIF_T1map.nii.gz fi else # use manual AIF - fslmaths anat/${PREFIX}_${REF_SPACE}_T1map.nii.gz -mas dce/${PREFIX}_${AIF_SUFFIX}.nii.gz dce/${PREFIX}_desc-AIF_T1map.nii.gz + fslmaths anat/${PREFIX}_${REF_SPACE}_T1map.nii.gz -mas dce/${PREFIX}_${AIF_SUFFIX}.nii.gz dce/${PREFIX}_label-AIF_T1map.nii.gz fi # ensure AIF is included in mask # fslcpgeom 2.nii T1_bet_mask_dyn.nii.gz - cp dce/${PREFIX}_desc-AIF_T1map.nii.gz dce/${PREFIX}_desc-AIFaligned_T1map.nii.gz - fslcpgeom anat/${PREFIX}_${REF_SPACE}_desc-brain_mask.nii.gz dce/${PREFIX}_desc-AIFaligned_T1map.nii - fslmaths dce/${PREFIX}_desc-AIFaligned_T1map.nii.gz -thr 0 dce/${PREFIX}_desc-AIFpos_T1map.nii &> /dev/null - rm dce/${PREFIX}_desc-AIFaligned_T1map.nii.gz - fslmaths anat/${PREFIX}_${REF_SPACE}_desc-brain_mask.nii.gz -add dce/${PREFIX}_desc-AIFpos_T1map.nii -thr 1 -bin anat/${PREFIX}_${REF_SPACE}_desc-brainAIF_mask.nii.gz &> /dev/null + cp dce/${PREFIX}_label-AIF_T1map.nii.gz dce/${PREFIX}_label-AIF_desc-aligned_T1map.nii.gz + fslcpgeom anat/${PREFIX}_${REF_SPACE}_label-brain_mask.nii.gz dce/${PREFIX}_label-AIF_desc-aligned_T1map.nii + fslmaths dce/${PREFIX}_label-AIF_desc-aligned_T1map.nii.gz -thr 0 dce/${PREFIX}_label-AIF_desc-pos_T1map.nii &> /dev/null + rm dce/${PREFIX}_label-AIF_desc-aligned_T1map.nii.gz + fslmaths anat/${PREFIX}_${REF_SPACE}_label-brain_mask.nii.gz -add dce/${PREFIX}_label-AIF_desc-pos_T1map.nii -thr 1 -bin anat/${PREFIX}_${REF_SPACE}_label-brainAIF_mask.nii.gz &> /dev/null + rm dce/${PREFIX}_label-AIF_desc-pos_T1map.nii # apply AIF mask to all DCE images if [ $EN_MOTION_CORR -eq 1 ] then - fslmaths dce/${PREFIX}_desc-hmc_DCE.nii.gz -mas anat/${PREFIX}_${REF_SPACE}_desc-brainAIF_mask.nii.gz dce/${PREFIX}_desc-AIFincluded_DCE.nii.gz &> /dev/null + fslmaths dce/${PREFIX}_desc-hmc_DCE.nii.gz -mas anat/${PREFIX}_${REF_SPACE}_label-brainAIF_mask.nii.gz dce/${PREFIX}_desc-AIFincluded_DCE.nii.gz &> /dev/null else - fslmaths $source_dir/dce/${PREFIX}_DCE.nii.gz -mas anat/${PREFIX}_${REF_SPACE}_desc-brainAIF_mask.nii.gz dce/${PREFIX}_desc-AIFincluded_DCE.nii.gz &> /dev/null + fslmaths $source_dir/dce/${PREFIX}_DCE.nii.gz -mas anat/${PREFIX}_${REF_SPACE}_label-brainAIF_mask.nii.gz dce/${PREFIX}_desc-AIFincluded_DCE.nii.gz &> /dev/null fi if [ $EN_BIAS1 -eq 1 ] then @@ -814,9 +827,9 @@ for source_dir in $DATA_DIR/$SCRIPT_LOOP_DIRS; do # apply wm mask to all DCE images if [ $EN_BIAS1 -eq 1 ] then - fslmaths dce/${PREFIX}_desc-bfc_DCE.nii.gz -mas anat/${PREFIX}_${VFA}_${REF_SPACE}_seg-WM_VFA.nii.gz dce/${PREFIX}_seg-WM_DCE.nii.gz &> /dev/null + fslmaths dce/${PREFIX}_desc-bfc_DCE.nii.gz -mas anat/${PREFIX}_${VFA}_${REF_SPACE}_label-WM_VFA.nii.gz dce/${PREFIX}_label-WM_DCE.nii.gz &> /dev/null else - fslmaths $source_dir/dce/${PREFIX}_DCE.nii.gz -mas anat/${PREFIX}_${REF_SPACE}_label-WM_mask.nii.gz dce/${PREFIX}_seg-WM_DCE.nii.gz &> /dev/null + fslmaths $source_dir/dce/${PREFIX}_DCE.nii.gz -mas anat/${PREFIX}_${REF_SPACE}_label-WM_mask.nii.gz dce/${PREFIX}_label-WM_DCE.nii.gz &> /dev/null fi # normalize dynamic images From 769459deb15df08459e1a60875f343a3737e2a93 Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Fri, 21 Aug 2026 19:06:45 -0700 Subject: [PATCH 06/29] Add support for Python execution in T1 mapping process --- preprocess_all.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/preprocess_all.sh b/preprocess_all.sh index e3d8901..f83855d 100755 --- a/preprocess_all.sh +++ b/preprocess_all.sh @@ -641,10 +641,15 @@ for source_dir in $DATA_DIR/$SCRIPT_LOOP_DIRS; do # T1 mapping where the input image is 'VFA.nii' # ------------------------------ - matlab -nodisplay -r "cd('$ROCKETSHIP_PATH/parametric_scripts/custom_scripts'); addpath '$ROCKETSHIP_PATH'; \ - addpath '$ROCKETSHIP_PATH/dce'; addpath '$ROCKETSHIP_PATH/external_programs'; \ - addpath '$ROCKETSHIP_PATH/external_programs/niftitools'; addpath '$ROCKETSHIP_PATH/parametric_scripts'; \ - addpath '$GPUFIT_PATH'; addpath '$GPUFIT_M_PATH'; T1mapping_fit('$source_dir/anat', '$SUBJECT_TP_PATH/anat', '${PREFIX}_${REF_SPACE}_${VFA_INPUT}.nii'); exit;" &> /dev/null + if [ $USE_PYTHON -eq 1 ] + then + ${ROCKETSHIP_PATH}/.venv/bin/python ${ROCKETSHIP_PATH}/run_parametric_python_case.py --subject-source $source_dir --subject-tp $SUBJECT_TP_PATH --output-dir $SUBJECT_TP_PATH --events off + else + matlab -nodisplay -r "cd('$ROCKETSHIP_PATH/parametric_scripts/custom_scripts'); addpath '$ROCKETSHIP_PATH'; \ + addpath '$ROCKETSHIP_PATH/dce'; addpath '$ROCKETSHIP_PATH/external_programs'; \ + addpath '$ROCKETSHIP_PATH/external_programs/niftitools'; addpath '$ROCKETSHIP_PATH/parametric_scripts'; \ + addpath '$GPUFIT_PATH'; addpath '$GPUFIT_M_PATH'; T1mapping_fit('$source_dir/anat', '$SUBJECT_TP_PATH/anat', '${PREFIX}_${REF_SPACE}_${VFA_INPUT}.nii'); exit;" + fi [ -f "anat/T1_map_t1_fa_fit_${PREFIX}_${REF_SPACE}_${VFA_INPUT}.nii" ] && mv anat/T1_map_t1_fa_fit_${PREFIX}_${REF_SPACE}_${VFA_INPUT}.nii anat/${PREFIX}_${REF_SPACE}_T1map.nii [ -f "anat/T1_map_t1_fa_fit_${PREFIX}_${REF_SPACE}_${VFA_INPUT}.mat" ] && mv anat/T1_map_t1_fa_fit_${PREFIX}_${REF_SPACE}_${VFA_INPUT}.mat anat/${PREFIX}_${REF_SPACE}_T1map.mat [ -f "anat/T1_map_t1_fa_fit_${PREFIX}_${REF_SPACE}_${VFA_INPUT}.txt" ] && mv anat/T1_map_t1_fa_fit_${PREFIX}_${REF_SPACE}_${VFA_INPUT}.txt anat/${PREFIX}_${REF_SPACE}_T1map.txt From 49a483bd198fec7fef170fd43b55d8d54df9e798 Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Fri, 21 Aug 2026 19:10:16 -0700 Subject: [PATCH 07/29] Add Python job to actions --- .github/workflows/DCE.yml | 75 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/.github/workflows/DCE.yml b/.github/workflows/DCE.yml index eab42ab..876ab93 100644 --- a/.github/workflows/DCE.yml +++ b/.github/workflows/DCE.yml @@ -10,7 +10,7 @@ permissions: contents: read jobs: - run: + run_case_matlab: runs-on: self-hosted @@ -36,8 +36,7 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip install --upgrade hd-bet # pip install flake8 pytest - - + # - name: Lint with flake8 # run: | # # stop the build if there are Python syntax errors or undefined names @@ -79,8 +78,76 @@ jobs: retention-days: 7 if-no-files-found: error + run_case_python: + + runs-on: self-hosted + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Checkout ROCKETSHIP repo + uses: actions/checkout@v6 + with: + repository: petmri/ROCKETSHIP + ref: dev + path: ROCKETSHIP + + #- name: Set up Python 3.10 + # uses: actions/setup-python@v3 + # with: + # python-version: "3.10" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip install --upgrade hd-bet + # pip install flake8 pytest + + # - name: Lint with flake8 + # run: | + # # stop the build if there are Python syntax errors or undefined names + # flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + #- name: Test with pytest + # run: | + # pytest + + #- name: Set up MATLAB + # uses: matlab-actions/setup-matlab@v1 + + - name: Preprocess the data + run: ./preprocess_all.sh -d /media/network_mriphysics/USC-PPG/RUNNER_DATA/sourcedata/raw -b -Z -m -c -p + + - name: Run DCE + run: ./DCE_all.sh -d /media/network_mriphysics/USC-PPG/RUNNER_DATA/sourcedata/raw -f -p + + - name: Output logs + run: | + date=$(date +"%Y-%m-%d") + cat /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/logs/preprocessing_log_${date}.txt + cat /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/logs/dce_log_${date}.txt + + - name: Convert HTML to PDF + run: wkhtmltopdf --enable-local-file-access /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.html /media/network_mriphysics/USC-PPG/RUNNER_DATA/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.pdf + continue-on-error: true + + - name: Upload analysis results + uses: actions/upload-artifact@v7 + with: + name: HTML_report + path: | + /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/dceprep/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.html + /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/dceprep/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.pdf + /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/dceprep/sub-1101743/ses-01/figures/ + retention-days: 7 + if-no-files-found: error + docker: - needs: run + needs: run_case_matlab strategy: fail-fast: false matrix: From 32e0495c69f2f07c1a677d02d7c3c89ac8b3233f Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Fri, 21 Aug 2026 19:35:05 -0700 Subject: [PATCH 08/29] add p to getopts list --- preprocess_all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/preprocess_all.sh b/preprocess_all.sh index f83855d..8c98cfd 100755 --- a/preprocess_all.sh +++ b/preprocess_all.sh @@ -29,7 +29,7 @@ prog=0 successes=0 # options -while getopts ":d:bBa:A:ZfhcC:mMsS:tl:T:w:" options; do +while getopts ":d:bBa:A:ZfhcC:mMsS:tl:T:w:p" options; do case "${options}" in a) AIF_SUFFIX=${OPTARG} From 3ee50b83e67d922440d1bbf2ac171d9fcffcfe6c Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Mon, 24 Aug 2026 13:10:58 -0700 Subject: [PATCH 09/29] genericizing a couple calls for env var compat, add ability to limit parallel jobs, new path search method, log autoAIF --- preprocess_all.sh | 89 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 77 insertions(+), 12 deletions(-) diff --git a/preprocess_all.sh b/preprocess_all.sh index 8c98cfd..e0e5de7 100755 --- a/preprocess_all.sh +++ b/preprocess_all.sh @@ -16,6 +16,9 @@ SKIP_IF_SUCCESS=0 SCRIPT_LOOP_DIRS=sub-*/ses-* AUTOAIF_WEIGHT_PATH="docker/files/model_weight_huber1.h5" AUTOAIF_MODEL="best" +HD_BET_COMMAND="${HD_BET_COMMAND:-hd-bet}" +AUTO_AIF_PYTHON="${AUTO_AIF_PYTHON:-python3}" +MAX_PARALLEL_JOBS=0 # internal vars (don't change) fail=0 @@ -29,7 +32,7 @@ prog=0 successes=0 # options -while getopts ":d:bBa:A:ZfhcC:mMsS:tl:T:w:p" options; do +while getopts ":d:bBa:A:ZfhcC:j:mMsS:tl:T:w:p" options; do case "${options}" in a) AIF_SUFFIX=${OPTARG} @@ -99,6 +102,7 @@ while getopts ":d:bBa:A:ZfhcC:mMsS:tl:T:w:p" options; do echo "-C [name]: enable comparison mode, which will output all files to the specified directory within each timepoint" echo "-d [dir_path]: specify BIDS compliant data directory containing all subject folders (sub-*/ses-*/anat|dce/*.nii|*.json)" echo "-h: display this message" + echo "-j [count]: limit concurrent VFA registration and FAST jobs (default is unlimited)" echo "-m: enable motion correction" echo "-s: skip preprocessing if DCE input file already exists" echo "-T [dir_path]: target the subject(s)/session(s) to run (default is 'sub-*/ses-*/')" @@ -111,6 +115,13 @@ while getopts ":d:bBa:A:ZfhcC:mMsS:tl:T:w:p" options; do l) INPUT_LIST=$DATA_DIR/../code/${OPTARG} ;; + j) + if [[ ! "$OPTARG" =~ ^[1-9][0-9]*$ ]]; then + echo "Invalid argument for -j. Use a positive integer." + exit 1 + fi + MAX_PARALLEL_JOBS=$OPTARG + ;; m) EN_MOTION_CORR=1 ;; @@ -148,16 +159,64 @@ if [ -z "$DATA_DIR" ] exit 1 fi +if ! "$HD_BET_COMMAND" --help &> /dev/null; then + echo "ERROR: HD-BET command '$HD_BET_COMMAND' is unavailable or cannot start. Set HD_BET_COMMAND to a working hd-bet executable." >&2 + exit 1 +fi + +resolve_tool_path() { + local configured_path=$1 + local marker_file=$2 + shift 2 + local candidate_path + + if [ -n "$configured_path" ]; then + if [ -f "$configured_path/$marker_file" ]; then + printf '%s\n' "$configured_path" + return 0 + fi + echo "ERROR: Configured path $configured_path does not contain $marker_file." >&2 + return 1 + fi + + for candidate_path in "$@"; do + if [ -f "$candidate_path/$marker_file" ]; then + printf '%s\n' "$candidate_path" + return 0 + fi + done + + find "$HOME" \ + \( -path "$HOME/.local/share/Trash" -o -path "$HOME/.local/share/Trash/*" -o -path "$HOME/.Trash" -o -path "$HOME/.Trash/*" \) -prune -o \ + -type f -name "$marker_file" -printf '%h\n' -quit 2> /dev/null +} + if [[ "$OSTYPE" == "linux-gnu" ]]; then - ROCKETSHIP_PATH=$(find $HOME -name '*run_dce_cli.m' -printf '%h\n' -quit || find / -name '*run_dce_cli.m' -printf '%h\n' -quit) &> /dev/null + ROCKETSHIP_PATH=$(resolve_tool_path "${ROCKETSHIP_PATH:-}" "run_dce_cli.m" "/opt/ROCKETSHIP/ROCKETSHIP-dev") SCRIPT_PATH=$(dirname "$(realpath $0)") - GPUFIT_PATH=$(find $HOME -name 'GpufitCudaAvailableMex.mexa64' -printf '%h\n' -quit || find / -name 'GpufitCudaAvailableMex.mexa64' -printf '%h\n' -quit) &> /dev/null - GPUFIT_M_PATH=$(find $HOME -name 'ModelID.m' -printf '%h\n' -quit || find / -name 'ModelID.m' -printf '%h\n' -quit) &> /dev/null + GPUFIT_PATH=$(resolve_tool_path "${GPUFIT_PATH:-}" "GpufitCudaAvailableMex.mexa64" "/opt/Gpufit/matlab64") + GPUFIT_M_PATH=$(resolve_tool_path "${GPUFIT_M_PATH:-}" "ModelID.m" "/opt/Gpufit/matlab") + if [ -z "$ROCKETSHIP_PATH" ] || [ -z "$GPUFIT_PATH" ] || [ -z "$GPUFIT_M_PATH" ]; then + echo "ERROR: Unable to locate ROCKETSHIP or GPUfit. Set ROCKETSHIP_PATH, GPUFIT_PATH, and GPUFIT_M_PATH to directories containing their required MATLAB files." >&2 + exit 1 + fi else ROCKETSHIP_PATH=$(find $HOME -type d -name ROCKETSHIP) SCRIPT_PATH=$(find $HOME -type d -name in-house_toolbox) GPUFIT_PATH=$(find $HOME -type d -name Gpufit-build) fi + +if [ $USE_AUTO_AIF -eq 1 ]; then + AUTO_AIF_PATH=$(resolve_tool_path "${AUTO_AIF_PATH:-}" "main_vif.py" "/opt/vascular_function") + if [ -z "$AUTO_AIF_PATH" ]; then + echo "ERROR: Unable to locate AutoAIF. Set AUTO_AIF_PATH to the directory containing main_vif.py." >&2 + exit 1 + fi + if ! "$AUTO_AIF_PYTHON" -c 'import tensorflow' &> /dev/null; then + echo "ERROR: AutoAIF Python '$AUTO_AIF_PYTHON' cannot import TensorFlow. Set AUTO_AIF_PYTHON to a TensorFlow-capable interpreter." >&2 + exit 1 + fi +fi cd $DATA_DIR || exit 1 # count timepoints @@ -337,7 +396,7 @@ for source_dir in $DATA_DIR/$SCRIPT_LOOP_DIRS; do SECONDS=0 echo -ne "HD-BET MP-RAGE [ ] $prog% ($current/$count) Calculating runtime... \r" - if [ ! -f "anat/${PREFIX}_desc-brain_mask.nii.gz" ] && [ -f "$source_dir/anat/${PREFIX}_T1w.nii.gz" ] + if [ ! -f "anat/${PREFIX}_label-brain_mask.nii.gz" ] && [ -f "$source_dir/anat/${PREFIX}_T1w.nii.gz" ] then if [ nvidia-smi ] then @@ -505,7 +564,7 @@ for source_dir in $DATA_DIR/$SCRIPT_LOOP_DIRS; do # VFA_NUM=$(echo $VFA | grep -o '[0-9]*') # FAST documentation recommends brain masking first # fslmaths $VFA -mas T1_bet_mask.nii.gz ${VFA_NUM}_masked.nii - cp anat/${PREFIX}_${VFA}_${REF_SPACE}_VFA.nii.gz anat/${PREFIX}_${VFA}_${REF_SPACE}_desc-brain_VFA.nii.gz + cp anat/${PREFIX}_${VFA}_${REF_SPACE}_VFA.nii.gz anat/${PREFIX}_${VFA}_${REF_SPACE}_label-brain_VFA.nii.gz done # gzip -f *_masked.nii @@ -514,15 +573,15 @@ for source_dir in $DATA_DIR/$SCRIPT_LOOP_DIRS; do VFA_FAST () { local VFA=$1 # VFA_NUM=$(echo $VFA | grep -o '[0-9]*') - fast -t 1 -n 3 -H 0.1 -I 4 -l 20.0 -B --nopve -o anat/${PREFIX}_${VFA}_${REF_SPACE}_desc-brain_VFA.nii.gz + fast -t 1 -n 3 -H 0.1 -I 4 -l 20.0 -B --nopve -o anat/${PREFIX}_${VFA}_${REF_SPACE}_label-brain_VFA.nii.gz # ETA=$(echo "scale=0; $mETA - ($SECONDS)/60" | bc -l) # prog=$(echo "scale=2; $prog + 6 / $count" | bc -l) # echo -ne "BFC FAST VFA${VFA_NUM} [=======> ] $prog% ($current/$count) ~$ETA min remaining \r" - mv anat/${PREFIX}_${VFA}_${REF_SPACE}_desc-brain_VFA_restore* anat/${PREFIX}_${VFA}_${REF_SPACE}_desc-bfc_VFA.nii.gz + mv anat/${PREFIX}_${VFA}_${REF_SPACE}_label-brain_VFA_restore* anat/${PREFIX}_${VFA}_${REF_SPACE}_desc-bfc_VFA.nii.gz # rm ${VFA}_masked_[mps]* # apply wm mask to all VFAs - fslmaths anat/${PREFIX}_${VFA}_${REF_SPACE}_desc-bfc_VFA.nii.gz -mas anat/${PREFIX}_${REF_SPACE}_label-WM_mask.nii.gz anat/${PREFIX}_${VFA}_${REF_SPACE}_seg-WM_VFA.nii.gz + fslmaths anat/${PREFIX}_${VFA}_${REF_SPACE}_desc-bfc_VFA.nii.gz -mas anat/${PREFIX}_${REF_SPACE}_label-WM_mask.nii.gz anat/${PREFIX}_${VFA}_${REF_SPACE}_label-WM_VFA.nii.gz } #echo "Bias field correction with FAST" @@ -694,10 +753,16 @@ for source_dir in $DATA_DIR/$SCRIPT_LOOP_DIRS; do # run AutoAIF if [ $EN_MOTION_CORR -eq 1 ] then - python3 $AUTO_AIF_PATH/main_vif.py --mode inference --input_path dce/${PREFIX}_desc-hmc_DCE.nii.gz --save_output_path $PWD/dce \ + "$AUTO_AIF_PYTHON" $AUTO_AIF_PATH/main_vif.py --mode inference --input_path dce/${PREFIX}_desc-hmc_DCE.nii.gz --save_output_path $PWD/dce \ --model_weight_path $SCRIPT_PATH/$AUTOAIF_WEIGHT_PATH \ --model_name $AUTOAIF_MODEL \ - --save_image 1 &> /dev/null + --save_image 1 &> dce/${PREFIX}_desc-autoaif.log + if [ ! -f "dce/${PREFIX}_desc-hmc_DCE_float_mask.nii" ] || [ ! -f "dce/${PREFIX}_desc-hmc_DCE_mask.nii" ]; then + echo "$source_dir AutoAIF failed. See dce/${PREFIX}_desc-autoaif.log. Skipping timepoint..." >> "$LOG_FILE" + cd "$DATA_DIR" + fail=1 + continue + fi # rename output mv dce/${PREFIX}_desc-hmc_DCE_float_mask.nii dce/${PREFIX}_label-AIF_desc-float_mask.nii mv dce/${PREFIX}_desc-hmc_DCE_mask.nii dce/${PREFIX}_label-AIF_desc-topvoxels_mask.nii @@ -706,7 +771,7 @@ for source_dir in $DATA_DIR/$SCRIPT_LOOP_DIRS; do # fslmaths aif_floats.nii -thr 0.95 aif_mask.nii fslmaths anat/${PREFIX}_${REF_SPACE}_T1map.nii.gz -mas dce/${PREFIX}_label-AIF_desc-topvoxels_mask.nii dce/${PREFIX}_label-AIF_T1map.nii else - python3 $AUTO_AIF_PATH/main_vif.py --mode inference --input_path $source_dir/dce/${PREFIX}_DCE.nii.gz --save_output_path $PWD/dce \ + "$AUTO_AIF_PYTHON" $AUTO_AIF_PATH/main_vif.py --mode inference --input_path $source_dir/dce/${PREFIX}_DCE.nii.gz --save_output_path $PWD/dce \ --model_weight_path $SCRIPT_PATH/$AUTOAIF_WEIGHT_PATH \ --model_name $AUTOAIF_MODEL \ --save_image 1 &> dce/${PREFIX}_desc-autoaif.log From 97ebacf09a0f3e3d350bf119a4ab437165da3b3f Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Mon, 24 Aug 2026 13:50:42 -0700 Subject: [PATCH 10/29] idk how autoAIF wasn't enabled in runner --- .github/workflows/DCE.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/DCE.yml b/.github/workflows/DCE.yml index 876ab93..6000476 100644 --- a/.github/workflows/DCE.yml +++ b/.github/workflows/DCE.yml @@ -52,7 +52,7 @@ jobs: # uses: matlab-actions/setup-matlab@v1 - name: Preprocess the data - run: ./preprocess_all.sh -d /media/network_mriphysics/USC-PPG/RUNNER_DATA/sourcedata/raw -b -Z -m -c + run: ./preprocess_all.sh -d /media/network_mriphysics/USC-PPG/RUNNER_DATA/sourcedata/raw -b -Z -m -c -A A - name: Run DCE run: ./DCE_all.sh -d /media/network_mriphysics/USC-PPG/RUNNER_DATA/sourcedata/raw -f @@ -120,7 +120,7 @@ jobs: # uses: matlab-actions/setup-matlab@v1 - name: Preprocess the data - run: ./preprocess_all.sh -d /media/network_mriphysics/USC-PPG/RUNNER_DATA/sourcedata/raw -b -Z -m -c -p + run: ./preprocess_all.sh -d /media/network_mriphysics/USC-PPG/RUNNER_DATA/sourcedata/raw -b -Z -m -c -p -A A - name: Run DCE run: ./DCE_all.sh -d /media/network_mriphysics/USC-PPG/RUNNER_DATA/sourcedata/raw -f -p From df27f4695b3c55aebd51f4383d76468cc37153e9 Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Mon, 24 Aug 2026 14:40:35 -0700 Subject: [PATCH 11/29] update checkout actions --- .github/workflows/DCE.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/DCE.yml b/.github/workflows/DCE.yml index 6000476..9df2062 100644 --- a/.github/workflows/DCE.yml +++ b/.github/workflows/DCE.yml @@ -16,10 +16,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Checkout ROCKETSHIP repo - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: petmri/ROCKETSHIP ref: dev @@ -84,10 +84,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Checkout ROCKETSHIP repo - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: petmri/ROCKETSHIP ref: dev @@ -156,7 +156,7 @@ jobs: runs-on: self-hosted steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Extract branch name id: extract_branch From f82bae69366a6b5ff17b26bedea8fb3cf6056a7b Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Mon, 24 Aug 2026 18:57:11 -0700 Subject: [PATCH 12/29] stupid runner not updating repo --- .github/workflows/DCE.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/DCE.yml b/.github/workflows/DCE.yml index 9df2062..71bf1b4 100644 --- a/.github/workflows/DCE.yml +++ b/.github/workflows/DCE.yml @@ -17,6 +17,9 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 + with: + clean: true + fetch-depth: 0 - name: Checkout ROCKETSHIP repo uses: actions/checkout@v7 @@ -24,6 +27,8 @@ jobs: repository: petmri/ROCKETSHIP ref: dev path: ROCKETSHIP + clean: true + fetch-depth: 0 #- name: Set up Python 3.10 # uses: actions/setup-python@v3 From 8baf94a9db20aaf3332881830b89b314be1bd105 Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Mon, 24 Aug 2026 19:06:04 -0700 Subject: [PATCH 13/29] surely this is the last line i need --- .github/workflows/DCE.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/DCE.yml b/.github/workflows/DCE.yml index 71bf1b4..ce499d0 100644 --- a/.github/workflows/DCE.yml +++ b/.github/workflows/DCE.yml @@ -18,6 +18,7 @@ jobs: - name: Checkout uses: actions/checkout@v7 with: + ref: ${{ github.ref_name }} clean: true fetch-depth: 0 From af3f0516d531a59875710dd5b3355bf20ec82836 Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Mon, 24 Aug 2026 19:06:55 -0700 Subject: [PATCH 14/29] forgot it for python run LUL --- .github/workflows/DCE.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/DCE.yml b/.github/workflows/DCE.yml index ce499d0..b4da801 100644 --- a/.github/workflows/DCE.yml +++ b/.github/workflows/DCE.yml @@ -91,6 +91,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 + with: + ref: ${{ github.ref_name }} + clean: true + fetch-depth: 0 - name: Checkout ROCKETSHIP repo uses: actions/checkout@v7 From 2611b69ee7f7282c8fde65a916a6fd15dfbc6388 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 25 Aug 2026 17:57:45 +0000 Subject: [PATCH 15/29] Fix PR checkout ref handling Co-authored-by: lsaca05 <46550669+lsaca05@users.noreply.github.com> --- .github/workflows/DCE.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/DCE.yml b/.github/workflows/DCE.yml index b4da801..e7d7bb1 100644 --- a/.github/workflows/DCE.yml +++ b/.github/workflows/DCE.yml @@ -17,11 +17,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 - with: - ref: ${{ github.ref_name }} - clean: true - fetch-depth: 0 - - name: Checkout ROCKETSHIP repo uses: actions/checkout@v7 with: @@ -91,11 +86,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 - with: - ref: ${{ github.ref_name }} - clean: true - fetch-depth: 0 - - name: Checkout ROCKETSHIP repo uses: actions/checkout@v7 with: From 36178cc68af6460bf7bc339dd7546db02047dec5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:16:07 +0000 Subject: [PATCH 16/29] Run workflow on all branches Co-authored-by: lsaca05 <46550669+lsaca05@users.noreply.github.com> --- .github/workflows/DCE.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/DCE.yml b/.github/workflows/DCE.yml index e7d7bb1..c96e68e 100644 --- a/.github/workflows/DCE.yml +++ b/.github/workflows/DCE.yml @@ -2,9 +2,7 @@ name: Run DCE then build docker on: push: - branches: [ "dev", "main" ] pull_request: - branches: [ "dev", "main" ] permissions: contents: read @@ -12,6 +10,7 @@ permissions: jobs: run_case_matlab: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: self-hosted steps: @@ -81,6 +80,7 @@ jobs: run_case_python: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: self-hosted steps: From 59d249167d2c5eac264ed492696499c3d15966ee Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Tue, 25 Aug 2026 11:25:14 -0700 Subject: [PATCH 17/29] another attempt at getting PRs to pull the right thing --- .github/workflows/DCE.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/DCE.yml b/.github/workflows/DCE.yml index c96e68e..3b378d4 100644 --- a/.github/workflows/DCE.yml +++ b/.github/workflows/DCE.yml @@ -16,6 +16,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + - name: Checkout ROCKETSHIP repo uses: actions/checkout@v7 with: @@ -86,6 +91,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + - name: Checkout ROCKETSHIP repo uses: actions/checkout@v7 with: From 4d8ad7697e5a6a0344586c923b6a115607f6fdce Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Tue, 25 Aug 2026 11:48:04 -0700 Subject: [PATCH 18/29] turns out me just dumb --- preprocess_all.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/preprocess_all.sh b/preprocess_all.sh index e0e5de7..9edf313 100755 --- a/preprocess_all.sh +++ b/preprocess_all.sh @@ -398,16 +398,22 @@ for source_dir in $DATA_DIR/$SCRIPT_LOOP_DIRS; do if [ ! -f "anat/${PREFIX}_label-brain_mask.nii.gz" ] && [ -f "$source_dir/anat/${PREFIX}_T1w.nii.gz" ] then - if [ nvidia-smi ] + if command -v nvidia-smi &> /dev/null && nvidia-smi &> /dev/null then - hd-bet -i $source_dir/anat/${PREFIX}_T1w.nii.gz -o anat/${PREFIX}_desc-brain.nii.gz --save_bet_mask &> /dev/null + "$HD_BET_COMMAND" -i $source_dir/anat/${PREFIX}_T1w.nii.gz -o anat/${PREFIX}_label-brain.nii.gz --save_bet_mask &> /dev/null mETA=$(echo "scale=0; $SECONDS * 34 * ($count - $current + 1) / 60" | bc -l) else - hd-bet -i $source_dir/anat/${PREFIX}_T1w.nii.gz -o anat/${PREFIX}_desc-brain.nii.gz -device cpu --save_bet_mask &> /dev/null + "$HD_BET_COMMAND" -i $source_dir/anat/${PREFIX}_T1w.nii.gz -o anat/${PREFIX}_label-brain.nii.gz -device cpu --save_bet_mask &> /dev/null mETA=$(echo "scale=0; $SECONDS * 2 * ($count - $current + 1) / 60" | bc -l) fi - mv anat/${PREFIX}_desc-brain_bet.nii.gz anat/${PREFIX}_desc-brain_mask.nii.gz - mv anat/${PREFIX}_desc-brain.nii.gz anat/${PREFIX}_desc-brain_T1w.nii.gz + if [ ! -f "anat/${PREFIX}_label-brain_bet.nii.gz" ] || [ ! -f "anat/${PREFIX}_label-brain.nii.gz" ]; then + echo "$source_dir HD-BET did not create expected output files. Skipping timepoint..." >> "$LOG_FILE" + cd "$DATA_DIR" + fail=1 + continue + fi + mv anat/${PREFIX}_label-brain_bet.nii.gz anat/${PREFIX}_label-brain_mask.nii.gz + mv anat/${PREFIX}_label-brain.nii.gz anat/${PREFIX}_label-brain_T1w.nii.gz elif [ -f "$source_dir/anat/${PREFIX}_T2w.nii.gz" ] then # assume mouse From 60450cf45383b75a86582854e17fec204347cf71 Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Tue, 25 Aug 2026 12:43:06 -0700 Subject: [PATCH 19/29] revert checkout specifying, only do docker on the core branches --- .github/workflows/DCE.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/DCE.yml b/.github/workflows/DCE.yml index 3b378d4..2dd2bdc 100644 --- a/.github/workflows/DCE.yml +++ b/.github/workflows/DCE.yml @@ -16,10 +16,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 - with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - name: Checkout ROCKETSHIP repo uses: actions/checkout@v7 @@ -91,10 +87,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 - with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - name: Checkout ROCKETSHIP repo uses: actions/checkout@v7 @@ -158,6 +150,7 @@ jobs: docker: needs: run_case_matlab + if: github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main' strategy: fail-fast: false matrix: From ccf00edd8d1f8c585f0d606142c30ad2a6aa7754 Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Tue, 25 Aug 2026 13:22:01 -0700 Subject: [PATCH 20/29] fail actions if any result files are missing --- .github/workflows/DCE.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/DCE.yml b/.github/workflows/DCE.yml index 2dd2bdc..8b4b929 100644 --- a/.github/workflows/DCE.yml +++ b/.github/workflows/DCE.yml @@ -68,6 +68,12 @@ jobs: run: wkhtmltopdf --enable-local-file-access /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.html /media/network_mriphysics/USC-PPG/RUNNER_DATA/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.pdf continue-on-error: true + - name: Validate analysis results + run: | + test -f /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/dceprep/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.html + test -f /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/dceprep/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.pdf + test -n "$(find /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/dceprep/sub-1101743/ses-01/figures/ -type f -print -quit)" + - name: Upload analysis results uses: actions/upload-artifact@v7 with: @@ -137,6 +143,12 @@ jobs: run: wkhtmltopdf --enable-local-file-access /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.html /media/network_mriphysics/USC-PPG/RUNNER_DATA/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.pdf continue-on-error: true + - name: Validate analysis results + run: | + test -f /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/dceprep/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.html + test -f /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/dceprep/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.pdf + test -n "$(find /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/dceprep/sub-1101743/ses-01/figures/ -type f -print -quit)" + - name: Upload analysis results uses: actions/upload-artifact@v7 with: From 73a4c94c64cfd3705ca792afc5f4fd4e515b08f4 Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Tue, 25 Aug 2026 14:09:25 -0700 Subject: [PATCH 21/29] use new env for to specify workspace ROCKETSHIP --- .github/workflows/DCE.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/DCE.yml b/.github/workflows/DCE.yml index 8b4b929..3ccdb53 100644 --- a/.github/workflows/DCE.yml +++ b/.github/workflows/DCE.yml @@ -12,6 +12,8 @@ jobs: if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: self-hosted + env: + ROCKETSHIP_PATH: ${{ github.workspace }}/ROCKETSHIP steps: - name: Checkout @@ -89,6 +91,8 @@ jobs: if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: self-hosted + env: + ROCKETSHIP_PATH: ${{ github.workspace }}/ROCKETSHIP steps: - name: Checkout From e12c6f8c28f0942b2f10603fa34d2a58ebd31be0 Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Tue, 25 Aug 2026 14:52:08 -0700 Subject: [PATCH 22/29] Refactor DCEPrep integration scripts to use environment variables for ROCKETSHIP and GPUFIT paths, streamline preprocessing, and update documentation for clarity. --- .github/workflows/DCE.yml | 48 +++++---------------------- DCE_all.sh | 11 +++--- docs/integration/gpufit.md | 17 ++++------ docs/integration/rocketship.md | 6 ++-- docs/integration/vascular-function.md | 15 +++++++-- docs/reference/preprocessing-cli.md | 8 +++++ docs/user-guide/outputs.md | 2 +- preprocess_all.sh | 1 - 8 files changed, 45 insertions(+), 63 deletions(-) diff --git a/.github/workflows/DCE.yml b/.github/workflows/DCE.yml index 3ccdb53..5571bab 100644 --- a/.github/workflows/DCE.yml +++ b/.github/workflows/DCE.yml @@ -28,32 +28,16 @@ jobs: clean: true fetch-depth: 0 - #- name: Set up Python 3.10 - # uses: actions/setup-python@v3 - # with: - # python-version: "3.10" + - name: Set up ROCKETSHIP venv + working-directory: ROCKETSHIP + run: python3 install.py -G -M - name: Install dependencies run: | python -m pip install --upgrade pip if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip install --upgrade hd-bet - # pip install flake8 pytest - - # - name: Lint with flake8 - # run: | - # # stop the build if there are Python syntax errors or undefined names - # flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - #- name: Test with pytest - # run: | - # pytest - - #- name: Set up MATLAB - # uses: matlab-actions/setup-matlab@v1 - + - name: Preprocess the data run: ./preprocess_all.sh -d /media/network_mriphysics/USC-PPG/RUNNER_DATA/sourcedata/raw -b -Z -m -c -A A @@ -105,32 +89,16 @@ jobs: ref: dev path: ROCKETSHIP - #- name: Set up Python 3.10 - # uses: actions/setup-python@v3 - # with: - # python-version: "3.10" + - name: Set up ROCKETSHIP venv + working-directory: ROCKETSHIP + run: python3 install.py -G -M - name: Install dependencies run: | python -m pip install --upgrade pip if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip install --upgrade hd-bet - # pip install flake8 pytest - - # - name: Lint with flake8 - # run: | - # # stop the build if there are Python syntax errors or undefined names - # flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - #- name: Test with pytest - # run: | - # pytest - - #- name: Set up MATLAB - # uses: matlab-actions/setup-matlab@v1 - + - name: Preprocess the data run: ./preprocess_all.sh -d /media/network_mriphysics/USC-PPG/RUNNER_DATA/sourcedata/raw -b -Z -m -c -p -A A diff --git a/DCE_all.sh b/DCE_all.sh index bf6a897..2e73045 100755 --- a/DCE_all.sh +++ b/DCE_all.sh @@ -99,10 +99,11 @@ if [ -z "$DATA_DIR" ] exit 1 fi if [[ "$OSTYPE" == "linux-gnu" ]]; then - ROCKETSHIP_PATH=$(find $HOME -name '*run_dce_cli.m' -printf '%h\n' -quit || find / -name '*run_dce_cli.m' -printf '%h\n' -quit) &> /dev/null + # respect a pre-set path (e.g. from CI) instead of searching the whole filesystem + ROCKETSHIP_PATH=${ROCKETSHIP_PATH:-$(find $HOME -name '*run_dce_cli.m' -printf '%h\n' -quit || find / -name '*run_dce_cli.m' -printf '%h\n' -quit) 2> /dev/null} SCRIPT_PATH=$(dirname "$(realpath $0)") - GPUFIT_PATH=$(find $HOME -name 'GpufitConstrainedMex.mexa64' -printf '%h\n' -quit || find / -name 'GpufitConstrainedMex.mexa64' -printf '%h\n' -quit) - GPUFIT_M_PATH=$(find $HOME -name 'ModelID.m' -printf '%h\n' -quit || find / -name 'ModelID.m' -printf '%h\n' -quit) + GPUFIT_PATH=${GPUFIT_PATH:-$(find $HOME -name 'GpufitConstrainedMex.mexa64' -printf '%h\n' -quit || find / -name 'GpufitConstrainedMex.mexa64' -printf '%h\n' -quit)} + GPUFIT_M_PATH=${GPUFIT_M_PATH:-$(find $HOME -name 'ModelID.m' -printf '%h\n' -quit || find / -name 'ModelID.m' -printf '%h\n' -quit)} else ROCKETSHIP_PATH=$(find $HOME -type d -name ROCKETSHIP) SCRIPT_PATH=$(find $HOME -type d -name in-house_toolbox) @@ -223,14 +224,14 @@ for der_dir in $SCRIPT_LOOP_DIR; do fi if [ $SKIP_IF_SUCCESS -eq 1 ] then - if [ -f "reports/${PREFIX}_desc-casereport.html" ] && [ -f "anat/${PREFIX}_space-DCEref_desc-wmparc.nii.gz" ] + if [ -f "reports/${PREFIX}_desc-casereport.html" ] && [ -f "anat/${PREFIX}_space-DCEref_seg-wmparc_dseg.nii.gz" ] then echo "Skipping $dir because it has already been processed." >> $LOG_FILE ((successes++)) if [ $PURGE_INTERMEDIATES -eq 1 ] # && [ $COMPARISON_MODE -eq 1 ] then cd anat - rm -f !(${PREFIX}*bfczunified_VFA.nii|${PREFIX}_space-DCEref_T1map*|${PREFIX}*label-*_T1map*|*.mat|*wmparc.nii.gz|*space-DCEref_desc-brain_mask.nii.gz) + rm -f !(${PREFIX}*bfczunified_VFA.nii|${PREFIX}_space-DCEref_T1map*|${PREFIX}*label-*_T1map*|*.mat|*wmparc.nii.gz|*space-DCEref_label-brain_mask.nii.gz) cd ../dce rm -f !(${PREFIX}*Ktrans*|${PREFIX}*bfcz_DCE*|${PREFIX}*AIFincluded*|${PREFIX}*AIF_T1map*|figures|*.log|*.txt|*.par) fi diff --git a/docs/integration/gpufit.md b/docs/integration/gpufit.md index 4623ee5..e28c46a 100644 --- a/docs/integration/gpufit.md +++ b/docs/integration/gpufit.md @@ -34,19 +34,14 @@ GPUfit (petmri fork) includes GPU-optimised implementations of: ## How DCEPrep Locates GPUfit -DCEPrep searches for GPUfit's MEX files and MATLAB model definitions at startup: +On Linux, `preprocess_all.sh` first uses `GPUFIT_PATH` and `GPUFIT_M_PATH` when +set, then the Docker locations `/opt/Gpufit/matlab64` and `/opt/Gpufit/matlab`, +and finally searches `$HOME` for the required files. Set both variables for a +nonstandard installation: ```bash -# T1 mapping MEX (preprocess_all.sh) -GPUFIT_PATH=$(find $HOME -name 'GpufitCudaAvailableMex.mexa64' -printf '%h\n' -quit \ - || find / -name 'GpufitCudaAvailableMex.mexa64' -printf '%h\n' -quit) - -# DCE analysis MEX (DCE_all.sh) -GPUFIT_PATH=$(find $HOME -name 'GpufitConstrainedMex.mexa64' -printf '%h\n' -quit \ - || find / -name 'GpufitConstrainedMex.mexa64' -printf '%h\n' -quit) - -# MATLAB model ID definitions (both scripts) -GPUFIT_M_PATH=$(find $HOME -name 'ModelID.m' -printf '%h\n' -quit) +export GPUFIT_PATH=/path/to/Gpufit/matlab64 +export GPUFIT_M_PATH=/path/to/Gpufit/matlab ``` Both `GPUFIT_PATH` and `GPUFIT_M_PATH` are then passed to MATLAB via `addpath`: diff --git a/docs/integration/rocketship.md b/docs/integration/rocketship.md index 5c3e6d2..b2acfa8 100644 --- a/docs/integration/rocketship.md +++ b/docs/integration/rocketship.md @@ -15,10 +15,12 @@ DCEPrep uses [ROCKETSHIP](https://github.com/petmri/ROCKETSHIP) (version 1.2+), ## How DCEPrep Locates ROCKETSHIP -DCEPrep finds the ROCKETSHIP installation at runtime by searching for `run_dce_cli.m`: +On Linux, `preprocess_all.sh` first uses `ROCKETSHIP_PATH` when set, then the +Docker location `/opt/ROCKETSHIP/ROCKETSHIP-dev`, and finally searches `$HOME` +for `run_dce_cli.m`. Set the variable explicitly for a nonstandard installation: ```bash -find / -name '*run_dce_cli.m' 2>/dev/null +export ROCKETSHIP_PATH=/path/to/ROCKETSHIP ``` In Docker, ROCKETSHIP is installed at a fixed path and the script preferences are mounted at: diff --git a/docs/integration/vascular-function.md b/docs/integration/vascular-function.md index cb43386..54ac557 100644 --- a/docs/integration/vascular-function.md +++ b/docs/integration/vascular-function.md @@ -33,11 +33,20 @@ vascular_function produces the following files per subject: ## How DCEPrep Locates vascular_function -At runtime DCEPrep searches for `main_vif.py`: +For automatic AIF selection, `preprocess_all.sh` first uses `AUTO_AIF_PATH` +when set, then the Docker location `/opt/vascular_function`, and finally +searches `$HOME` for `main_vif.py`. Set the variable explicitly for a +nonstandard installation: ```bash -AUTO_AIF_PATH=$(find $HOME -wholename '*main_vif.py' -printf '%h\n' -quit \ - || find / -name '*main_vif.py' -printf '%h\n' -quit) +export AUTO_AIF_PATH=/path/to/vascular_function +``` + +`main_vif.py` requires TensorFlow. `preprocess_all.sh` uses `python3` by default; +set `AUTO_AIF_PYTHON` when TensorFlow is installed in a dedicated environment: + +```bash +export AUTO_AIF_PYTHON=/path/to/tensorflow-environment/bin/python ``` In Docker, vascular_function is installed at `/opt/vascular_function` and the pre-trained model weights are mounted at: diff --git a/docs/reference/preprocessing-cli.md b/docs/reference/preprocessing-cli.md index 9f95000..bc7fec2 100644 --- a/docs/reference/preprocessing-cli.md +++ b/docs/reference/preprocessing-cli.md @@ -21,6 +21,7 @@ Full reference for `preprocess_all.sh`. | `-B` | — | No | Enable **second round** of bias field correction (applied after z-normalization). Only relevant when `-Z` is also set. | | `-c` | — | No | Clean the case's derivatives folder before processing. Ensures a fresh run; disables skip behavior. | | `-C` | `name` | No | Enable [comparison mode](../how-to/comparison-mode.md). Writes outputs to `derivatives/sub-##/ses-##//`. | +| `-j` | `count` | No | Limit concurrent VFA registration and FAST jobs. Defaults to unlimited to preserve existing behavior. Start with `2` when running multiple cases on one host. | | `-m` | — | No | Enable head motion correction using FSL `mcflirt`. | | `-s` | — | No | Skip preprocessing if the final DCE output file already exists. | | `-t` | — | No | Run only up to T1 map generation (skip DCE processing). Useful for T1-only protocols. | @@ -28,6 +29,13 @@ Full reference for `preprocess_all.sh`. | `-w` | `path` | No | Path to the AutoAIF neural network weights file. | | `-Z` | — | No | Enable z-axis slice normalization on VFA and DCE images. | +### Environment variables + +| Variable | Description | +|---|---| +| `HD_BET_COMMAND` | HD-BET executable to use. Defaults to `hd-bet`; set it when the default launcher is not associated with a working HD-BET environment. | +| `AUTO_AIF_PYTHON` | Python interpreter for AutoAIF. Defaults to `python3` and must be able to import TensorFlow when `-A A` is used. | + --- ## Example Invocations diff --git a/docs/user-guide/outputs.md b/docs/user-guide/outputs.md index 75a0454..9af8b11 100644 --- a/docs/user-guide/outputs.md +++ b/docs/user-guide/outputs.md @@ -134,7 +134,7 @@ The spreadsheet contains the following column groups: | **Demographics** | Date, APOE, Sex, Age | | **Scanner** | Machine, Institution, Coil | | **Acquisition** | TR, Time_resolution, TE, Flip_angle, n_reps | -| **QC** | Approximate SNR, AIFitness, aif_fitted_r2, manual_aif_status, max_disp | +| **QC** | Approximate_SNR, AIFitness, aif_fitted_r2, manual_aif_status, max_disp | | **T1** | T1_blood, T1_wm_median, T1_gm_median | | **Ktrans** | Ktrans_wm_median, Ktrans_gm_median, plus per-region medians (Hippo, PhG, Putamen, Pallidum, Thalamus, Caudate, Amygdala, Entorhinal cortex, Fusiform gyrus cortex/WM, Insula WM, Superior/Inferior temporal cortex, Posterior cingulate cortex, Medial temporal cortex) | | **Vp** | Per-region Vp medians (same regions as Ktrans) | diff --git a/preprocess_all.sh b/preprocess_all.sh index 9edf313..5fb0569 100755 --- a/preprocess_all.sh +++ b/preprocess_all.sh @@ -38,7 +38,6 @@ while getopts ":d:bBa:A:ZfhcC:j:mMsS:tl:T:w:p" options; do AIF_SUFFIX=${OPTARG} ;; A) - AUTO_AIF_PATH=$(find $HOME -wholename '*main_vif.py' -printf '%h\n' -quit || find / -name '*main_vif.py' -printf '%h\n' -quit) &> /dev/null case "${OPTARG}" in M) USE_AUTO_AIF=0 From cc89ab8565f48613e61fc2c81cf2aad33eef43fe Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Wed, 26 Aug 2026 13:26:56 -0700 Subject: [PATCH 23/29] correct path search strings --- DCE_all.sh | 4 ++-- preprocess_all.sh | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/DCE_all.sh b/DCE_all.sh index 2e73045..1151c9a 100755 --- a/DCE_all.sh +++ b/DCE_all.sh @@ -100,7 +100,7 @@ if [ -z "$DATA_DIR" ] fi if [[ "$OSTYPE" == "linux-gnu" ]]; then # respect a pre-set path (e.g. from CI) instead of searching the whole filesystem - ROCKETSHIP_PATH=${ROCKETSHIP_PATH:-$(find $HOME -name '*run_dce_cli.m' -printf '%h\n' -quit || find / -name '*run_dce_cli.m' -printf '%h\n' -quit) 2> /dev/null} + ROCKETSHIP_PATH=${ROCKETSHIP_PATH:-$(find $HOME -name '*run_dce_cli.m' -printf '%h\n' -quit || find / -name '*run_dce_cli.m' -printf '%h\n' -quit 2> /dev/null)} SCRIPT_PATH=$(dirname "$(realpath $0)") GPUFIT_PATH=${GPUFIT_PATH:-$(find $HOME -name 'GpufitConstrainedMex.mexa64' -printf '%h\n' -quit || find / -name 'GpufitConstrainedMex.mexa64' -printf '%h\n' -quit)} GPUFIT_M_PATH=${GPUFIT_M_PATH:-$(find $HOME -name 'ModelID.m' -printf '%h\n' -quit || find / -name 'ModelID.m' -printf '%h\n' -quit)} @@ -259,7 +259,7 @@ for der_dir in $SCRIPT_LOOP_DIR; do rm -f dce/dce_patlak_fit_rois.xls if [ $USE_PYTHON -eq 1 ] then - ${ROCKETSHIP_PATH}/.venv/bin/python ${ROCKETSHIP_PATH}/run_dce_cli_python_case.py --subject-source $SUBJECT_TP_PATH --subject-tp $SUBJECT_TP_PATH --output-dir $SUBJECT_TP_PATH --events off + ${ROCKETSHIP_PATH}/.venv/bin/python ${ROCKETSHIP_PATH}/run_dce_python_case.py --subject-source $SUBJECT_TP_PATH --subject-tp $SUBJECT_TP_PATH --output-dir $SUBJECT_TP_PATH --events off else matlab -nodisplay -r "cd('$ROCKETSHIP_PATH'); addpath '$GPUFIT_PATH'; addpath '$GPUFIT_M_PATH'; run_dce_cli('$DATA_DIR/$SUBJECT/$SESSION/', '$SUBJECT_TP_PATH/'); exit;" fi diff --git a/preprocess_all.sh b/preprocess_all.sh index 5fb0569..9089ba9 100755 --- a/preprocess_all.sh +++ b/preprocess_all.sh @@ -76,7 +76,6 @@ while getopts ":d:bBa:A:ZfhcC:j:mMsS:tl:T:w:p" options; do DATE=$(date +%Y-%m-%d) # derivatives dir is up 2 levels from data dir DERIV_DIR=$(dirname $(dirname $DATA_DIR))/derivatives - echo $DERIV_DIR if [ ! -d "$DERIV_DIR" ] then mkdir -p "$DERIV_DIR" From 3209acedaf181ee356c0a1befed1ee33bbf58d52 Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Thu, 27 Aug 2026 12:57:55 -0700 Subject: [PATCH 24/29] fix actions path to autoAIF weights --- .github/workflows/DCE.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/DCE.yml b/.github/workflows/DCE.yml index 5571bab..a68f1f2 100644 --- a/.github/workflows/DCE.yml +++ b/.github/workflows/DCE.yml @@ -39,7 +39,7 @@ jobs: pip install --upgrade hd-bet - name: Preprocess the data - run: ./preprocess_all.sh -d /media/network_mriphysics/USC-PPG/RUNNER_DATA/sourcedata/raw -b -Z -m -c -A A + run: ./preprocess_all.sh -d /media/network_mriphysics/USC-PPG/RUNNER_DATA/sourcedata/raw -b -Z -m -c -A A -w /home/mrispec/Code/DCEPrep/docker/files/model_weight_huber1.h5 - name: Run DCE run: ./DCE_all.sh -d /media/network_mriphysics/USC-PPG/RUNNER_DATA/sourcedata/raw -f @@ -100,7 +100,7 @@ jobs: pip install --upgrade hd-bet - name: Preprocess the data - run: ./preprocess_all.sh -d /media/network_mriphysics/USC-PPG/RUNNER_DATA/sourcedata/raw -b -Z -m -c -p -A A + run: ./preprocess_all.sh -d /media/network_mriphysics/USC-PPG/RUNNER_DATA/sourcedata/raw -b -Z -m -c -p -A A -w /home/mrispec/Code/DCEPrep/docker/files/model_weight_huber1.h5 - name: Run DCE run: ./DCE_all.sh -d /media/network_mriphysics/USC-PPG/RUNNER_DATA/sourcedata/raw -f -p From ca307f15c9faeed37c9f117f0455639dcfdbea20 Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Thu, 27 Aug 2026 13:23:05 -0700 Subject: [PATCH 25/29] hopeful autoAIF weight path fix for runner --- preprocess_all.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/preprocess_all.sh b/preprocess_all.sh index 9089ba9..644db1c 100755 --- a/preprocess_all.sh +++ b/preprocess_all.sh @@ -758,7 +758,7 @@ for source_dir in $DATA_DIR/$SCRIPT_LOOP_DIRS; do if [ $EN_MOTION_CORR -eq 1 ] then "$AUTO_AIF_PYTHON" $AUTO_AIF_PATH/main_vif.py --mode inference --input_path dce/${PREFIX}_desc-hmc_DCE.nii.gz --save_output_path $PWD/dce \ - --model_weight_path $SCRIPT_PATH/$AUTOAIF_WEIGHT_PATH \ + --model_weight_path $AUTOAIF_WEIGHT_PATH \ --model_name $AUTOAIF_MODEL \ --save_image 1 &> dce/${PREFIX}_desc-autoaif.log if [ ! -f "dce/${PREFIX}_desc-hmc_DCE_float_mask.nii" ] || [ ! -f "dce/${PREFIX}_desc-hmc_DCE_mask.nii" ]; then @@ -776,7 +776,7 @@ for source_dir in $DATA_DIR/$SCRIPT_LOOP_DIRS; do fslmaths anat/${PREFIX}_${REF_SPACE}_T1map.nii.gz -mas dce/${PREFIX}_label-AIF_desc-topvoxels_mask.nii dce/${PREFIX}_label-AIF_T1map.nii else "$AUTO_AIF_PYTHON" $AUTO_AIF_PATH/main_vif.py --mode inference --input_path $source_dir/dce/${PREFIX}_DCE.nii.gz --save_output_path $PWD/dce \ - --model_weight_path $SCRIPT_PATH/$AUTOAIF_WEIGHT_PATH \ + --model_weight_path $AUTOAIF_WEIGHT_PATH \ --model_name $AUTOAIF_MODEL \ --save_image 1 &> dce/${PREFIX}_desc-autoaif.log if [ ! -f "dce/${PREFIX}_DCE_float_mask.nii" ] || [ ! -f "dce/${PREFIX}_DCE_mask.nii" ]; then From 7979f10ec251b2d16862d698e1503721f6e408cc Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Thu, 27 Aug 2026 14:00:45 -0700 Subject: [PATCH 26/29] remove PDF gen/upload from actions --- .github/workflows/DCE.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/DCE.yml b/.github/workflows/DCE.yml index a68f1f2..5942b65 100644 --- a/.github/workflows/DCE.yml +++ b/.github/workflows/DCE.yml @@ -50,9 +50,9 @@ jobs: cat /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/logs/preprocessing_log_${date}.txt cat /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/logs/dce_log_${date}.txt - - name: Convert HTML to PDF - run: wkhtmltopdf --enable-local-file-access /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.html /media/network_mriphysics/USC-PPG/RUNNER_DATA/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.pdf - continue-on-error: true + # - name: Convert HTML to PDF + # run: wkhtmltopdf --enable-local-file-access /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.html /media/network_mriphysics/USC-PPG/RUNNER_DATA/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.pdf + # continue-on-error: true - name: Validate analysis results run: | @@ -66,7 +66,6 @@ jobs: name: HTML_report path: | /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/dceprep/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.html - /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/dceprep/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.pdf /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/dceprep/sub-1101743/ses-01/figures/ retention-days: 7 if-no-files-found: error @@ -111,9 +110,9 @@ jobs: cat /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/logs/preprocessing_log_${date}.txt cat /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/logs/dce_log_${date}.txt - - name: Convert HTML to PDF - run: wkhtmltopdf --enable-local-file-access /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.html /media/network_mriphysics/USC-PPG/RUNNER_DATA/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.pdf - continue-on-error: true + # - name: Convert HTML to PDF + # run: wkhtmltopdf --enable-local-file-access /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.html /media/network_mriphysics/USC-PPG/RUNNER_DATA/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.pdf + # continue-on-error: true - name: Validate analysis results run: | @@ -127,7 +126,6 @@ jobs: name: HTML_report path: | /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/dceprep/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.html - /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/dceprep/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.pdf /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/dceprep/sub-1101743/ses-01/figures/ retention-days: 7 if-no-files-found: error From 0c01fdffd47687b3ba6aea1d4ad3e5c80ba66758 Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Thu, 27 Aug 2026 14:12:40 -0700 Subject: [PATCH 27/29] missed a spot --- .github/workflows/DCE.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/DCE.yml b/.github/workflows/DCE.yml index 5942b65..2f902c1 100644 --- a/.github/workflows/DCE.yml +++ b/.github/workflows/DCE.yml @@ -57,7 +57,6 @@ jobs: - name: Validate analysis results run: | test -f /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/dceprep/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.html - test -f /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/dceprep/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.pdf test -n "$(find /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/dceprep/sub-1101743/ses-01/figures/ -type f -print -quit)" - name: Upload analysis results @@ -117,7 +116,6 @@ jobs: - name: Validate analysis results run: | test -f /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/dceprep/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.html - test -f /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/dceprep/sub-1101743/ses-01/reports/sub-1101743_ses-01_desc-casereport.pdf test -n "$(find /media/network_mriphysics/USC-PPG/RUNNER_DATA/derivatives/dceprep/sub-1101743/ses-01/figures/ -type f -print -quit)" - name: Upload analysis results From 73a01b7445e8ef2b6f357d81edbc1d05fdc58d61 Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Fri, 28 Aug 2026 18:58:10 -0700 Subject: [PATCH 28/29] Refactor path resolution for ROCKETSHIP and GPUFIT, enhance error handling, and update output file naming conventions to include gzip compression. --- DCE_all.sh | 56 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/DCE_all.sh b/DCE_all.sh index 1151c9a..f797fde 100755 --- a/DCE_all.sh +++ b/DCE_all.sh @@ -98,12 +98,43 @@ if [ -z "$DATA_DIR" ] echo "ERROR: Please use '-d [dir_path]' to pass the path to your main data directory to this script." exit 1 fi + +resolve_tool_path() { + local configured_path=$1 + local marker_file=$2 + shift 2 + local candidate_path + + if [ -n "$configured_path" ]; then + if [ -f "$configured_path/$marker_file" ]; then + printf '%s\n' "$configured_path" + return 0 + fi + echo "ERROR: Configured path $configured_path does not contain $marker_file." >&2 + return 1 + fi + + for candidate_path in "$@"; do + if [ -f "$candidate_path/$marker_file" ]; then + printf '%s\n' "$candidate_path" + return 0 + fi + done + + find "$HOME" \ + \( -path "$HOME/.local/share/Trash" -o -path "$HOME/.local/share/Trash/*" -o -path "$HOME/.Trash" -o -path "$HOME/.Trash/*" \) -prune -o \ + -type f -name "$marker_file" -printf '%h\n' -quit 2> /dev/null +} + if [[ "$OSTYPE" == "linux-gnu" ]]; then - # respect a pre-set path (e.g. from CI) instead of searching the whole filesystem - ROCKETSHIP_PATH=${ROCKETSHIP_PATH:-$(find $HOME -name '*run_dce_cli.m' -printf '%h\n' -quit || find / -name '*run_dce_cli.m' -printf '%h\n' -quit 2> /dev/null)} + ROCKETSHIP_PATH=$(resolve_tool_path "${ROCKETSHIP_PATH:-}" "run_dce_cli.m" "/opt/ROCKETSHIP/ROCKETSHIP-dev") SCRIPT_PATH=$(dirname "$(realpath $0)") - GPUFIT_PATH=${GPUFIT_PATH:-$(find $HOME -name 'GpufitConstrainedMex.mexa64' -printf '%h\n' -quit || find / -name 'GpufitConstrainedMex.mexa64' -printf '%h\n' -quit)} - GPUFIT_M_PATH=${GPUFIT_M_PATH:-$(find $HOME -name 'ModelID.m' -printf '%h\n' -quit || find / -name 'ModelID.m' -printf '%h\n' -quit)} + GPUFIT_PATH=$(resolve_tool_path "${GPUFIT_PATH:-}" "GpufitConstrainedMex.mexa64" "/opt/Gpufit/matlab64") + GPUFIT_M_PATH=$(resolve_tool_path "${GPUFIT_M_PATH:-}" "ModelID.m" "/opt/Gpufit/matlab") + if [ -z "$ROCKETSHIP_PATH" ] || [ -z "$GPUFIT_PATH" ] || [ -z "$GPUFIT_M_PATH" ]; then + echo "ERROR: Unable to locate ROCKETSHIP or GPUfit. Set ROCKETSHIP_PATH, GPUFIT_PATH, and GPUFIT_M_PATH to directories containing their required MATLAB files." >&2 + exit 1 + fi else ROCKETSHIP_PATH=$(find $HOME -type d -name ROCKETSHIP) SCRIPT_PATH=$(find $HOME -type d -name in-house_toolbox) @@ -259,17 +290,18 @@ for der_dir in $SCRIPT_LOOP_DIR; do rm -f dce/dce_patlak_fit_rois.xls if [ $USE_PYTHON -eq 1 ] then - ${ROCKETSHIP_PATH}/.venv/bin/python ${ROCKETSHIP_PATH}/run_dce_python_case.py --subject-source $SUBJECT_TP_PATH --subject-tp $SUBJECT_TP_PATH --output-dir $SUBJECT_TP_PATH --events off + ${ROCKETSHIP_PATH}/.venv/bin/python ${ROCKETSHIP_PATH}/run_dce_python_case.py --subject-source "$DATA_DIR/$SUBJECT/$SESSION/" --subject-tp $SUBJECT_TP_PATH --output-dir $SUBJECT_TP_PATH --events off else matlab -nodisplay -r "cd('$ROCKETSHIP_PATH'); addpath '$GPUFIT_PATH'; addpath '$GPUFIT_M_PATH'; run_dce_cli('$DATA_DIR/$SUBJECT/$SESSION/', '$SUBJECT_TP_PATH/'); exit;" fi - mv dce/dce_*_fit_Ktrans.nii dce/${PREFIX}_Ktrans.nii - mv dce/dce_*_fit_ktrans_ci_low.nii dce/${PREFIX}_Ktrans_ci_low.nii - mv dce/dce_*_fit_ktrans_ci_high.nii dce/${PREFIX}_Ktrans_ci_high.nii - mv dce/dce_*_fit_vp.nii dce/${PREFIX}_vp.nii - mv dce/dce_*_fit_vp_ci_low.nii dce/${PREFIX}_vp_ci_low.nii - mv dce/dce_*_fit_vp_ci_high.nii dce/${PREFIX}_vp_ci_high.nii - mv dce/dce_*_fit_sse.nii dce/${PREFIX}_sse.nii + gzip dce/*.nii + mv dce/*_fit_*trans.nii.gz dce/${PREFIX}_Ktrans.nii.gz + mv dce/*_fit_*trans_ci_low.nii.gz dce/${PREFIX}_desc-cilow_Ktrans.nii.gz + mv dce/*_fit_*trans_ci_high.nii.gz dce/${PREFIX}_desc-cihigh_Ktrans.nii.gz + mv dce/*_fit_vp.nii.gz dce/${PREFIX}_vp.nii.gz + mv dce/*_fit_vp_ci_low.nii.gz dce/${PREFIX}_desc-cilow_vp.nii.gz + mv dce/*_fit_vp_ci_high.nii.gz dce/${PREFIX}_desc-cihigh_vp.nii.gz + mv dce/*_fit_sse.nii.gz dce/${PREFIX}_sse.nii.gz # move images into figures folder mv dce/dce*.png figures/ rm -f dce/*.fig From 9c08ddb8625d31ff234f43bfee9bb85321fff8a4 Mon Sep 17 00:00:00 2001 From: Lucas Saca Date: Tue, 1 Sep 2026 13:54:09 -0700 Subject: [PATCH 29/29] gzip all DCE nii outputs, fix log output w/ w/o -p, fix plot_anat coord cuts, fix caserep aif NaNs, fix rawdata->sourcedata/raw refs, fix a couple other naming changes, fix exclude sheet key diffs --- DCE_all.sh | 22 ++++++++++++++-------- case_report.py | 26 ++++++++++++++++---------- ktrans_report.py | 2 +- population_report.py | 12 ++++++------ 4 files changed, 37 insertions(+), 25 deletions(-) diff --git a/DCE_all.sh b/DCE_all.sh index f797fde..70d8ec8 100755 --- a/DCE_all.sh +++ b/DCE_all.sh @@ -294,7 +294,7 @@ for der_dir in $SCRIPT_LOOP_DIR; do else matlab -nodisplay -r "cd('$ROCKETSHIP_PATH'); addpath '$GPUFIT_PATH'; addpath '$GPUFIT_M_PATH'; run_dce_cli('$DATA_DIR/$SUBJECT/$SESSION/', '$SUBJECT_TP_PATH/'); exit;" fi - gzip dce/*.nii + gzip dce/*.nii &> /dev/null mv dce/*_fit_*trans.nii.gz dce/${PREFIX}_Ktrans.nii.gz mv dce/*_fit_*trans_ci_low.nii.gz dce/${PREFIX}_desc-cilow_Ktrans.nii.gz mv dce/*_fit_*trans_ci_high.nii.gz dce/${PREFIX}_desc-cihigh_Ktrans.nii.gz @@ -303,12 +303,18 @@ for der_dir in $SCRIPT_LOOP_DIR; do mv dce/*_fit_vp_ci_high.nii.gz dce/${PREFIX}_desc-cihigh_vp.nii.gz mv dce/*_fit_sse.nii.gz dce/${PREFIX}_sse.nii.gz # move images into figures folder - mv dce/dce*.png figures/ + mv dce/dce*.png figures/ &> /dev/null rm -f dce/*.fig - if [ ! -f "dce/${PREFIX}_Ktrans.nii" ] + if [ ! -f "dce/${PREFIX}_Ktrans.nii.gz" ] then echo $dir "Missing Ktrans maps. DCE failed or inputs were not generated. Hopefully message below is relevant." >> $LOG_FILE - tail -1 dce/A_dceR1info.log >> $LOG_FILE + if [ $USE_PYTHON -eq 1 ] && [ -f "reports/dce_pipeline_events.jsonl" ] + then + tail -1 reports/dce_pipeline_events.jsonl >> $LOG_FILE + elif [ -f "dce/A_dceR1info.log" ] + then + tail -1 dce/A_dceR1info.log >> $LOG_FILE + fi cd $DATA_DIR/../derivatives fail=1 continue @@ -334,9 +340,9 @@ for der_dir in $SCRIPT_LOOP_DIR; do fslmaths anat/${PREFIX}_space-DCEref_T1map.nii -mas anat/${PREFIX}_space-DCEref_label-CSF_mask.nii.gz anat/${PREFIX}_space-DCEref_label-CSF_T1map.nii # Apply masks to Ktrans map - fslmaths dce/${PREFIX}_Ktrans.nii -mas anat/${PREFIX}_space-DCEref_label-WM_mask.nii.gz dce/${PREFIX}_label-WM_Ktrans.nii - fslmaths dce/${PREFIX}_Ktrans.nii -mas anat/${PREFIX}_space-DCEref_label-GM_mask.nii.gz dce/${PREFIX}_label-GM_Ktrans.nii - fslmaths dce/${PREFIX}_Ktrans.nii -mas anat/${PREFIX}_space-DCEref_label-CSF_mask.nii.gz dce/${PREFIX}_label-CSF_Ktrans.nii + fslmaths dce/${PREFIX}_Ktrans.nii.gz -mas anat/${PREFIX}_space-DCEref_label-WM_mask.nii.gz dce/${PREFIX}_label-WM_Ktrans.nii.gz + fslmaths dce/${PREFIX}_Ktrans.nii.gz -mas anat/${PREFIX}_space-DCEref_label-GM_mask.nii.gz dce/${PREFIX}_label-GM_Ktrans.nii.gz + fslmaths dce/${PREFIX}_Ktrans.nii.gz -mas anat/${PREFIX}_space-DCEref_label-CSF_mask.nii.gz dce/${PREFIX}_label-CSF_Ktrans.nii.gz # registration QC python3 $SCRIPT_PATH/ktrans_analysis.py $dir $PREFIX @@ -392,7 +398,7 @@ for der_dir in $SCRIPT_LOOP_DIR; do # --metric MI[ $FSLDIR/data/standard/MNI152_T1_1mm_brain.nii.gz,anat/${PREFIX}_space-DCEref_T1w.nii.gz,1,32,Regular,0.25 ] \ # --convergence [ 1000x500x250x100,1e-6,10 ] --shrink-factors 12x8x4x2 --smoothing-sigmas 4x3x2x1vox # antsApplyTransforms -i dce/${PREFIX}_Ktrans.nii -r $FSLDIR/data/standard/MNI152_T1_1mm_brain.nii.gz -t ${PREFIX}_space-MNI_T1w0Warp.nii.gz -t t1w_MNI0GenericAffine.mat -t [T1_dyn0GenericAffine.mat, 1] -o Ktrans_MNI.nii.gz - antsApplyTransforms -i dce/${PREFIX}_Ktrans.nii -r $FSLDIR/data/standard/MNI152_T1_1mm_brain.nii.gz -t anat/${PREFIX}_space-MNI_T1w0GenericAffine.mat -t [ anat/${PREFIX}_from-T1w_to-DCEref.mat, 1] -o dce/${PREFIX}_space-MNI_Ktrans.nii.gz + antsApplyTransforms -i dce/${PREFIX}_Ktrans.nii.gz -r $FSLDIR/data/standard/MNI152_T1_1mm_brain.nii.gz -t anat/${PREFIX}_space-MNI_T1w0GenericAffine.mat -t [ anat/${PREFIX}_from-T1w_to-DCEref.mat, 1] -o dce/${PREFIX}_space-MNI_Ktrans.nii.gz # antsApplyTransforms -i dce_patlak_fit_vp.nii -r $FSLDIR/data/standard/MNI152_T1_1mm_brain.nii.gz -t t1w_MNI1Warp.nii.gz -t t1w_MNI0GenericAffine.mat -t [T1_dyn0GenericAffine.mat, 1] -o vp_MNI.nii.gz # flirt -in dce_patlak_fit_Ktrans.nii -ref $FSLDIR/data/standard/MNI152_T1_1mm.nii.gz -out ktrans_2_MNI.nii.gz -init DCE2MNI.mat -applyxfm else diff --git a/case_report.py b/case_report.py index 6eb06d7..356f95e 100644 --- a/case_report.py +++ b/case_report.py @@ -25,7 +25,7 @@ # if source_dir[-1] == '/': # source_dir = source_dir[:-1] -files_to_reorient = [f'anat/{prefix}_flip-01_space-DCEref_VFA.nii.gz', f'dce/{prefix}_Ktrans.nii', +files_to_reorient = [f'anat/{prefix}_flip-01_space-DCEref_VFA.nii.gz', f'dce/{prefix}_Ktrans.nii.gz', f'anat/{prefix}_space-DCEref_T1w.nii.gz', f'anat/{prefix}_space-DCEref_label-WM_mask.nii.gz', f'anat/{prefix}_space-DCEref_T1map.nii', f'anat/{prefix}_space-DCEref_label-brain_mask.nii.gz', f'anat/{prefix}_space-DCEref_label-GM_mask.nii.gz', f'anat/{prefix}_space-DCEref_seg-wmparc_dseg.nii.gz', @@ -70,7 +70,7 @@ def space_ras_path(path): try: subprocess.run(command, check=True) if nifti_stem(file) == f'dce/{prefix}_Ktrans': - ktrans = nib.load(resolve_nifti_path(space_ras_path(f'dce/{prefix}_Ktrans.nii'))) + ktrans = nib.load(resolve_nifti_path(space_ras_path(f'dce/{prefix}_Ktrans.nii.gz'))) ktrans_data = ktrans.get_fdata() ktrans_flipped = np.flip(ktrans_data, axis=1) ktrans_flipped = nib.Nifti1Image(ktrans_flipped, ktrans.affine, ktrans.header) @@ -84,8 +84,10 @@ def space_ras_path(path): ktrans_z_slices = min(dimensions) midpt = int(ktrans_coords-5*ktrans_z_slices/2) max_coord = int(ktrans_coords-5*ktrans_z_slices) - plotting.plot_anat(ktrans_flipped, display_mode='z', cut_coords=range(ktrans_coords, midpt, -5), axes=axes[0], vmin=0, vmax=expected_ktrans_vmax, cmap='gnuplot', annotate=False, colorbar=True) - plotting.plot_anat(ktrans_flipped, display_mode='z', cut_coords=range(midpt, max_coord, -5), axes=axes[1], vmin=0, vmax=expected_ktrans_vmax, cmap='gnuplot', annotate=False) + coords1 = list(range(ktrans_coords, midpt, -5)) + coords2 = list(range(midpt, max_coord, -5)) + plotting.plot_anat(ktrans_flipped, display_mode='z', cut_coords=coords1, axes=axes[0], vmin=0, vmax=expected_ktrans_vmax, cmap='gnuplot', annotate=False, colorbar=True) + plotting.plot_anat(ktrans_flipped, display_mode='z', cut_coords=coords2, axes=axes[1], vmin=0, vmax=expected_ktrans_vmax, cmap='gnuplot', annotate=False) plt.savefig('figures/ktrans.svg', bbox_inches='tight', pad_inches = 0) plt.close() except Exception as e: @@ -103,17 +105,19 @@ def space_ras_path(path): try: subprocess.run(command, check=True) if nifti_stem(file) == f'dce/{prefix}_Ktrans': - ktrans = nib.load(resolve_nifti_path(space_ras_path(f'dce/{prefix}_Ktrans.nii'))) + ktrans = nib.load(resolve_nifti_path(space_ras_path(f'dce/{prefix}_Ktrans.nii.gz'))) ktrans_data = ktrans.get_fdata() ktrans_flipped = np.flip(ktrans_data, axis=1) ktrans_flipped = nib.Nifti1Image(ktrans_flipped, ktrans.affine, ktrans.header) dimensions = ktrans.header.get_data_shape() voxel_size = ktrans.header.get_zooms() + range1 = list(range(-56, -21, 5)) + range2 = list(range(-21, 13, 5)) # plot Ktrans, different coords fig, axes = plt.subplots(nrows=2, ncols=1, figsize=(15, 5), gridspec_kw={'hspace': -.1, 'wspace': -.1}, dpi=300) - plotting.plot_anat(ktrans_flipped, display_mode='z', cut_coords=range(-56, -21, 5), axes=axes[0], vmin=0, vmax=expected_ktrans_vmax, cmap='gnuplot', annotate=False, colorbar=True) - plotting.plot_anat(ktrans_flipped, display_mode='z', cut_coords=range(-21, 13, 5), axes=axes[1], vmin=0, vmax=expected_ktrans_vmax, cmap='gnuplot', annotate=False) + plotting.plot_anat(ktrans_flipped, display_mode='z', cut_coords=range1, axes=axes[0], vmin=0, vmax=expected_ktrans_vmax, cmap='gnuplot', annotate=False, colorbar=True) + plotting.plot_anat(ktrans_flipped, display_mode='z', cut_coords=range2, axes=axes[1], vmin=0, vmax=expected_ktrans_vmax, cmap='gnuplot', annotate=False) plt.savefig('figures/ktrans.svg', bbox_inches='tight', pad_inches = 0) plt.close() except Exception as e: @@ -322,15 +326,17 @@ def space_ras_path(path): img = nib.load(resolve_nifti_path(f'{source_dir}/dce/{prefix}_DCE.nii.gz')) img_data = img.get_fdata() # binarize AIF -aif_data[aif_data > 0] = 1 -aif_data[aif_data < 0] = 0 +aif_data = np.where(np.isfinite(aif_data) & (aif_data > 0), 1.0, 0.0) # mask DCE where AIF is 1 # but first ensure that DCE and AIF have same number of dimensions if len(aif_data.shape) < len(img_data.shape): aif_data = np.expand_dims(aif_data, axis=-1) aif_data_roi = img_data * aif_data # sum AIF data for each time point, z-slice independent -aif_curve = np.sum(aif_data_roi, axis=(0, 1, 2)) / np.sum(aif_data[aif_data > 0]) +aif_voxel_count = np.count_nonzero(aif_data) +if aif_voxel_count == 0: + raise ValueError(f'AIF mask is empty after removing NaNs for {prefix}') +aif_curve = np.sum(aif_data_roi, axis=(0, 1, 2)) / aif_voxel_count # divide by AIF mean of timepoints before contrast agent arrival baseline = get_baseline_from_curve(aif_curve) aif_curve_ratio = aif_curve / baseline diff --git a/ktrans_report.py b/ktrans_report.py index 1c140cc..6aa6d80 100644 --- a/ktrans_report.py +++ b/ktrans_report.py @@ -32,7 +32,7 @@ curves.append(png_np_array) curves.append(mpimg.imread('figures/dceAIF_fitting.png')) curves.append(mpimg.imread('figures/dce_timecurves.png')) -ktrans = nib.load('dce/' + prefix + '_Ktrans.nii') +ktrans = nib.load('dce/' + prefix + '_Ktrans.nii.gz') analysis = mpimg.imread('figures/' + prefix + '_desc-analysis.png') zeros = mpimg.imread('figures/' + prefix + '_desc-zeros.png') aif_curve = mpimg.imread('figures/dceAIF_fitting.png') diff --git a/population_report.py b/population_report.py index 6dcf18c..6f21d17 100644 --- a/population_report.py +++ b/population_report.py @@ -168,7 +168,7 @@ def get_case_stats(subject_id, timepoint): total_timepoints.append(subject_id + '/' + timepoint) # Check for missing data in rawdata folder prefix = f"{subject_id}_{timepoint}" - rawdata_dir = os.path.join(dir, "../rawdata", subject_id, timepoint) + rawdata_dir = os.path.join(dir, "../sourcedata/raw", subject_id, timepoint) missing_files = [] # Check for anat/prefix_T1w.nii.gz t1w_path = os.path.join(rawdata_dir, f"anat/{prefix}_T1w.nii.gz") @@ -195,7 +195,7 @@ def get_case_stats(subject_id, timepoint): # read AIF curve by applying aif.nii to dce.nii try: dce = os.path.join(dceprep_dir, subject_id, timepoint, f"dce/{subject_id}_{timepoint}_desc-bfcz_DCE.nii.gz") - aif = os.path.join(dceprep_dir, subject_id, timepoint, f"dce/{subject_id}_{timepoint}_desc-AIF_T1map.nii.gz") + aif = os.path.join(dceprep_dir, subject_id, timepoint, f"dce/{subject_id}_{timepoint}_label-AIF_T1map.nii.gz") if os.path.exists(dce) and os.path.exists(aif): # load files dce_img = nib.load(dce) @@ -257,7 +257,7 @@ def get_case_stats(subject_id, timepoint): else: manual_aif_status = "AUTO" # get fields we want from json - json_file = os.path.join(dir, "../rawdata", subject_id, timepoint, f"dce/{subject_id}_{timepoint}_DCE.json") + json_file = os.path.join(dir, "../sourcedata/raw", subject_id, timepoint, f"dce/{subject_id}_{timepoint}_DCE.json") try: with open(json_file, 'r') as f: data = json.load(f) @@ -465,7 +465,7 @@ def get_case_stats(subject_id, timepoint): return # read ktrans map try: - ktrans_map = os.path.join(dceprep_dir, subject_id, timepoint, f"dce/{subject_id}_{timepoint}_Ktrans.nii") + ktrans_map = os.path.join(dceprep_dir, subject_id, timepoint, f"dce/{subject_id}_{timepoint}_Ktrans.nii.gz") ktrans_map = nib.load(ktrans_map) ktrans_map = ktrans_map.get_fdata() except: @@ -513,7 +513,7 @@ def get_case_stats(subject_id, timepoint): error = "" try: prefix = f"{subject_id}_{timepoint}" - wmparc_path = os.path.join(dceprep_dir, subject_id, timepoint, f"anat/{prefix}_space-DCEref_desc-wmparc.nii.gz") + wmparc_path = os.path.join(dceprep_dir, subject_id, timepoint, f"anat/{prefix}_space-DCEref_seg-wmparc_dseg.nii.gz") if os.path.isfile(wmparc_path): wmparc = nib.load(wmparc_path) wmparc = wmparc.get_fdata() @@ -2573,7 +2573,7 @@ def get_case_stats(subject_id, timepoint): if len(population_data_exclude) > 0: df_exclude = pd.DataFrame(population_data_exclude) df_exclude = df_exclude.T - df_exclude = df_exclude[order_exclude] + df_exclude = df_exclude.reindex(columns=order_exclude) df_exclude.index.name = "Subject_ID" df_exclude.to_excel(writer, sheet_name='Pre-Exclude') for column in df_exclude.columns: