analysis: expand utility proof with full preprocessing and pathway evaluation

- Added KNN imputation, variance-stabilizing transform, and ComBat-like correction
- Added pathway-level consistency metrics (top-pathway Jaccard and pathway-score Spearman)
- Added structured corruption scenarios (batch shift + block missingness + extreme corruption)
- Updated per-run outputs to cover all methods across all scenarios
- Re-ran utility benchmark and refreshed benchmark JSON + robustness CSVs

Author: SID-6921

results/benchmarks/utility_proof_results.json

@@ -0,0 +1,5 @@
+spearman_log2fc,jaccard_significant,pathway_top_jaccard,pathway_score_spearman,ref_sig_count,test_sig_count,abs_sig_count_delta,snr_before,snr_after
+0.44709385705043314,0.05398920215956809,0.42857142857142855,0.9642857142857141,499,1258,759,5.673000317875098,5.673000317875098
+0.4622020969861404,0.054906542056074766,0.25,0.8999999999999998,499,1307,808,5.695769628367074,5.695769628367074
+0.45489563607598554,0.05660377358490566,0.25,1.0000000000000002,499,1237,738,5.679527332692107,5.679527332692107
+0.4719314397897845,0.051753881541115584,0.42857142857142855,0.9999999999999998,499,1330,831,5.715843745902316,5.715843745902316

results/robustness/GSE3037_extreme_corruption_baseline_runs.csv

@@ -0,0 +1,5 @@
+spearman_log2fc,jaccard_significant,pathway_top_jaccard,pathway_score_spearman,ref_sig_count,test_sig_count,abs_sig_count_delta,snr_before,snr_after
+0.28430895024968006,0.03481012658227848,1.0,0.9868131868131869,499,155,344,5.673000317875098,7.0901556357349484
+0.29281728650136607,0.05063291139240506,1.0,0.9736263736263736,499,165,334,5.695769628367074,7.113848355745308
+0.28100351717031674,0.032208588957055216,1.0,0.9868131868131869,499,174,325,5.679527332692107,7.082016458340329
+0.29721330572064186,0.04006410256410257,1.0,0.956043956043956,499,150,349,5.715843745902316,7.144445041577256

results/robustness/GSE3037_extreme_corruption_combat_like_runs.csv

@@ -0,0 +1,5 @@
+spearman_log2fc,jaccard_significant,pathway_top_jaccard,pathway_score_spearman,ref_sig_count,test_sig_count,abs_sig_count_delta,snr_before,snr_after
+0.4341046024527466,0.07741935483870968,1.0,0.9956043956043955,499,1505,1006,5.673000317875098,6.8933002113558945
+0.4355895642518959,0.09437854564208355,1.0,0.9912087912087912,499,1623,1124,5.695769628367074,6.909610668617224
+0.43288966750781155,0.09220907297830375,1.0,0.9868131868131869,499,1716,1217,5.679527332692107,6.87148513305869
+0.44889872066951064,0.08745627186406797,1.0,0.9780219780219781,499,1677,1178,5.715843745902316,6.942246636364176

results/robustness/GSE3037_extreme_corruption_knn_imputation_runs.csv

@@ -0,0 +1,5 @@
+spearman_log2fc,jaccard_significant,pathway_top_jaccard,pathway_score_spearman,ref_sig_count,test_sig_count,abs_sig_count_delta,snr_before,snr_after
+0.4008575254154838,0.059509918319719954,1.0,0.9692307692307693,499,1317,818,5.673000317875098,7.088811531789388
+0.38424368455882274,0.060884549109707066,1.0,0.9736263736263736,499,1348,849,5.695769628367074,7.113348344054456
+0.39034023816052965,0.05931198102016608,1.0,0.9692307692307693,499,1287,788,5.679527332692107,7.082013611361583
+0.3989901165279867,0.05959137343927355,1.0,0.9692307692307693,499,1368,869,5.715843745902316,7.144442161111397

results/robustness/GSE3037_extreme_corruption_minimal_imputation_runs.csv

@@ -0,0 +1,5 @@
+spearman_log2fc,jaccard_significant,pathway_top_jaccard,pathway_score_spearman,ref_sig_count,test_sig_count,abs_sig_count_delta,snr_before,snr_after
+0.39990140580857625,0.08276385725132877,1.0,0.9692307692307693,499,927,428,5.673000317875098,7.026399166632589
+0.3926450683050673,0.08147080028839221,1.0,0.9736263736263736,499,1001,502,5.695769628367074,7.032694556496736
+0.3951947364796511,0.08646616541353383,1.0,0.9692307692307693,499,946,447,5.679527332692107,7.003928022756065
+0.40357141589642276,0.08272327964860908,1.0,0.9736263736263736,499,980,481,5.715843745902316,7.067992299128932

results/robustness/GSE3037_extreme_corruption_quantile_normalization_runs.csv

@@ -0,0 +1,5 @@
+spearman_log2fc,jaccard_significant,pathway_top_jaccard,pathway_score_spearman,ref_sig_count,test_sig_count,abs_sig_count_delta,snr_before,snr_after
+0.3755439823744381,0.03359683794466403,0.25,0.6835164835164835,499,24,475,5.673000317875098,29.159178784597515
+0.3661435891353097,0.03536345776031434,0.42857142857142855,0.5780219780219781,499,28,471,5.695769628367074,30.075933243152136
+0.36835833056563205,0.03937007874015748,0.42857142857142855,0.7142857142857143,499,29,470,5.679527332692107,29.973106250000207
+0.37838677611657046,0.043137254901960784,0.6666666666666666,0.745054945054945,499,33,466,5.715843745902316,28.654560166415536

results/robustness/GSE3037_extreme_corruption_variance_stabilizing_runs.csv

@@ -1,11 +1,5 @@
-spearman_log2fc,jaccard_significant,ref_sig_count,test_sig_count,abs_sig_count_delta,snr_before,snr_after
-0.7008437793287231,0.2287480680061824,499,296,203,7.959742776253464,7.959742776253464
-0.7025905374718199,0.2914653784219002,499,303,196,7.939168328526337,7.939168328526337
-0.7061958051313502,0.24378881987577639,499,302,197,7.960946628068019,7.960946628068019
-0.7022537967275345,0.25499231950844853,499,318,181,7.958295423972096,7.958295423972096
-0.7042427546406693,0.2409448818897638,499,289,210,7.956239622510563,7.956239622510563
-0.6916258236648027,0.2554517133956386,499,307,192,7.9644413774133405,7.9644413774133405
-0.7107295743208489,0.25235109717868337,499,300,199,7.971140101415122,7.971140101415122
-0.7091742176224782,0.2465331278890601,499,310,189,7.962751786343125,7.962751786343125
-0.6931107080986104,0.2621664050235479,499,305,194,7.950504698557044,7.950504698557044
-0.7038519704005592,0.2523076923076923,499,315,184,7.966980110184102,7.966980110184102
+spearman_log2fc,jaccard_significant,pathway_top_jaccard,pathway_score_spearman,ref_sig_count,test_sig_count,abs_sig_count_delta,snr_before,snr_after
+0.7118397837166881,0.2410179640718563,1.0,0.967032967032967,499,330,169,7.948599921301808,7.948599921301808
+0.6971980230844617,0.24378881987577639,1.0,1.0000000000000002,499,302,197,7.951502721194278,7.951502721194278
+0.7099748959184399,0.21865443425076453,1.0,1.0000000000000002,499,298,201,7.957124734117111,7.957124734117111
+0.7084921887497916,0.27675840978593275,1.0,0.9780219780219781,499,336,163,7.9519666529463295,7.9519666529463295

results/robustness/GSE3037_high_missingness_baseline_runs.csv

@@ -0,0 +1,5 @@
+spearman_log2fc,jaccard_significant,pathway_top_jaccard,pathway_score_spearman,ref_sig_count,test_sig_count,abs_sig_count_delta,snr_before,snr_after
+0.3482410910711664,0.08139534883720931,1.0,0.9868131868131869,499,245,254,7.948599921301808,9.15150656080579
+0.3480297730991651,0.05339105339105339,1.0,0.9824175824175824,499,231,268,7.951502721194278,9.163962134420514
+0.3480769349681957,0.07680945347119646,1.0,0.9912087912087912,499,230,269,7.957124734117111,9.165031354626265
+0.3547971400746402,0.07725321888412018,1.0,0.9780219780219781,499,254,245,7.9519666529463295,9.150249176883126

results/robustness/GSE3037_high_missingness_combat_like_runs.csv

@@ -0,0 +1,5 @@
+spearman_log2fc,jaccard_significant,pathway_top_jaccard,pathway_score_spearman,ref_sig_count,test_sig_count,abs_sig_count_delta,snr_before,snr_after
+0.696980612428104,0.3225806451612903,1.0,0.9868131868131869,499,444,55,7.948599921301808,8.948604827383212
+0.6932455162855266,0.32793522267206476,1.0,1.0,499,485,14,7.951502721194278,8.951262792076466
+0.6956961309773778,0.32967032967032966,1.0,0.9956043956043955,499,469,30,7.957124734117111,8.955561624121215
+0.7057399164152562,0.3319783197831978,1.0,0.9780219780219781,499,484,15,7.9519666529463295,8.943585353682204