From 4419e7e8ef01da18a2f97ccafaa5609f5c426588 Mon Sep 17 00:00:00 2001 From: kinichen Date: Mon, 6 Jul 2026 17:02:05 -0400 Subject: [PATCH] Fix optional performance filtering for methods and polish scatterplot --- HT_LLM/similarity/algorithm_similarity.py | 2 +- .../filter_methods_by_performance.py | 45 ++++-- .../similarity/heatmaps_feature_similarity.py | 2 +- ...ot.py => scatterplot_algorithm_feature.py} | 136 +++++++++++++++--- 4 files changed, 155 insertions(+), 30 deletions(-) rename HT_LLM/similarity/{algorithm_feature_scatterplot.py => scatterplot_algorithm_feature.py} (54%) diff --git a/HT_LLM/similarity/algorithm_similarity.py b/HT_LLM/similarity/algorithm_similarity.py index c93a3c4..0437833 100644 --- a/HT_LLM/similarity/algorithm_similarity.py +++ b/HT_LLM/similarity/algorithm_similarity.py @@ -67,7 +67,7 @@ # Filter methods in heatmap by performance threshold -threshold = 0.6 # must exist; else, run filter_methods_by_performance.py with the desired threshold first +threshold = 0.0 # must exist; else, run filter_methods_by_performance.py with the desired threshold first eligible_methods_path = os.path.join( "sample_data", f"threshold_{int(threshold * 100)}", "filtered_methods.npy" diff --git a/HT_LLM/similarity/filter_methods_by_performance.py b/HT_LLM/similarity/filter_methods_by_performance.py index 9815273..11e2f9d 100644 --- a/HT_LLM/similarity/filter_methods_by_performance.py +++ b/HT_LLM/similarity/filter_methods_by_performance.py @@ -9,23 +9,43 @@ import numpy as np -threshold = 0.6 # minimum performance threshold for a method to be included +threshold = 0.0 # minimum performance threshold in at least one task=experiment +# and one run for a method to be included -performances = np.load("sample_data/ALL_ML_SCORES_real.npy", allow_pickle=True).item() -# print(performances.keys()) - -metric = "SVM balanced accuracy" # change to another performance key if needed output_root = Path("sample_data") / f"threshold_{int(threshold * 100)}" output_root.mkdir(parents=True, exist_ok=True) output_path = output_root / "filtered_methods.npy" -tasks = np.asarray(performances["task"]) -methods = np.asarray(performances["dFC method"]) -scores = np.asarray(performances[metric], dtype=float) +performances = np.load("sample_data/ALL_ML_SCORES_real.npy", allow_pickle=True).item() +# print(performances.keys()) + +metric = "SVM balanced accuracy" +embedding = "PLS" +group = "test" + +tasks = performances["task"] +methods = performances["dFC method"] +runs = performances["run"] + +all_embeddings = performances["embedding"] +all_groups = performances["group"] +all_metric_scores = performances[metric] + +# Filter for scores that have a specific embedding and group using zip, +# and keep the corresponding task, dFC method, and run labels for each score +filtered_rows = [ + (task, method, run, score) + for emb, grp, task, method, run, score in zip( + all_embeddings, all_groups, tasks, methods, runs, all_metric_scores + ) + if emb == embedding and grp == group +] +# print(filtered_rows) # If several runs exist for the same (task, dFC method), keep the best run's score. best_score_by_task_method = {} -for task, method, score in zip(tasks, methods, scores): +for tuple_row in filtered_rows: + task, method, run, score = tuple_row if np.isnan(score): continue @@ -33,6 +53,13 @@ if key not in best_score_by_task_method or score > best_score_by_task_method[key]: best_score_by_task_method[key] = float(score) +# Check +test_scores = np.asarray(list(best_score_by_task_method.values()), dtype=float) +print( + f"Test scores - Min: {np.min(test_scores)}, Max: {np.max(test_scores)}, Mean: {np.mean(test_scores)}" +) + + # Keep methods that have at least one task with best-run score >= threshold. eligible_methods = { method diff --git a/HT_LLM/similarity/heatmaps_feature_similarity.py b/HT_LLM/similarity/heatmaps_feature_similarity.py index 8190e07..c6abdea 100644 --- a/HT_LLM/similarity/heatmaps_feature_similarity.py +++ b/HT_LLM/similarity/heatmaps_feature_similarity.py @@ -18,7 +18,7 @@ # %% # Filter methods in heatmaps by performance threshold -threshold = 0.6 # must exist; else, run filter_methods_by_performance.py with the desired threshold first +threshold = 0.0 # must exist; else, run filter_methods_by_performance.py with the desired threshold first eligible_methods_path = os.path.join( "sample_data", f"threshold_{int(threshold * 100)}", "filtered_methods.npy" diff --git a/HT_LLM/similarity/algorithm_feature_scatterplot.py b/HT_LLM/similarity/scatterplot_algorithm_feature.py similarity index 54% rename from HT_LLM/similarity/algorithm_feature_scatterplot.py rename to HT_LLM/similarity/scatterplot_algorithm_feature.py index 509f2a4..92c028f 100644 --- a/HT_LLM/similarity/algorithm_feature_scatterplot.py +++ b/HT_LLM/similarity/scatterplot_algorithm_feature.py @@ -6,9 +6,10 @@ import numpy as np import pandas as pd import seaborn as sns +import statsmodels.api as sm from matplotlib.lines import Line2D -threshold = 0.6 # minimum performance threshold for a method to be included +threshold = 0.0 # minimum performance threshold for a method to be included print(f"Method Filtering Threshold: {threshold}") OUTPUT_DIR = ( Path("HT_LLM/similarity/scatterplot_algorithm_feature_results") @@ -56,10 +57,10 @@ # Use a separate cutoff for categorizing whether each method in a pair is high-performing. # If this equals `threshold`, most pairs may be classified as high-high because the -# upstream files are already filtered using `threshold` +# upstream files are already filtered for performance using `threshold` # Note, however, that upstream files are filtered for at least 1 experiment/task (more lenient), whereas # this threshold is applied to the average performance across all experiments/tasks for each method. -high_performance_threshold = threshold +high_performance_threshold = 0.60 scatter_df["performance_pair_type"] = np.select( [ @@ -81,8 +82,8 @@ ) -### Plot -fig, ax = plt.subplots(figsize=(12, 8)) +##### Plotting ##### +fig, ax = plt.subplots(figsize=(14, 8)) performance_pair_palette = { "both high performance": "#D0021B", @@ -102,36 +103,105 @@ ax=ax, ) +# Add a linear trendline and report the model fit between the two plotted axes. +# i.e., assumes {feature_similarity} = beta_0 + beta_1 * {algorithm_similarity} + epsilon +trend_df = scatter_df[["algorithm_similarity", "feature_similarity"]].dropna() +if len(trend_df) >= 2: + trend_x = trend_df["algorithm_similarity"].to_numpy(dtype=float) + trend_y = trend_df["feature_similarity"].to_numpy(dtype=float) + + trend_model = sm.OLS(trend_y, sm.add_constant(trend_x, has_constant="add")).fit() + + # Plot trendline using the fitted model + trendline_x = np.linspace(trend_x.min(), trend_x.max(), 100) + trendline_y = trend_model.predict(sm.add_constant(trendline_x, has_constant="add")) + ax.plot( + trendline_x, + trendline_y, + color="darkgreen", + linestyle="-", + linewidth=1.5, + alpha=0.8, + label="_nolegend_", + ) + + # R^2 = proportion of variance in y explained by a linear relationship with x + # p-value = probability of observing a slope as extreme as the fitted slope if \ + # the null hypothesis (slope = 0) is true (i.e., prob. of observing by chance) + # Note: pvalues[1] is the slope (beta_1) p-value. Tests H_0: beta_1 = 0 vs. H_A: beta_1 != 0 + slope_p_value = trend_model.pvalues[1] + + ax.text( + 0.98, + 0.03, + f"$R^2$ = {trend_model.rsquared:.3f}\nOLS slope $p$ = {slope_p_value:.3g}", + transform=ax.transAxes, + ha="right", + va="bottom", + fontsize=9, + color="darkgreen", + bbox={ + "boxstyle": "round,pad=0.5", + "facecolor": "white", + "edgecolor": "grey", + "alpha": 0.85, + }, + ) + print(f"OLS slope p-value: {slope_p_value}") + # Label certain points by their method pair name for identification -high_as_threshold = 0.6 # For vertical line; Only label points with algorithm similarity above this threshold -high_fs_threshold = 0.6 # For horizontal line -counter = 0 +high_as_threshold = 0.7 # For vertical line; Only label points with algorithm similarity above this threshold +high_fs_threshold = 0.7 # For horizontal line +labelled_pairs = ( + scatter_df.loc[scatter_df["algorithm_similarity"] > high_as_threshold, "pair_key"] + .dropna() + .astype(str) + .tolist() +) +labelled_methods = sorted( + {method for pair_key in labelled_pairs for method in str(pair_key).split(" x ")} +) +method_id_by_name = { + method: method_id for method_id, method in enumerate(labelled_methods, start=1) +} + + +def encode_pair_key(pair_key): + """Replace method names in a pair key with their numeric method IDs to avoid overlapping labels in plot.""" + return " x ".join( + str(method_id_by_name[method]) for method in str(pair_key).split(" x ") + ) + + for idx, row in scatter_df.iterrows(): # Only label points satisfying this condition if row["algorithm_similarity"] > high_as_threshold: ax.text( row["algorithm_similarity"] + 0.005, # Add a slight x-offset manually row["feature_similarity"] + 0.005, # Add a slight y-offset manually - row["pair_key"], # The labelled text is the method pair's name + encode_pair_key(row["pair_key"]), # Label using compact numeric method IDs color=performance_pair_palette[row["performance_pair_type"]], weight="bold", size=7, ) - counter += 1 # Sanity checks for correct number of method pairs (not missing any) print("Feature pairs:", len(feature_pairs)) print("Algorithm pairs:", len(algorithm_pairs)) print("Merged pairs:", len(scatter_df)) -print("Plotted points:", counter) ax.set_xlim(0, 1) ax.set_ylim(-0.2, 1) ax.axvline(x=high_as_threshold, color="grey", linestyle="--") ax.axhline(y=high_fs_threshold, color="grey", linestyle="--") +# Performance legend legend_handles, legend_labels = ax.get_legend_handles_labels() -size_title_idx = legend_labels.index("absolute_performance_difference") +legend_labels = [ + "performance_difference" if label == "absolute_performance_difference" else label + for label in legend_labels +] +size_title_idx = legend_labels.index("performance_difference") for label_idx in range(size_title_idx + 1, len(legend_labels)): try: legend_labels[label_idx] = f"{float(legend_labels[label_idx]):.0%}" @@ -140,23 +210,47 @@ legend_handles.insert(size_title_idx, Line2D([], [], linestyle="none")) legend_labels.insert(size_title_idx, "") -legend = ax.legend( - title="SVM Balanced Accuracy Performance\n", +perf_legend = ax.legend( + title="SVM Balanced Accuracy\n", handles=legend_handles, labels=legend_labels, loc="lower left", - bbox_to_anchor=(1.02, 0), + bbox_to_anchor=(1.01, 0), fontsize=8, title_fontsize=9, ) -# legend.get_title().set_fontweight("bold") -for legend_text in legend.get_texts(): +# perf_legend.get_title().set_fontweight("bold") +for legend_text in perf_legend.get_texts(): if legend_text.get_text() in [ "performance_pair_type", - "absolute_performance_difference", + "performance_difference", ]: legend_text.set_fontweight("bold") +ax.add_artist( + perf_legend +) # Re-add the first legend back to the axes (don't overwrite it) + + +# Method IDs legend +if method_id_by_name: + method_handles = [Line2D([], [], linestyle="none") for _ in method_id_by_name] + method_labels = [ + f"{method_id}: {method}" for method, method_id in method_id_by_name.items() + ] + method_legend = ax.legend( + title="Labelled Methods", + handles=method_handles, + labels=method_labels, + loc="upper left", + bbox_to_anchor=(1.01, 1), + fontsize=8, + title_fontsize=9, + handlelength=0, + handletextpad=0, + ) + # method_legend.get_title().set_fontweight("bold") + ax.minorticks_on() ax.set_xlabel("Algorithm similarity") ax.set_ylabel("dFC feature similarity") @@ -166,7 +260,11 @@ y=1.02, ) -plt.tight_layout() +# Make the full figure wider and reserve the extra width for legends on the right. +# The scatterplot axes stay about as wide as before; the added figure width becomes +# the outside legend area. +fig.subplots_adjust(right=0.75) + plt.savefig( OUTPUT_DIR / "algorithm_vs_feature_similarity_scatter.png", dpi=600,