diff --git a/.gitignore b/.gitignore index a41f38a..f6e516c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,12 @@ __pycache__ sample_data/ +HT_LLM/similarity/*_results/ + +# HPC computing related +*.sh +slurm_out/ + # build related pydfc.egg-info build diff --git a/.spyproject/config/backups/codestyle.ini.bak b/.spyproject/config/backups/codestyle.ini.bak new file mode 100644 index 0000000..0f54b4c --- /dev/null +++ b/.spyproject/config/backups/codestyle.ini.bak @@ -0,0 +1,8 @@ +[codestyle] +indentation = True +edge_line = True +edge_line_columns = 79 + +[main] +version = 0.2.0 + diff --git a/.spyproject/config/backups/encoding.ini.bak b/.spyproject/config/backups/encoding.ini.bak new file mode 100644 index 0000000..a17aced --- /dev/null +++ b/.spyproject/config/backups/encoding.ini.bak @@ -0,0 +1,6 @@ +[encoding] +text_encoding = utf-8 + +[main] +version = 0.2.0 + diff --git a/.spyproject/config/backups/vcs.ini.bak b/.spyproject/config/backups/vcs.ini.bak new file mode 100644 index 0000000..fd66eae --- /dev/null +++ b/.spyproject/config/backups/vcs.ini.bak @@ -0,0 +1,7 @@ +[vcs] +use_version_control = False +version_control_system = + +[main] +version = 0.2.0 + diff --git a/.spyproject/config/backups/workspace.ini.bak b/.spyproject/config/backups/workspace.ini.bak new file mode 100644 index 0000000..e4c8489 --- /dev/null +++ b/.spyproject/config/backups/workspace.ini.bak @@ -0,0 +1,12 @@ +[workspace] +restore_data_on_startup = True +save_data_on_exit = True +save_history = True +save_non_project_files = False +project_type = 'empty-project-type' +recent_files = ['examples/dFC_methods_demo.py', '../test_comp.py'] + +[main] +version = 0.2.0 +recent_files = [] + diff --git a/.spyproject/config/codestyle.ini b/.spyproject/config/codestyle.ini new file mode 100644 index 0000000..0f54b4c --- /dev/null +++ b/.spyproject/config/codestyle.ini @@ -0,0 +1,8 @@ +[codestyle] +indentation = True +edge_line = True +edge_line_columns = 79 + +[main] +version = 0.2.0 + diff --git a/.spyproject/config/defaults/defaults-codestyle-0.2.0.ini b/.spyproject/config/defaults/defaults-codestyle-0.2.0.ini new file mode 100644 index 0000000..0b95e5c --- /dev/null +++ b/.spyproject/config/defaults/defaults-codestyle-0.2.0.ini @@ -0,0 +1,5 @@ +[codestyle] +indentation = True +edge_line = True +edge_line_columns = 79 + diff --git a/.spyproject/config/defaults/defaults-encoding-0.2.0.ini b/.spyproject/config/defaults/defaults-encoding-0.2.0.ini new file mode 100644 index 0000000..0ce193c --- /dev/null +++ b/.spyproject/config/defaults/defaults-encoding-0.2.0.ini @@ -0,0 +1,3 @@ +[encoding] +text_encoding = utf-8 + diff --git a/.spyproject/config/defaults/defaults-vcs-0.2.0.ini b/.spyproject/config/defaults/defaults-vcs-0.2.0.ini new file mode 100644 index 0000000..ee25483 --- /dev/null +++ b/.spyproject/config/defaults/defaults-vcs-0.2.0.ini @@ -0,0 +1,4 @@ +[vcs] +use_version_control = False +version_control_system = + diff --git a/.spyproject/config/defaults/defaults-workspace-0.2.0.ini b/.spyproject/config/defaults/defaults-workspace-0.2.0.ini new file mode 100644 index 0000000..2a73ab7 --- /dev/null +++ b/.spyproject/config/defaults/defaults-workspace-0.2.0.ini @@ -0,0 +1,6 @@ +[workspace] +restore_data_on_startup = True +save_data_on_exit = True +save_history = True +save_non_project_files = False + diff --git a/.spyproject/config/encoding.ini b/.spyproject/config/encoding.ini new file mode 100644 index 0000000..a17aced --- /dev/null +++ b/.spyproject/config/encoding.ini @@ -0,0 +1,6 @@ +[encoding] +text_encoding = utf-8 + +[main] +version = 0.2.0 + diff --git a/.spyproject/config/vcs.ini b/.spyproject/config/vcs.ini new file mode 100644 index 0000000..fd66eae --- /dev/null +++ b/.spyproject/config/vcs.ini @@ -0,0 +1,7 @@ +[vcs] +use_version_control = False +version_control_system = + +[main] +version = 0.2.0 + diff --git a/.spyproject/config/workspace.ini b/.spyproject/config/workspace.ini new file mode 100644 index 0000000..43f664a --- /dev/null +++ b/.spyproject/config/workspace.ini @@ -0,0 +1,12 @@ +[workspace] +restore_data_on_startup = True +save_data_on_exit = True +save_history = True +save_non_project_files = False +project_type = 'empty-project-type' +recent_files = ['../test_comp.py', 'examples/dFC_methods_demo.py', 'task_dFC/ML.py', 'pydfc/__init__.py', 'pydfc/dfc_utils.py', 'pydfc/ml_utils.py', 'pydfc/task_utils.py'] + +[main] +version = 0.2.0 +recent_files = [] + diff --git a/HT_LLM/similarity/algorithm_similarity.py b/HT_LLM/similarity/algorithm_similarity.py new file mode 100644 index 0000000..5bf2751 --- /dev/null +++ b/HT_LLM/similarity/algorithm_similarity.py @@ -0,0 +1,428 @@ +""" +Algorithm Similarity (AS): Bag of Operations (BOO) with operation counts. + +Idea +---- +Parse each dFC method's .py file into an Abstract Syntax Tree (AST), extract every +function/class call that touches a known numerical/scientific library (numpy, scipy, +sklearn, hmmlearn, statsmodels, ...), resolve it to a fully-qualified name +(e.g. "np.corrcoef" -> "numpy.corrcoef"), and represent the method as a BAG +(multiset) of operations. + +Unlike a plain set-based Jaccard score, this implementation keeps the number of +times each operation appears. Pairwise AS is the WEIGHTED Jaccard similarity: + + sum(min(count_a[op], count_b[op])) / sum(max(count_a[op], count_b[op])) + +This metric asks: "how similar are the methods' scripts in both the operations they use and how +often they use them?" + +Usage +----- +python BOO_algorithm_similarity.py /path/to/dfc_methods/*.py +""" + +import ast +import csv +import itertools +import json +import re +import sys +from collections import Counter +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +import seaborn as sns +from scipy.cluster.hierarchy import leaves_list, linkage +from scipy.spatial.distance import squareform + +# Only calls whose resolved root module starts with one of these are kept as +# "operations". Everything else (self.*, local helper functions, plain +# built-ins like zip/len/range) is treated as implementation glue and dropped. +LIBRARY_PREFIXES = ( + "numpy", + "scipy", + "sklearn", + "hmmlearn", + "statsmodels", + "ksvd", + "pycwt", +) + +NON_AIGM_SET = { + "CAP", + "Windowless", + "Clustering", + "DiscreteHMM", + "ContinuousHMM", + "Time-Freq", + "SlidingWindow", +} +NON_AIGM_COLOR = "darkorange" + +DEFAULT_OUTPUT_DIR = "HT_LLM/similarity/algorithm_similarity_results" +METRIC_NAME = "BOO_weighted_jaccard" +EXCLUDED_METHOD_FILES = {"__init__.py", "base_dfc_method.py"} + + +def _build_import_map(tree): + """Map local alias -> fully-qualified module/object path, from this file's imports.""" + import_map = {} + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + local_name = alias.asname or alias.name.split(".")[0] + import_map[local_name] = alias.name + elif isinstance(node, ast.ImportFrom): + if node.module is None: + continue + for alias in node.names: + local_name = alias.asname or alias.name + import_map[local_name] = f"{node.module}.{alias.name}" + return import_map + + +def _dotted_name(node): + """Best-effort reconstruction of a dotted attribute chain, e.g. Attribute(Attribute(Name)) -> 'a.b.c'.""" + parts = [] + while isinstance(node, ast.Attribute): + parts.append(node.attr) + node = node.value + if isinstance(node, ast.Name): + parts.append(node.id) + return ".".join(reversed(parts)) + return None # call target is something we can't statically resolve (e.g. a subscript) + + +def _resolve_call_name(raw_name, import_map): + """Resolve an imported alias in a call name to its fully-qualified name.""" + head, *rest = raw_name.split(".") + if head in import_map: + return ".".join([import_map[head]] + rest) + return raw_name + + +def _is_library_operation(resolved_name): + """Return True when a resolved call belongs to one of the tracked libraries.""" + return resolved_name.split(".")[0] in LIBRARY_PREFIXES + + +def extract_operation_counts(filepath, verbose=False): + """Return Counter({operation_name: count}) for resolved library operations.""" + with open(filepath, "r") as f: + source = f.read() + tree = ast.parse(source) + import_map = _build_import_map(tree) + + operation_counts = Counter() + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + func = node.func + if isinstance(func, ast.Name): + raw = func.id + elif isinstance(func, ast.Attribute): + raw = _dotted_name(func) + else: + continue + if raw is None: + continue + + resolved = _resolve_call_name(raw, import_map) + if _is_library_operation(resolved): + operation_counts[resolved] += 1 + # else: skip self.*, locally-defined helpers, and plain built-ins + + if verbose: + total_calls = sum(operation_counts.values()) + print( + f"\n{filepath} -> {len(operation_counts)} distinct operations, " + f"{total_calls} counted calls:" + ) + for op, count in sorted(operation_counts.items()): + print(f" {op}: {count}") + return operation_counts + + +def extract_operations(filepath, verbose=False): + """Return the set of distinct operations. Kept for backward compatibility.""" + return set(extract_operation_counts(filepath, verbose=verbose)) + + +def weighted_jaccard_similarity(counts_a, counts_b): + """Compute weighted Jaccard similarity between two operation-count bags.""" + operations = set(counts_a) | set(counts_b) # complete set of all operations + + overlap = sum(min(counts_a[op], counts_b[op]) for op in operations) + union = sum(max(counts_a[op], counts_b[op]) for op in operations) + similarity = overlap / union + + if not operations: # neither script captured any operations from tracked libraries + similarity = 0.0 + + return overlap, union, similarity + + +def _to_readable_method_name(filepath): + """Read the method display name from the script's class-level MEASURE_NAME.""" + with open(filepath, "r", encoding="utf-8") as f: + tree = ast.parse(f.read(), filename=str(filepath)) + + for node in ast.walk(tree): + if not isinstance(node, ast.ClassDef): + continue + + for statement in node.body: + if isinstance(statement, ast.Assign): + targets = statement.targets + value = statement.value + elif isinstance(statement, ast.AnnAssign): + targets = [statement.target] + value = statement.value + else: + continue + + has_measure_name = any( + isinstance(target, ast.Name) and target.id == "MEASURE_NAME" + for target in targets + ) + + if has_measure_name: + if isinstance(value, ast.Constant) and isinstance(value.value, str): + return value.value + raise ValueError( + f"MEASURE_NAME in {filepath} must be a string literal to be used as a label." + ) + + raise ValueError(f"No class-level MEASURE_NAME string found in {filepath}.") + + +def _make_unique_labels(filepaths): + """Generate unique labels from each method script's MEASURE_NAME.""" + counts = {} + labels = [] + for filepath in filepaths: + base = _to_readable_method_name(filepath) + counts[base] = counts.get(base, 0) + 1 + labels.append(base if counts[base] == 1 else f"{base}_{counts[base]}") + return labels + + +def _hierarchical_cluster_order(matrix, cluster_method="average"): + """Return indices that order similar methods next to each other.""" + + if matrix.shape[0] < 2: + return np.arange(matrix.shape[0]) + + distance = (1 - matrix) / 2 + distance = (distance + distance.T) / 2 + np.fill_diagonal(distance, 0) + condensed = squareform(distance) + + linkage_matrix = linkage(condensed, method=cluster_method) + return leaves_list(linkage_matrix) + + +def plot_similarity_heatmap( + matrix, + labels, + title="Algorithm Similarity (Bag of Operations with Weighted Jaccard)", + annot=False, + figsize=(10, 8), + cluster=True, + cluster_method="average", +): + """Return a heatmap figure for a saved AS matrix. + + Note: The ordering is controlled by this script's own hierarchical clustering, + while the visual formatting mirrors the feature similarity's heatmap style. + """ + labels = list(labels) + matrix = np.squeeze(matrix) + + highlight_color = NON_AIGM_COLOR + highlight_method_names = NON_AIGM_SET + + if cluster: + order = _hierarchical_cluster_order(matrix, cluster_method=cluster_method) + matrix = matrix[np.ix_(order, order)] + labels = [labels[i] for i in order] + + fig, ax = plt.subplots(figsize=figsize) + sns.heatmap( + matrix, + annot=annot, + xticklabels=labels, + yticklabels=labels, + cmap="viridis", + vmin=0.0, + vmax=1.0, + ax=ax, + ) + + ax.set_title(title, fontsize=12) + ax.set_xlabel("dFC Method", fontsize=11) + ax.set_ylabel("dFC Method", fontsize=11) + ax.set_xticklabels(labels, rotation=45, ha="right", fontsize=6) + ax.set_yticklabels(labels, rotation=0, fontsize=6) + + # Highlight selected method labels in orange, leave all other labels black. + for tick_label in ax.get_xticklabels(): + tick_label.set_color( + highlight_color + if tick_label.get_text() in highlight_method_names + else "black" + ) + + for tick_label in ax.get_yticklabels(): + tick_label.set_color( + highlight_color + if tick_label.get_text() in highlight_method_names + else "black" + ) + + plt.tight_layout() + return fig, ax + + +def save_similarity_outputs(output_dir, labels, matrix, table): + """Save the matrix, ordered labels, pairwise table, and heatmap.""" + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + np.save(output_dir / f"AS_{METRIC_NAME}_matrix.npy", matrix) + np.save( + output_dir / f"AS_{METRIC_NAME}_method_names.npy", np.array(labels, dtype=object) + ) + + with open( + output_dir / f"AS_{METRIC_NAME}_pairs.csv", "w", newline="", encoding="utf-8" + ) as f: + fieldnames = [ + "method_a", + "method_b", + "source_a", + "source_b", + "similarity", + "weighted_overlap", + "weighted_union", + "n_shared_distinct", + "n_distinct_ops_a", + "n_distinct_ops_b", + "n_total_ops_a", + "n_total_ops_b", + "shared_operation_counts", + ] + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(table) + + try: + fig, _ = plot_similarity_heatmap(matrix, labels, cluster=True) + except ImportError: + print( + "Skipping heatmap export because matplotlib is not available in this environment." + ) + else: + fig.savefig( + str(output_dir / f"AS_{METRIC_NAME}_heatmap.png"), + dpi=600, + bbox_inches="tight", + ) + fig.savefig( + str(output_dir / f"AS_{METRIC_NAME}_heatmap.pdf"), + bbox_inches="tight", + ) + + plt.close(fig) + + +def load_similarity_outputs(output_dir): + """Load the saved AS matrix, ordered method names/labels, and searchable pairwise table.""" + output_dir = Path(output_dir) + matrix = np.load(output_dir / f"AS_{METRIC_NAME}_matrix.npy") + labels = np.load( + output_dir / f"AS_{METRIC_NAME}_method_names.npy", allow_pickle=True + ).tolist() + with open( + output_dir / f"AS_{METRIC_NAME}_pairs.csv", "r", newline="", encoding="utf-8" + ) as f: + table = list(csv.DictReader(f)) + return labels, matrix, table + + +def main(filepaths): + source_paths = [ + str(Path(fp)) for fp in filepaths if Path(fp).name not in EXCLUDED_METHOD_FILES + ] + + if not source_paths: + raise ValueError( + "No concrete dFC method files provided. Pass method scripts such as " + "pydfc/dfc_methods/*.py; base_dfc_method.py and __init__.py are skipped." + ) + + labels = _make_unique_labels(source_paths) + operation_bags = {} + for label, path in zip(labels, source_paths): + operation_bags[label] = extract_operation_counts(path, verbose=True) + + print("\nPairwise Algorithm Similarity (weighted Jaccard over operation counts):") + names = list(operation_bags.keys()) + + # Initialize with zeros so the main diagonal stays 0.0 for simple visualization. + alg_sim = np.zeros((len(names), len(names)), dtype=float) + + pairwise_rows = [] + + for i, j in itertools.combinations(range(len(names)), 2): # only off diagonal pairs + method_a = names[i] + method_b = names[j] + counts_a = operation_bags[method_a] + counts_b = operation_bags[method_b] + weighted_overlap, weighted_union, similarity = weighted_jaccard_similarity( + counts_a, counts_b + ) + alg_sim[i, j] = similarity + alg_sim[j, i] = similarity + + shared = sorted(set(counts_a) & set(counts_b)) + shared_counts = { + op: {"method_a": counts_a[op], "method_b": counts_b[op]} for op in shared + } + pairwise_rows.append( + { + "method_a": method_a, + "method_b": method_b, + "source_a": source_paths[i], + "source_b": source_paths[j], + "similarity": similarity, + "weighted_overlap": weighted_overlap, + "weighted_union": weighted_union, + "n_shared_distinct": len(shared), + "n_distinct_ops_a": len(counts_a), + "n_distinct_ops_b": len(counts_b), + "n_total_ops_a": sum(counts_a.values()), + "n_total_ops_b": sum(counts_b.values()), + "shared_operation_counts": json.dumps(shared_counts, sort_keys=True), + } + ) + + print( + f"{method_a:35s} vs {method_b:35s} AS = {similarity:.3f} " + f"overlap/union = {weighted_overlap}/{weighted_union} " + f"shared = {shared}" + ) + + save_similarity_outputs( + DEFAULT_OUTPUT_DIR, + names, + alg_sim, + pairwise_rows, + ) + print(f"Saved outputs to {DEFAULT_OUTPUT_DIR}/") + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/HT_LLM/similarity/compute_feature_similarity.py b/HT_LLM/similarity/compute_feature_similarity.py new file mode 100644 index 0000000..ebed76b --- /dev/null +++ b/HT_LLM/similarity/compute_feature_similarity.py @@ -0,0 +1,164 @@ +import pickle +import sys +from collections import defaultdict +from pathlib import Path + +import numpy as np + +from pydfc.comparison import SimilarityAssessment # pip install pydfc + +# FULL PATH usually looks like: +# "{path_to_datasets}/{dataset_id}/derivatives/dFC_assessed/{subject_id}/{session_id}/*.npy" +# where * has the format "dFC_{identifier}_{method_number}" +# where identifier has the format "{session_id}_{task_id}_{run_id}" +# However, session_id and run_id could be absent, and their keys would be set to None in the output dictionary! + +if len(sys.argv) < 2: + print("Missing a path to the datasets directory") + print("Usage: sbatch run_dfc.sh ") + sys.exit(1) + +path_to_datasets = sys.argv[1] + +root = Path(path_to_datasets) + + +# Create a dictionary to store similarity assessment results +# of the form: similarity[dataset_id][subject_id][session_id][run_id][task_id] = {"matrix": matrix, "methods": method_numbers} +# where matrix.shape = (1, num_methods, num_methods) and contains the similarity values between methods + +similarity = defaultdict( + lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(dict))) +) + +for dataset_dir in root.iterdir(): + + if not dataset_dir.is_dir(): + continue + + if not dataset_dir.name.startswith("ds"): + continue + + dataset_id = dataset_dir.name + + dfc_dir = dataset_dir / "derivatives" / "dFC_assessed" + + if not dfc_dir.is_dir(): + print(f"Skipping {dataset_id} since /derivatives/dFC_assessed not found") + continue + + for subject_dir in dfc_dir.iterdir(): + + if not subject_dir.is_dir(): + continue + + subject_id = subject_dir.name + + # If no session folders, treat the subject directory as the session directory + # to avoid file path issues. If this case, session_id will be set to None later. + session_dirs = [ + p for p in subject_dir.iterdir() if p.is_dir() and p.name.startswith("ses-") + ] + + if not session_dirs: + session_dirs = [subject_dir] + + for session_dir in session_dirs: + + # Group files by identifier + files_by_identifier = defaultdict(list) + + for npy_file in session_dir.glob("dFC_*.npy"): + + filename = npy_file.stem # removed .npy + + _, rest = filename.split( + "_", 1 + ) # e.g., "dFC", "ses-wave1bas_task-Stroop_run-2_24" + identifier, method_number = rest.rsplit( + "_", 1 + ) # e.g., "ses-wave1bas_task-Stroop_run-2", "24" + + files_by_identifier[identifier].append((int(method_number), npy_file)) + + # Process one identifier at a time (similarity across methods) + for identifier, file_info in files_by_identifier.items(): + + # Initialize session_id and run_id as None in case they don't exist + session_id = None + run_id = None + task_id = None # must exist, see check later to catch error. + + # Get session, task, and run from identifier (if they exist) + for part in identifier.split("_"): + if part.startswith("ses-"): # e.g., "ses-wave1bas" + session_id = part + + elif part.startswith("run-"): # e.g., "run-2" + run_id = part + + elif part.startswith("task-"): # e.g., "task-Stroop" + task_id = part + + else: + print( + f"Warning: Unrecognized part '{part}' in identifier '{identifier}' \ + of subject '{subject_id}' in dataset '{dataset_id}'. Ignoring this part." + ) + + if task_id is None: + print( + f"Error: task_id not found in identifier '{identifier}' of subject '{subject_id}' \ + in dataset '{dataset_id}'. Skipping this file." + ) + continue + + # Sort methods numerically + file_info.sort(key=lambda x: x[0]) + + method_numbers = [] + + # This is a list of the dFC objects from various methods + # that share the same identifier i.e., they came from the same + # BOLD time series, but they were computed using different methods + # Each dFC in the list is recognized as a dFC object by pydfc + dFC_lst = [] + + for method_num, path in file_info: + method_numbers.append(method_num) + dFC_lst.append(np.load(path, allow_pickle=True).item()) + + # Note: type(output) = dict with + # dict_keys(['measure_lst', 'TS_info_lst', 'common_TRs', 'time_record_dict', 'all']) + similarity_assessment = SimilarityAssessment(dFC_lst=dFC_lst) + output = similarity_assessment.assess_similarity_fast(dFC_lst=dFC_lst) + + similarity[dataset_id][subject_id][session_id][run_id][task_id] = { + "matrix": output, + "methods": method_numbers, + } + + print(f"Finished processing subject {subject_id} in dataset {dataset_id}") + + +output_dir = Path( + "/home/kinichen/scratch/data/pydfc_validator/similarity_assessments_complete" +) +output_dir.mkdir(parents=True, exist_ok=True) +output_file = output_dir / "similarity.pkl" + + +# Convert to normal dict for pickling. Need to do recursively because of the nested defaultdicts. +def to_dict(d): + if isinstance(d, defaultdict): + return {k: to_dict(v) for k, v in d.items()} + return d + + +similarity = to_dict(similarity) + +with open(output_file, "wb") as f: + pickle.dump(similarity, f) + + +print(f"Saved results to: {output_file}") diff --git a/HT_LLM/similarity/heatmaps_feature_similarity.py b/HT_LLM/similarity/heatmaps_feature_similarity.py new file mode 100644 index 0000000..2fb45b7 --- /dev/null +++ b/HT_LLM/similarity/heatmaps_feature_similarity.py @@ -0,0 +1,441 @@ +# %% +# Note: if similarity.pkl file is too large (>5 GB), need to request more memory on a compute node via: +# $ salloc --account=def- --mem=128G --cpus-per-task=8 --time=4:00:00 +# or submit a batch job + +# %% +import os +import pickle + +import matplotlib.pyplot as plt +import numpy as np +import seaborn as sns +from scipy.cluster.hierarchy import leaves_list, linkage +from scipy.spatial.distance import squareform + +DEFAULT_OUTPUT_DIR = "HT_LLM/similarity/feature_similarity_results" +os.makedirs(f"{DEFAULT_OUTPUT_DIR}/pdf", exist_ok=True) +os.makedirs(f"{DEFAULT_OUTPUT_DIR}/png", exist_ok=True) + + +NON_AIGM_SET = { + "CAP", + "Windowless", + "Clustering", + "DiscreteHMM", + "ContinuousHMM", + "Time-Freq", + "SlidingWindow", +} +NON_AIGM_COLOR = "darkorange" + + +# %% +root = "/home/kinichen/scratch/data/pydfc_validator/similarity_assessments_complete" + +with open(f"{root}/similarity.pkl", "rb") as f: + similarity = pickle.load(f) + +print( + "Datasets:", similarity.keys() +) # layer 1 of hierarchy is datasets, then subjects, sessions, runs, tasks, etc. (pydFC objects) + + +# %% +dataset_id = "ds003465" +subject_id = "sub-f1027ao" +session_id = "ses-wave1bas" +run_id = "run-2" +task_id = "task-Stroop" + +sim_ex = similarity[dataset_id][subject_id][session_id][run_id][task_id] +measures = sim_ex["matrix"]["measure_lst"] +methods = [ + method.MEASURE_NAME for method in measures +] # extract method names from the pydfc dfc_methods objects +print("Example methods:", methods[:5]) + +matrix_ex = sim_ex["matrix"]["all"]["spearman"] # similarity matrix for all methods +print("Example matrix shape:", matrix_ex.shape) + + +# %% +######### Helper functions to collect and aggregate similarity matrices based on filters +# for various levels (dataset, subject, session, run, task) ######### + + +def collect_similarity_matrices( + similarity: dict, + dataset_id=None, + subject_id=None, + session_id=None, + run_id=None, + task_id=None, + similarity_key="all", + metric="spearman", +): + """ + Collect all similarity matrices matching the specified filters. If a filter is None, + it matches all values for that level and aggregates over/across it. + + Returns: + matrices: list of np.ndarray of shape (1, n_methods, n_methods) + """ + + matrices = [] + + for ds, ds_data in similarity.items(): + + if dataset_id is not None and ds != dataset_id: + continue + + for sub, sub_data in ds_data.items(): + + if subject_id is not None and sub != subject_id: + continue + + for ses, ses_data in sub_data.items(): + + if session_id is not None and ses != session_id: + continue + + for run, run_data in ses_data.items(): + + if run_id is not None and run != run_id: + continue + + for task, task_data in run_data.items(): + + if task_id is not None and task != task_id: + continue + + matrices.append(task_data["matrix"][similarity_key][metric]) + + return matrices + + +def aggregate_similarity_matrices(matrices, aggregation="mean"): + """ + Parameters + ---------- + matrices : list of arrays + Each array has shape (1, n_methods, n_methods) + + Returns + ------- + aggregated_matrix : np.ndarray + Shape (n_methods, n_methods) + + n_matrices : int + Number of matrices contributing to the aggregation (sample size) + """ + + if len(matrices) == 0: + raise ValueError("No matrices found.") + + arr = np.concatenate(matrices, axis=0) + + if aggregation == "mean": + aggregated = np.mean(arr, axis=0) + + elif aggregation == "median": + aggregated = np.median(arr, axis=0) + + elif aggregation == "std": + aggregated = np.std(arr, axis=0) + + else: + raise ValueError(f"Unknown aggregation: {aggregation}") + + return aggregated, len(matrices) + + +def plot_similarity_heatmap( + matrix, + aggregation_size=None, + method_names=methods, + title="Similarity Heatmap", + annot=False, + figsize=(10, 8), + cluster=False, + cluster_method="average", +): + + method_names = list(method_names) + + # Highlight non-AIGM names in a different color + highlight_color = NON_AIGM_COLOR + highlight_method_names = NON_AIGM_SET + matrix = np.squeeze(matrix) + + # Optional hierarchical clustering to reorder methods based on similarity to each other + if cluster: + + # Convert similarity to distance + distance = (1 - matrix) / 2 + + # Ensure exact symmetry and diagonal is 0 + distance = (distance + distance.T) / 2 + np.fill_diagonal(distance, 0) + + # Convert to condensed format required by scipy + condensed = squareform(distance) + + # Hierarchical clustering + Z = linkage(condensed, method=cluster_method) + + # Obtain reordered indices + order = leaves_list(Z) + + # Reorder matrix + matrix = matrix[np.ix_(order, order)] + + # Reorder labels + method_names = [method_names[i] for i in order] + + plotted_methods_order = list(method_names) + + plt.figure(figsize=figsize) + + ax = sns.heatmap( + matrix, + annot=annot, + xticklabels=method_names, + yticklabels=method_names, + cmap="viridis", + vmin=-0.2, + vmax=1.0, + ) + + if aggregation_size is not None: + title += f" (n={aggregation_size})" + + plt.title(title, fontsize=12) + plt.xlabel("dFC Method", fontsize=11) + plt.ylabel("dFC Method", fontsize=11) + plt.xticks(rotation=45, ha="right", fontsize=6) + plt.yticks(rotation=0, fontsize=6) + + # Highlight selected method labels in a different color. + for tick_label in ax.get_xticklabels(): + tick_label.set_color( + highlight_color + if tick_label.get_text() in highlight_method_names + else "black" + ) + + for tick_label in ax.get_yticklabels(): + tick_label.set_color( + highlight_color + if tick_label.get_text() in highlight_method_names + else "black" + ) + + plt.tight_layout() + + plt.savefig(f"{DEFAULT_OUTPUT_DIR}/pdf/{title}.pdf", bbox_inches="tight") + plt.savefig(f"{DEFAULT_OUTPUT_DIR}/png/{title}.png", dpi=600, bbox_inches="tight") + plt.close() + + return plotted_methods_order + + +""" +# OUTDATED: Individual heatmap for each experiment/dataset. UPDATED version below +# is a subplot of all experiments/datasets together + +# %% +### Average over everything (all subjects, sessions, runs, and datasets) for a specific TASK ### + +task_ids = sorted( + { + task + for ds_data in similarity.values() + for sub_data in ds_data.values() + for ses_data in sub_data.values() + for run_data in ses_data.values() + for task in run_data.keys() + } +) + +for task_id in task_ids: + + matrices = collect_similarity_matrices(similarity, task_id=task_id) + + aggregated, aggregation_size = aggregate_similarity_matrices( + matrices, aggregation="mean" + ) + + plot_similarity_heatmap(aggregated, aggregation_size, title=f"{task_id}") + + +# %% +### Average over everything for a specific DATASET ### + +dataset_ids = sorted(similarity.keys()) + +for dataset_id in dataset_ids: + + matrices = collect_similarity_matrices(similarity, dataset_id=dataset_id) + + aggregated, aggregation_size = aggregate_similarity_matrices( + matrices, aggregation="mean" + ) + + plot_similarity_heatmap(aggregated, aggregation_size, title=f"{dataset_id}") + +""" + +# %% +### Average over EVERYTHING ### + +matrices = collect_similarity_matrices(similarity) + +aggregated, aggregation_size = aggregate_similarity_matrices(matrices, aggregation="mean") + +methods_order = plot_similarity_heatmap( + aggregated, + aggregation_size, + title="Mean dFC feature similarity between methods", + cluster=True, +) + + +# %% +### Standard deviation over EVERYTHING ### + +# Measures which method pairs are more stable vs. more variable across filters + +matrices = collect_similarity_matrices(similarity) + +aggregated, aggregation_size = aggregate_similarity_matrices(matrices, aggregation="std") + +plot_similarity_heatmap( + aggregated, + aggregation_size, + title="Standard deviation of dFC feature similarity between methods", + method_names=methods_order, +) + + +# %% +### 3 x 3 subplot heatmap: mean similarity for each TASK ### + + +def plot_task_similarity_heatmap_grid( + similarity, + task_ids, + ordered_method_names, + original_method_names=methods, + aggregation="mean", + title="Task-Specific Mean dFC Feature Similarity", + figsize=(14, 12), + nrows=3, + ncols=3, + tick_fontsize=3, + title_fontsize=12, +): + """Plot one heatmap per task using a shared method ordering and colorbar.""" + + ordered_method_names = list(ordered_method_names) + original_method_names = list(original_method_names) + order = [original_method_names.index(name) for name in ordered_method_names] + + if len(task_ids) > nrows * ncols: + raise ValueError(f"Expected at most {nrows * ncols} tasks, got {len(task_ids)}.") + + highlight_color = NON_AIGM_COLOR + highlight_method_names = NON_AIGM_SET + + fig, axes = plt.subplots(nrows, ncols, figsize=figsize, constrained_layout=False) + axes = np.ravel(axes) + + # Dedicated colorbar axis placed off to the right of the subplot grid. + cbar_ax = fig.add_axes([0.92, 0.18, 0.02, 0.64]) + + for ax_idx, ax in enumerate(axes): + if ax_idx >= len(task_ids): + ax.axis("off") + continue + + task_id = task_ids[ax_idx] + matrices = collect_similarity_matrices(similarity, task_id=task_id) + aggregated, aggregation_size = aggregate_similarity_matrices( + matrices, + aggregation=aggregation, + ) + + matrix = np.squeeze(aggregated) + matrix = matrix[np.ix_(order, order)] + + sns.heatmap( + matrix, + ax=ax, + xticklabels=ordered_method_names, + yticklabels=ordered_method_names, + cmap="viridis", + vmin=-0.2, + vmax=1.0, + cbar=ax_idx == 0, + cbar_ax=cbar_ax if ax_idx == 0 else None, + square=True, + ) + + ax.set_title(f"{task_id} (n={aggregation_size})", fontsize=title_fontsize) + ax.set_xlabel("") + ax.set_ylabel("") + ax.tick_params(axis="x", labelrotation=45, labelsize=tick_fontsize) + ax.tick_params(axis="y", labelrotation=0, labelsize=tick_fontsize) + + for tick_label in ax.get_xticklabels(): + tick_label.set_horizontalalignment("right") + tick_label.set_color( + highlight_color + if tick_label.get_text() in highlight_method_names + else "black" + ) + + for tick_label in ax.get_yticklabels(): + tick_label.set_color( + highlight_color + if tick_label.get_text() in highlight_method_names + else "black" + ) + + fig.suptitle(title, fontsize=14, y=0.98) + # fig.supxlabel("dFC Method", fontsize=11, y=0.01) + # fig.supylabel("dFC Method", fontsize=11, x=0.01) + fig.subplots_adjust( + left=0.07, + right=0.90, + bottom=0.08, + top=0.93, + wspace=0.25, + hspace=0.35, + ) + + fig.savefig(f"{DEFAULT_OUTPUT_DIR}/pdf/{title}.pdf", bbox_inches="tight") + fig.savefig(f"{DEFAULT_OUTPUT_DIR}/png/{title}.png", dpi=600, bbox_inches="tight") + plt.close() + + +# Make task- (aka experiment-) specific subplot + +task_ids = sorted( + { + task + for ds_data in similarity.values() + for sub_data in ds_data.values() + for ses_data in sub_data.values() + for run_data in ses_data.values() + for task in run_data.keys() + } +) + +plot_task_similarity_heatmap_grid( + similarity=similarity, task_ids=task_ids, ordered_method_names=methods_order +) + + +# %% +print(f"Complete! Figures saved to {DEFAULT_OUTPUT_DIR}") diff --git a/README.rst b/README.rst index e774b97..f9ba123 100644 --- a/README.rst +++ b/README.rst @@ -165,3 +165,21 @@ If you are new to **pydfc**, we recommend starting with: This optional AI-assisted workflow is designed to complement — not replace — the documentation and example scripts. + +Generating New dFC Methods with AI +----------------------------------- + +You can ask an AI coding assistant (Claude, Copilot, Codex, etc.) to implement +brand-new dFC methods and add them directly to ``pydfc``. Just describe what +you want at whatever level of specificity feels right: + +- *"Generate 5 new creative dFC methods."* +- *"Generate 3 new state-based methods."* +- *"Implement a dFC method based on Granger causality."* +- *"Add a method that uses Riemannian geometry on covariance matrices."* +- *"Here is a paper — implement the method it describes."* (paste the PDF or text) + +The AI will read the existing codebase, follow the conventions in +``docs/ADDING_DFC_METHODS.md``, write the new method file, and register it in +``pydfc/dfc_methods/__init__.py`` so it works immediately alongside all other +methods. diff --git a/docs/ADDING_DFC_METHODS.md b/docs/ADDING_DFC_METHODS.md new file mode 100644 index 0000000..c139aeb --- /dev/null +++ b/docs/ADDING_DFC_METHODS.md @@ -0,0 +1,418 @@ +# Adding New dFC Methods to PydFC + +This guide summarizes the conventions for adding a dynamic functional +connectivity (dFC) method to PydFC. It is based on the current codebase patterns +and the validation workflow in `tests/test_validation`. + +PydFC methods should make their assumptions explicit. Based on the repository +context and Torabi et al., 2024, different dFC methods can produce substantially +different temporal estimates, so new methods should be treated as +assumption-dependent estimators rather than ground truth. + +## File Layout + +Use one Python file per dFC method: + +```text +pydfc/dfc_methods/my_new_method.py +``` + +Critical rule: method scripts must be self-sufficient. + +- Do not make a new method rely on an additional helper script in `pydfc/dfc_methods/` (for example, a shared `*_core.py` that contains required logic). +- Keep the method's full executable logic in its own method file so that each method remains portable and independently readable. +- Shared repository infrastructure imports (for example `BaseDFCMethod`, `DFC`, `TIME_SERIES`) are still expected. + +Do not put multiple concrete dFC methods in one module unless they are tightly +coupled variants that must share a public implementation. The established +package style is one method class per file, for example: + +```text +pydfc/dfc_methods/sliding_window.py +pydfc/dfc_methods/time_freq.py +pydfc/dfc_methods/cap.py +``` + +Each concrete method should inherit directly from: + +```python +from .base_dfc_method import BaseDFCMethod +``` + +Do not modify `base_dfc_method.py` just to add a method. + +## Required Class Shape + +A method class should define: + +- `__init__(self, **params)` +- class-level `MEASURE_NAME` string constant +- `measure_name` property +- `dFC(...)` or method-specific computation helpers +- `estimate_FCS(...)` +- `estimate_dFC(...)` + +`MEASURE_NAME` is required for automatic registry discovery in +`multi_analysis_utils.create_measure_obj`. + +State-free methods usually return `self` from `estimate_FCS`, because there are +no group-level functional connectivity states to fit. + +State-based methods should implement `estimate_FCS` when they require fitting +states, clusters, dictionaries, or transition models before subject-level dFC +estimation. + +## Required Attributes + +Initialize these attributes in `__init__`: + +```python +self.logs_ = "" +self.TPM = [] +self.FCS_ = [] +self.FCS_fit_time_ = None +self.dFC_assess_time_ = None +``` + +Define `params_name_lst` explicitly. Include every parameter used by the method +and every shared preprocessing parameter needed by `BaseDFCMethod`: + +```python +self.params_name_lst = [ + "measure_name", + "is_state_based", + # method-specific parameters here + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", +] +``` + +Then populate `self.params` from `params`: + +```python +self.params = {} +for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) +``` + +Always set: + +```python +self.params["measure_name"] = self.MEASURE_NAME +self.params["is_state_based"] = False # or True for state-based methods +``` + +Set defaults for method-specific parameters after `self.params` is created: + +```python +if self.params["min_periods"] is None: + self.params["min_periods"] = 10 +``` + +## State-Free Method Template + +This is a minimal single-subject state-free method template: + +```python +""" +My new dFC method. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class MY_NEW_METHOD(BaseDFCMethod): + """Short description of the method assumption.""" + + MEASURE_NAME = "MyNewMethod" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["min_periods"] is None: + self.params["min_periods"] = 10 + + @property + def measure_name(self): + return self.params["measure_name"] + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + FCSs = [] + TR_array = [] + + for tr in range(min_periods - 1, time_series.shape[1]): + matrix = np.corrcoef(time_series[:, : tr + 1]) + matrix[np.isnan(matrix)] = 0 + matrix[np.diag_indices_from(matrix)] = 1 + FCSs.append(matrix) + TR_array.append(tr) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC +``` + +## Output Contract + +`estimate_dFC` must return a `pydfc.dfc.DFC` object. + +For state-free methods, call: + +```python +dFC = DFC(measure=self) +dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) +``` + +where: + +- `FCSs` has shape `[n_time_samples, n_regions, n_regions]` +- `TR_array` has one integer TR index for each matrix in `FCSs` +- `TR_array` is sorted in ascending order +- every FC matrix is square + +Use the same orientation as existing methods: + +```text +time_series.data shape = [n_regions, n_timepoints] +``` + +If the method cannot estimate all TRs, returning a subset is valid as long as +`TR_array` identifies the corresponding time points. + +## Preprocessing Contract + +Use: + +```python +time_series = self.manipulate_time_series4dFC(time_series) +``` + +inside `estimate_dFC`. + +This applies shared options such as: + +- `num_select_nodes` +- `Fs_ratio` +- `normalization` +- `noise_ratio` +- `num_time_point` + +Include these keys in `params_name_lst`; otherwise `BaseDFCMethod` may fail when +it tries to access them. + +For state-based `estimate_FCS`, use: + +```python +time_series = self.manipulate_time_series4FCS(time_series) +``` + +and include any FCS-only parameters such as `num_subj` if the method uses them. + +## ML Pipeline Registration (State-Based Methods Only) + +If the new method is state-based (`is_state_based = True`), you **must** also +register it in `pydfc/ml_utils.py` inside `process_SB_features`. This function +applies the correct feature transformation before classification. Omitting this +step causes the function to return `None`, which crashes the ML pipeline with a +`TypeError` at `subject_center`. + +Determine which branch your method belongs to: + +- **Softmax → ILR** (`if` branch, methods like `CAP`, `Clustering`): use this + when `FCS_proba` stores raw distances or dissimilarity scores that must first + be converted to a probability simplex via softmax. +- **ILR only** (`elif` branch, methods like `GaussianMixtureStates`, + `ContinuousHMM`, `NMFStates`): use this when `FCS_proba` already contains + proper probabilities (non-negative, rows summing to 1). + +Add the method name to the correct branch: + +```python +# pydfc/ml_utils.py — process_SB_features +elif measure_name in [ + "ContinuousHMM", + ... + "NMFStates", # ← add your method here if FCS_proba rows sum to 1 + ... +]: + X_transformed = ilr_transform(X) +``` + +A quick check: inspect `estimate_dFC` in the method file and look at how +`FCS_proba` is set. If it is produced by a row-wise normalization +(`/ row_sums`) or a soft-assignment model (GMM, HMM posterior), it belongs in +the ILR-only branch. If it stores distances or un-normalized scores, it belongs +in the softmax + ILR branch. + +## Package Export + +After adding a method file, update: + +```text +pydfc/dfc_methods/__init__.py +``` + +Example: + +```python +from .my_new_method import MY_NEW_METHOD + +__all__ = [ + ... + "MY_NEW_METHOD", +] +``` + +Be aware that package-level imports can fail if a method imports optional +dependencies that are not installed. If a method needs an optional package, keep +the dependency localized and make the failure message clear. + +## Validation Registration + +No manual registration is needed. The validation framework auto-discovers every +class that inherits from `BaseDFCMethod` and has a `MEASURE_NAME` attribute by +scanning `pydfc.dfc_methods` at runtime. As long as the method is exported from +`pydfc/dfc_methods/__init__.py`, it will appear automatically in: + +```bash +python -W ignore -m tests.test_validation.validate_dfc --list-methods +``` + +To add a CLI shorthand alias for the `--methods` argument, add one entry to +`_ALIASES` in `tests/test_validation/dfc_method_wrappers.py`: + +```python +"MyNewMethod": ["mynew", "mnm"], +``` + +This is optional — the full `MEASURE_NAME` always works without an alias. + +## Visualization Registration + +To include a method in the visual comparison script, update: + +```text +tests/test_validation/visualize_dfc.py +``` + +Add the method class and parameters to `method_specs()`. + +The visualization script checks that every parameter in the config is supported +by the method’s `params_name_lst`. This is intentional: if a parameter is +misspelled or unsupported, visualization should fail early instead of silently +ignoring it. + +## Recommended Validation Commands + +Run syntax checks: + +```bash +python -m py_compile pydfc/dfc_methods/my_new_method.py +``` + +List all registered methods and their aliases: + +```bash +python -W ignore -m tests.test_validation.validate_dfc --list-methods +``` + +Run API conformance checks on a specific method: + +```bash +python -W ignore -m tests.test_validation.validate_dfc --methods mynew --verbose 2 +``` + +Run API conformance checks on all registered methods: + +```bash +python -W ignore -m tests.test_validation.validate_dfc > results.txt 2>&1 +``` + +Passing all 6 sub-checks means the method satisfies the PydFC contract and its +output is structurally sound. It does not assess neurobiological validity. + +## Scientific Reporting Checklist + +When adding or describing a method, document: + +- Whether it is state-free or state-based +- Whether it assumes temporal smoothness, recurrence, event synchrony, phase + synchrony, latent states, or another dependency model +- What each major hyperparameter controls +- Whether outputs are correlation-like, partial-correlation-like, phase-locking, + event coactivation, kernel similarity, or another dependency score +- What range of values is expected +- Whether negative values are meaningful +- How the method differs from Sliding Window, CAP, HMM, Windowless, or + Time-Frequency methods + +Based on the repository context and Torabi et al., 2024, method choice can +substantially affect dFC results. New methods should therefore be compared +against established methods rather than interpreted in isolation. + +## Common Mistakes + +- Putting several unrelated method classes in one file. +- Forgetting to include shared preprocessing keys in `params_name_lst`. +- Returning a raw NumPy array from `estimate_dFC` instead of a `DFC` object. +- Returning matrices without setting `TR_array`. +- Passing unsupported parameters from wrappers or visualization scripts. +- Modifying `base_dfc_method.py` when the method can be implemented as a normal + subclass. +- Importing optional dependencies at package level in a way that breaks unrelated + methods. +- For state-based methods: forgetting to add the method name to `process_SB_features` + in `pydfc/ml_utils.py`. The function silently returns `None` if the method is + missing from both branches, crashing the ML pipeline. See the + "ML Pipeline Registration" section above. diff --git a/pydfc/comparison/similarity_assessment.py b/pydfc/comparison/similarity_assessment.py index 5fc1066..94b2866 100644 --- a/pydfc/comparison/similarity_assessment.py +++ b/pydfc/comparison/similarity_assessment.py @@ -223,8 +223,26 @@ def extract_feature(self, dFC_mat, feature2extract, graph_property=None): return feature def dFC_mat_lst_similarity( - self, dFC_mat_lst, feature2extract, metric, graph_property=None + self, dFC_mat_lst, feature2extract, metric, graph_property=None, precompute=False ): + if precompute: + # pre-compute features once per matrix (avoids O(n^2) recomputation) + features = [ + self.extract_feature( + dFC_mat, + feature2extract=feature2extract, + graph_property=graph_property, + ) + for dFC_mat in dFC_mat_lst + ] + # pre-rank for spearman: Pearson(rank(a), rank(b)) == spearmanr(a, b) + if metric == "spearman": + ranked_features = [ + np.apply_along_axis( + lambda x: stats.rankdata(x, method="average"), 1, feat + ) + for feat in features + ] sim_mat_over_sample = None for i, dFC_mat_i in enumerate(dFC_mat_lst): @@ -235,16 +253,20 @@ def dFC_mat_lst_similarity( assert dFC_mat_i.shape == dFC_mat_j.shape, "shape mismatch" - feature_i = self.extract_feature( - dFC_mat_i, - feature2extract=feature2extract, - graph_property=graph_property, - ) # (samples, variables) - feature_j = self.extract_feature( - dFC_mat_j, - feature2extract=feature2extract, - graph_property=graph_property, - ) # (samples, variables) + if precompute: + feature_i = features[i] # (samples, variables) + feature_j = features[j] # (samples, variables) + else: + feature_i = self.extract_feature( + dFC_mat_i, + feature2extract=feature2extract, + graph_property=graph_property, + ) # (samples, variables) + feature_j = self.extract_feature( + dFC_mat_j, + feature2extract=feature2extract, + graph_property=graph_property, + ) # (samples, variables) sim_over_sample = list() for sample in range(feature_i.shape[0]): @@ -259,9 +281,15 @@ def dFC_mat_lst_similarity( 0, 1 ] elif metric == "spearman": - sim, p = stats.spearmanr( - feature_i[sample, :], feature_j[sample, :] - ) + if precompute: + sim = np.corrcoef( + ranked_features[i][sample, :], + ranked_features[j][sample, :], + )[0, 1] + else: + sim, _ = stats.spearmanr( + feature_i[sample, :], feature_j[sample, :] + ) elif metric == "MI": sim = mutual_information( X=feature_i[sample, :], Y=feature_j[sample, :], N_bins=100 @@ -282,6 +310,54 @@ def dFC_mat_lst_similarity( return sim_mat_over_sample + def assess_similarity_fast(self, dFC_lst): + """ """ + methods_assess = {} + + # sort dFC_lst according to methods names + old_list = [dFC.measure.measure_name for dFC in dFC_lst] + new_list = deepcopy(old_list) + new_list.sort() + + new_order = find_new_order(old_list, new_list) + dFC_lst = [dFC_lst[i] for i in new_order] + + common_TRs = TR_intersection(dFC_lst) + + measure_lst = list() + TS_info_lst = list() + dFC_mat_lst = list() + for dFC in dFC_lst: + measure_lst.append(dFC.measure) + TS_info_lst.append(dFC.TS_info) + dFC_mat_lst.append(dFC.get_dFC_mat(TRs=common_TRs)) + + methods_assess["measure_lst"] = measure_lst + methods_assess["TS_info_lst"] = TS_info_lst + methods_assess["common_TRs"] = common_TRs + + ########## time record ########## + + time_record_dict = {} + for i, dFC in enumerate(dFC_lst): + time_record = {} + time_record["FCS_fit"] = dFC.measure.FCS_fit_time + time_record["dFC_assess"] = dFC.measure.dFC_assess_time + time_record_dict[str(i)] = time_record + methods_assess["time_record_dict"] = time_record_dict + + ########## subj_dFC_sim ########## + # returns correlation/MI/spearman corr/euclidean distance between results of dFC + # measures in a subject + metric_list = ["spearman"] + methods_assess["all"] = {} + for metric in metric_list: + methods_assess["all"][metric] = self.dFC_mat_lst_similarity( + dFC_mat_lst, feature2extract="all", metric=metric, precompute=True + ) + ############################################## + return methods_assess + def assess_similarity(self, dFC_lst, downsampling_method="default", **param_dict): """ downsampling_method: 'default' picks FCs at common_TRs diff --git a/pydfc/dfc_methods/__init__.py b/pydfc/dfc_methods/__init__.py index 5acb0b8..30b88ba 100644 --- a/pydfc/dfc_methods/__init__.py +++ b/pydfc/dfc_methods/__init__.py @@ -1,21 +1,189 @@ """The :mod:`pydfc.dfc_methods` contains dFC methods objects.""" +from .adaptive_exponential_window import ADAPTIVE_EXPONENTIAL_WINDOW +from .agglomerative_states import AGGLOMERATIVE_STATES +from .amplitude_envelope_correlation import AMPLITUDE_ENVELOPE_CORRELATION from .base_dfc_method import BaseDFCMethod +from .bayesian_gaussian_mixture_states import BAYESIAN_GAUSSIAN_MIXTURE_STATES +from .birch_states import BIRCH_STATES from .cap import CAP +from .changepoint_reset_window import CHANGEPOINT_RESET_WINDOW from .continuous_hmm import HMM_CONT +from .copula_tail_dependence import COPULA_TAIL_DEPENDENCE +from .curvature_correlation import CURVATURE_CORRELATION +from .dcc_connectivity import DCC_CONNECTIVITY +from .derivative_weighted_window import DERIVATIVE_WEIGHTED_WINDOW +from .differential_coactivation import DIFFERENTIAL_COACTIVATION from .discrete_hmm import HMM_DISC +from .dynamic_partial_correlation import DYNAMIC_PARTIAL_CORRELATION +from .edge_coactivation import EDGE_COACTIVATION +from .event_synchronization import EVENT_SYNCHRONIZATION +from .exponential_window import EXPONENTIAL_WINDOW +from .gaussian_mixture_states import GAUSSIAN_MIXTURE_STATES +from .graph_diffusion_coactivation import GRAPH_DIFFUSION_COACTIVATION +from .instantaneous_phase_coherence import INSTANTANEOUS_PHASE_COHERENCE +from .kalman_covariance import KALMAN_COVARIANCE +from .lagged_kmeans_states import LAGGED_KMEANS_STATES +from .lagged_max_correlation import LAGGED_MAX_CORRELATION +from .leading_eigenvector_dynamics import LEADING_EIGENVECTOR_DYNAMICS +from .local_jacobian_coupling import LOCAL_JACOBIAN_COUPLING +from .markov_smoothed_gmm_states import MARKOV_SMOOTHED_GMM_STATES +from .markov_smoothed_kmeans_states import MARKOV_SMOOTHED_KMEANS_STATES +from .minibatch_kmeans_states import MINIBATCH_KMEANS_STATES +from .multiscale_window import MULTISCALE_WINDOW +from .mutual_compression import MUTUAL_COMPRESSION +from .nmf_states import NMF_STATES +from .oja_subspace_connectivity import OJA_SUBSPACE_CONNECTIVITY +from .persistent_homology import PERSISTENT_HOMOLOGY +from .phase_amplitude_cross import PHASE_AMPLITUDE_CROSS +from .phase_lag_index_window import PHASE_LAG_INDEX_WINDOW +from .phase_locking_window import PHASE_LOCKING_WINDOW +from .point_process_connectivity import POINT_PROCESS_CONNECTIVITY +from .pooled_kmeans_states import POOLED_KMEANS_STATES +from .positive_negative_asymmetry import POSITIVE_NEGATIVE_ASYMMETRY +from .precision_shrinkage_window import PRECISION_SHRINKAGE_WINDOW +from .quantum_mutual_information import QUANTUM_MUTUAL_INFORMATION +from .random_fourier_dependence import RANDOM_FOURIER_DEPENDENCE +from .recurrence_kernel_dependence import RECURRENCE_KERNEL_DEPENDENCE +from .reservoir_echo_state import RESERVOIR_ECHO_STATE +from .robust_sliding_window import ROBUST_SLIDING_WINDOW from .sliding_window import SLIDING_WINDOW from .sliding_window_clustr import SLIDING_WINDOW_CLUSTR +from .sparse_coactivation_code import SPARSE_COACTIVATION_CODE +from .spectral_similarity import SPECTRAL_SIMILARITY +from .spectral_states import SPECTRAL_STATES +from .state_space_neighborhood import STATE_SPACE_NEIGHBORHOOD +from .stft_coherence import STFT_COHERENCE +from .synchrony_likelihood_window import SYNCHRONY_LIKELIHOOD_WINDOW +from .tangent_space_fc import TANGENT_SPACE_FC +from .temporal_asymmetry import TEMPORAL_ASYMMETRY +from .temporal_derivative_multiplication import TEMPORAL_DERIVATIVE_MULTIPLICATION from .time_freq import TIME_FREQ +from .time_reversal_asymmetry import TIME_REVERSAL_ASYMMETRY +from .volatility_weighted import VOLATILITY_WEIGHTED from .windowless import WINDOWLESS +from .adaptive_dcc_random_hybrid_ensemble import ADAPTIVE_DCC_RANDOM_HYBRID_ENSEMBLE +from .adaptive_edge_kalman_hybrid import ADAPTIVE_EDGE_KALMAN_HYBRID +from .adaptive_quantum_reservoir_hybrid import ADAPTIVE_QUANTUM_RESERVOIR_HYBRID +from .adaptive_random_sparse_hybrid import ADAPTIVE_RANDOM_SPARSE_HYBRID +from .adaptive_robust_differential_hybrid_ensemble import ADAPTIVE_ROBUST_DIFFERENTIAL_HYBRID_ENSEMBLE +from .changepoint_multiscale_exp_hybrid_ensemble import CHANGEPOINT_MULTISCALE_EXP_HYBRID_ENSEMBLE +from .changepoint_robust_exp_hybrid import CHANGEPOINT_ROBUST_EXP_HYBRID +from .dcc_changepoint_sparse_hybrid import DCC_CHANGEPOINT_SPARSE_HYBRID +from .dcc_edge_exponential_hybrid import DCC_EDGE_EXPONENTIAL_HYBRID +from .dcc_kalman_volatility_hybrid_ensemble import DCC_KALMAN_VOLATILITY_HYBRID_ENSEMBLE +from .differential_edge_kalman_hybrid import DIFFERENTIAL_EDGE_KALMAN_HYBRID +from .edge_kalman_exp_hybrid_ensemble import EDGE_KALMAN_EXP_HYBRID_ENSEMBLE +from .edge_random_reservoir_hybrid_ensemble import EDGE_RANDOM_RESERVOIR_HYBRID_ENSEMBLE +from .edge_stft_quantum_hybrid import EDGE_STFT_QUANTUM_HYBRID +from .kalman_reservoir_multiscale_hybrid import KALMAN_RESERVOIR_MULTISCALE_HYBRID +from .multiscale_dcc_kalman_hybrid import MULTISCALE_DCC_KALMAN_HYBRID +from .quantum_multiscale_kalman_hybrid_ensemble import QUANTUM_MULTISCALE_KALMAN_HYBRID_ENSEMBLE +from .quantum_random_exp_hybrid import QUANTUM_RANDOM_EXP_HYBRID +from .random_fourier_kalman_hybrid import RANDOM_FOURIER_KALMAN_HYBRID +from .reservoir_changepoint_robust_hybrid_ensemble import RESERVOIR_CHANGEPOINT_ROBUST_HYBRID_ENSEMBLE +from .reservoir_edge_dcc_hybrid import RESERVOIR_EDGE_DCC_HYBRID +from .robust_differential_stft_hybrid import ROBUST_DIFFERENTIAL_STFT_HYBRID +from .robust_sparse_edge_hybrid import ROBUST_SPARSE_EDGE_HYBRID +from .robust_volatility_sliding_hybrid_ensemble import ROBUST_VOLATILITY_SLIDING_HYBRID_ENSEMBLE +from .sliding_volatility_dcc_hybrid import SLIDING_VOLATILITY_DCC_HYBRID +from .sparse_dcc_exp_hybrid_ensemble import SPARSE_DCC_EXP_HYBRID_ENSEMBLE +from .stft_edge_adaptive_hybrid_ensemble import STFT_EDGE_ADAPTIVE_HYBRID_ENSEMBLE +from .stft_exp_kalman_hybrid import STFT_EXP_KALMAN_HYBRID +from .stft_quantum_sparse_hybrid_ensemble import STFT_QUANTUM_SPARSE_HYBRID_ENSEMBLE +from .volatility_adaptive_edge_hybrid import VOLATILITY_ADAPTIVE_EDGE_HYBRID __all__ = [ "BaseDFCMethod", + "AGGLOMERATIVE_STATES", + "BAYESIAN_GAUSSIAN_MIXTURE_STATES", + "BIRCH_STATES", "CAP", "SLIDING_WINDOW_CLUSTR", "HMM_CONT", "HMM_DISC", + "EXPONENTIAL_WINDOW", + "ADAPTIVE_EXPONENTIAL_WINDOW", + "MULTISCALE_WINDOW", + "EDGE_COACTIVATION", + "PHASE_LOCKING_WINDOW", + "DERIVATIVE_WEIGHTED_WINDOW", + "CHANGEPOINT_RESET_WINDOW", + "KALMAN_COVARIANCE", + "GAUSSIAN_MIXTURE_STATES", + "LAGGED_MAX_CORRELATION", + "LAGGED_KMEANS_STATES", + "MARKOV_SMOOTHED_GMM_STATES", + "MARKOV_SMOOTHED_KMEANS_STATES", + "MINIBATCH_KMEANS_STATES", + "POOLED_KMEANS_STATES", + "PRECISION_SHRINKAGE_WINDOW", + "RECURRENCE_KERNEL_DEPENDENCE", + "RANDOM_FOURIER_DEPENDENCE", + "SPECTRAL_STATES", + "EVENT_SYNCHRONIZATION", + "COPULA_TAIL_DEPENDENCE", + "OJA_SUBSPACE_CONNECTIVITY", + "GRAPH_DIFFUSION_COACTIVATION", "SLIDING_WINDOW", + "TEMPORAL_DERIVATIVE_MULTIPLICATION", "TIME_FREQ", "WINDOWLESS", + "AMPLITUDE_ENVELOPE_CORRELATION", + "LEADING_EIGENVECTOR_DYNAMICS", + "INSTANTANEOUS_PHASE_COHERENCE", + "DYNAMIC_PARTIAL_CORRELATION", + "PHASE_LAG_INDEX_WINDOW", + "POINT_PROCESS_CONNECTIVITY", + "STFT_COHERENCE", + "ROBUST_SLIDING_WINDOW", + "SYNCHRONY_LIKELIHOOD_WINDOW", + "NMF_STATES", + "DIFFERENTIAL_COACTIVATION", + "CURVATURE_CORRELATION", + "VOLATILITY_WEIGHTED", + "TEMPORAL_ASYMMETRY", + "PHASE_AMPLITUDE_CROSS", + "MUTUAL_COMPRESSION", + "TANGENT_SPACE_FC", + "SPECTRAL_SIMILARITY", + "POSITIVE_NEGATIVE_ASYMMETRY", + "STATE_SPACE_NEIGHBORHOOD", + "DCC_CONNECTIVITY", + "PERSISTENT_HOMOLOGY", + "TIME_REVERSAL_ASYMMETRY", + "QUANTUM_MUTUAL_INFORMATION", + "RESERVOIR_ECHO_STATE", + "LOCAL_JACOBIAN_COUPLING", + "SPARSE_COACTIVATION_CODE", + "ADAPTIVE_DCC_RANDOM_HYBRID_ENSEMBLE", + "ADAPTIVE_EDGE_KALMAN_HYBRID", + "ADAPTIVE_QUANTUM_RESERVOIR_HYBRID", + "ADAPTIVE_RANDOM_SPARSE_HYBRID", + "ADAPTIVE_ROBUST_DIFFERENTIAL_HYBRID_ENSEMBLE", + "CHANGEPOINT_MULTISCALE_EXP_HYBRID_ENSEMBLE", + "CHANGEPOINT_ROBUST_EXP_HYBRID", + "DCC_CHANGEPOINT_SPARSE_HYBRID", + "DCC_EDGE_EXPONENTIAL_HYBRID", + "DCC_KALMAN_VOLATILITY_HYBRID_ENSEMBLE", + "DIFFERENTIAL_EDGE_KALMAN_HYBRID", + "EDGE_KALMAN_EXP_HYBRID_ENSEMBLE", + "EDGE_RANDOM_RESERVOIR_HYBRID_ENSEMBLE", + "EDGE_STFT_QUANTUM_HYBRID", + "KALMAN_RESERVOIR_MULTISCALE_HYBRID", + "MULTISCALE_DCC_KALMAN_HYBRID", + "QUANTUM_MULTISCALE_KALMAN_HYBRID_ENSEMBLE", + "QUANTUM_RANDOM_EXP_HYBRID", + "RANDOM_FOURIER_KALMAN_HYBRID", + "RESERVOIR_CHANGEPOINT_ROBUST_HYBRID_ENSEMBLE", + "RESERVOIR_EDGE_DCC_HYBRID", + "ROBUST_DIFFERENTIAL_STFT_HYBRID", + "ROBUST_SPARSE_EDGE_HYBRID", + "ROBUST_VOLATILITY_SLIDING_HYBRID_ENSEMBLE", + "SLIDING_VOLATILITY_DCC_HYBRID", + "SPARSE_DCC_EXP_HYBRID_ENSEMBLE", + "STFT_EDGE_ADAPTIVE_HYBRID_ENSEMBLE", + "STFT_EXP_KALMAN_HYBRID", + "STFT_QUANTUM_SPARSE_HYBRID_ENSEMBLE", + "VOLATILITY_ADAPTIVE_EDGE_HYBRID", ] diff --git a/pydfc/dfc_methods/adaptive_dcc_random_hybrid_ensemble.py b/pydfc/dfc_methods/adaptive_dcc_random_hybrid_ensemble.py new file mode 100644 index 0000000..c93c173 --- /dev/null +++ b/pydfc/dfc_methods/adaptive_dcc_random_hybrid_ensemble.py @@ -0,0 +1,286 @@ +""" +AdaptiveDccRandom_hybrid_ensemble. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class ADAPTIVE_DCC_RANDOM_HYBRID_ENSEMBLE(BaseDFCMethod): + """State-free hybrid of AdaptiveExponentialWindow, DCCConnectivity, RandomFourierDependence.""" + + MEASURE_NAME = "AdaptiveDccRandom_hybrid_ensemble" + COMPONENTS = ['AdaptiveExponentialWindow', 'DCCConnectivity', 'RandomFourierDependence'] + HYBRID_RULE = "ensemble_mean" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/adaptive_edge_kalman_hybrid.py b/pydfc/dfc_methods/adaptive_edge_kalman_hybrid.py new file mode 100644 index 0000000..0aa7c7f --- /dev/null +++ b/pydfc/dfc_methods/adaptive_edge_kalman_hybrid.py @@ -0,0 +1,286 @@ +""" +AdaptiveEdgeKalman_hybrid. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class ADAPTIVE_EDGE_KALMAN_HYBRID(BaseDFCMethod): + """State-free hybrid of AdaptiveExponentialWindow, EdgeCoactivation, KalmanCovariance.""" + + MEASURE_NAME = "AdaptiveEdgeKalman_hybrid" + COMPONENTS = ['AdaptiveExponentialWindow', 'EdgeCoactivation', 'KalmanCovariance'] + HYBRID_RULE = "adaptive_gate" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/adaptive_exponential_window.py b/pydfc/dfc_methods/adaptive_exponential_window.py new file mode 100644 index 0000000..0ba579a --- /dev/null +++ b/pydfc/dfc_methods/adaptive_exponential_window.py @@ -0,0 +1,117 @@ +""" +Adaptive exponentially weighted dFC method. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class ADAPTIVE_EXPONENTIAL_WINDOW(BaseDFCMethod): + MEASURE_NAME = "AdaptiveExponentialWindow" + """Exponentially weighted correlation with data-adaptive forgetting.""" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + "alpha_min", + "alpha_max", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 10 + if self.params["alpha_min"] is None: + self.params["alpha_min"] = 0.02 + if self.params["alpha_max"] is None: + self.params["alpha_max"] = 0.35 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _corr_from_cov(self, covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 0, + ) + corr[np.diag_indices_from(corr)] = 1 + corr[np.isnan(corr)] = 0 + return corr + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0) + weights_sum = np.sum(weights) + if weights_sum <= 0: + weights = np.ones(samples.shape[1], dtype=float) + weights_sum = np.sum(weights) + weights = weights / weights_sum + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + diff_energy = np.zeros(time_series.shape[1]) + diff_energy[1:] = np.mean(np.abs(np.diff(time_series, axis=1)), axis=0) + baseline = np.median(diff_energy[1 : max(min_periods, 2)]) + 1e-8 + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + local_energy = diff_energy[tr] / baseline + adapt = local_energy / (1.0 + local_energy) + alpha = ( + self.params["alpha_min"] + + (self.params["alpha_max"] - self.params["alpha_min"]) * adapt + ) + age = tr - np.arange(tr + 1) + weights = alpha * np.power(1.0 - alpha, age) + FCSs.append(self._weighted_corr(time_series[:, : tr + 1], weights)) + TR_array.append(tr) + baseline = 0.98 * baseline + 0.02 * max(diff_energy[tr], 1e-8) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/adaptive_quantum_reservoir_hybrid.py b/pydfc/dfc_methods/adaptive_quantum_reservoir_hybrid.py new file mode 100644 index 0000000..7f5e6b1 --- /dev/null +++ b/pydfc/dfc_methods/adaptive_quantum_reservoir_hybrid.py @@ -0,0 +1,286 @@ +""" +AdaptiveQuantumReservoir_hybrid. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class ADAPTIVE_QUANTUM_RESERVOIR_HYBRID(BaseDFCMethod): + """State-free hybrid of AdaptiveExponentialWindow, QuantumMutualInformationFC, ReservoirEchoStateFC.""" + + MEASURE_NAME = "AdaptiveQuantumReservoir_hybrid" + COMPONENTS = ['AdaptiveExponentialWindow', 'QuantumMutualInformationFC', 'ReservoirEchoStateFC'] + HYBRID_RULE = "information_state_gate" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/adaptive_random_sparse_hybrid.py b/pydfc/dfc_methods/adaptive_random_sparse_hybrid.py new file mode 100644 index 0000000..bde2455 --- /dev/null +++ b/pydfc/dfc_methods/adaptive_random_sparse_hybrid.py @@ -0,0 +1,286 @@ +""" +AdaptiveRandomSparse_hybrid. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class ADAPTIVE_RANDOM_SPARSE_HYBRID(BaseDFCMethod): + """State-free hybrid of AdaptiveExponentialWindow, RandomFourierDependence, SparseCoactivationCodeFC.""" + + MEASURE_NAME = "AdaptiveRandomSparse_hybrid" + COMPONENTS = ['AdaptiveExponentialWindow', 'RandomFourierDependence', 'SparseCoactivationCodeFC'] + HYBRID_RULE = "sparse_kernel_gate" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/adaptive_robust_differential_hybrid_ensemble.py b/pydfc/dfc_methods/adaptive_robust_differential_hybrid_ensemble.py new file mode 100644 index 0000000..fca9de1 --- /dev/null +++ b/pydfc/dfc_methods/adaptive_robust_differential_hybrid_ensemble.py @@ -0,0 +1,286 @@ +""" +AdaptiveRobustDifferential_hybrid_ensemble. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class ADAPTIVE_ROBUST_DIFFERENTIAL_HYBRID_ENSEMBLE(BaseDFCMethod): + """State-free hybrid of AdaptiveExponentialWindow, RobustSlidingWindow, DifferentialCoactivationFC.""" + + MEASURE_NAME = "AdaptiveRobustDifferential_hybrid_ensemble" + COMPONENTS = ['AdaptiveExponentialWindow', 'RobustSlidingWindow', 'DifferentialCoactivationFC'] + HYBRID_RULE = "ensemble_mean" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/agglomerative_states.py b/pydfc/dfc_methods/agglomerative_states.py new file mode 100644 index 0000000..d95cc6b --- /dev/null +++ b/pydfc/dfc_methods/agglomerative_states.py @@ -0,0 +1,164 @@ +"""Agglomerative state-based dFC.""" + +import time + +import numpy as np +from sklearn.cluster import AgglomerativeClustering + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +def _corr(samples): + if samples.shape[0] < 2: + return np.eye(samples.shape[1], dtype=float) + cov = np.cov(samples, rowvar=False) + std = np.sqrt(np.maximum(np.diag(cov), 1e-12)) + den = np.outer(std, std) + corr = np.divide(cov, den, out=np.zeros_like(cov), where=den > 0) + corr[np.diag_indices_from(corr)] = 1.0 + return 0.5 * (corr + corr.T) + + +def _softmax_dist(features, centers, temperature): + distances = np.sum((features[:, None, :] - centers[None, :, :]) ** 2, axis=2) + logits = -distances / max(float(temperature), 1e-6) + logits = logits - np.max(logits, axis=1, keepdims=True) + probs = np.exp(logits) + probs = probs / np.maximum(np.sum(probs, axis=1, keepdims=True), 1e-12) + return np.argmin(distances, axis=1).astype(int), probs + + +class AGGLOMERATIVE_STATES(BaseDFCMethod): + MEASURE_NAME = "AgglomerativeStates" + """Hierarchical state partitions learned by agglomerative clustering.""" + + def __init__(self, **params): + self._train_sample_limit = 3000 + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "n_states", + "assignment_temperature", + "normalization", + "num_subj", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {name: params.get(name, None) for name in self.params_name_lst} + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = True + if self.params["n_states"] is None: + self.params["n_states"] = 5 + if self.params["assignment_temperature"] is None: + self.params["assignment_temperature"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _chunks(self, time_series): + return [ + time_series.get_subj_ts(subjs_id=subj).data.T.copy() + for subj in time_series.subj_id_lst + ] + + def _fit_matrix(self, chunks): + each = max(1, int(self._train_sample_limit) // max(len(chunks), 1)) + sampled = [] + for chunk in chunks: + if chunk.shape[0] <= each: + sampled.append(chunk) + else: + idx = np.linspace(0, chunk.shape[0] - 1, each, dtype=int) + sampled.append(chunk[idx, :]) + return np.concatenate(sampled, axis=0) + + def estimate_FCS(self, time_series): + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4FCS(time_series) + tic = time.time() + + chunks = self._chunks(time_series) + fit_matrix = self._fit_matrix(chunks) + n_states = min(int(self.params["n_states"]), fit_matrix.shape[0]) + self.params["n_states"] = n_states + + labels_fit = AgglomerativeClustering( + n_clusters=n_states, linkage="ward" + ).fit_predict(fit_matrix) + self.centers_ = np.zeros((n_states, fit_matrix.shape[1]), dtype=float) + fallback = np.mean(fit_matrix, axis=0) + for state in range(n_states): + mask = labels_fit == state + self.centers_[state, :] = ( + np.mean(fit_matrix[mask], axis=0) if np.any(mask) else fallback + ) + + labels_chunks, _ = zip( + *[ + _softmax_dist(chunk, self.centers_, self.params["assignment_temperature"]) + for chunk in chunks + ] + ) + self.Z = np.concatenate(labels_chunks, axis=0) + self.FCS_ = np.zeros( + (n_states, chunks[0].shape[1], chunks[0].shape[1]), dtype=float + ) + for state in range(n_states): + samples = [ + chunk[labels == state, :] + for chunk, labels in zip(chunks, labels_chunks) + if np.any(labels == state) + ] + self.FCS_[state, :, :] = ( + _corr(np.concatenate(samples, axis=0)) + if samples + else np.eye(chunks[0].shape[1]) + ) + + counts = np.full((n_states, n_states), 1.0, dtype=float) + for labels in labels_chunks: + for a, b in zip(labels[:-1], labels[1:]): + counts[a, b] += 1.0 + self.TPM = counts / np.maximum(np.sum(counts, axis=1, keepdims=True), 1e-12) + + self.set_mean_activity(time_series) + self.set_FCS_fit_time(time.time() - tic) + return self + + def estimate_dFC(self, time_series): + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + features = time_series.data.T.copy() + labels, probs = _softmax_dist( + features, self.centers_, self.params["assignment_temperature"] + ) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC( + FCSs=self.FCS_, + FCS_idx=labels, + FCS_proba=probs, + TS_info=time_series.info_dict, + TR_array=np.arange(features.shape[0], dtype=int), + ) + return dFC diff --git a/pydfc/dfc_methods/amplitude_envelope_correlation.py b/pydfc/dfc_methods/amplitude_envelope_correlation.py new file mode 100644 index 0000000..d406b5e --- /dev/null +++ b/pydfc/dfc_methods/amplitude_envelope_correlation.py @@ -0,0 +1,104 @@ +""" +Amplitude Envelope Correlation (AEC) dFC method. + +Reference: Brookes et al. (2011). Investigating the electrophysiological basis of +resting state networks using magnetoencephalography. PNAS, 108(40), 16783-16788. +doi:10.1073/pnas.1112685108 + +Extended to fMRI BOLD by Hipp et al. (2012). Low-frequency fluctuations in MEG +and correlates in fMRI. Nat Neurosci, 15, 1067-1070. doi:10.1038/nn.3101 +""" + +import time + +import numpy as np +from scipy.signal import hilbert + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class AMPLITUDE_ENVELOPE_CORRELATION(BaseDFCMethod): + """Sliding-window Pearson correlation of Hilbert amplitude envelopes. + + Each region's BOLD signal is transformed to its analytic signal via the + Hilbert transform; the instantaneous amplitude (envelope) replaces the raw + signal. Standard Pearson correlation is then computed within each sliding + window of the envelope time series, yielding one FC matrix per window + centre. This method specifically captures amplitude-modulation coupling, + separating it from the phase-synchrony component measured by PLV methods. + """ + + MEASURE_NAME = "AmplitudeEnvelopeCorrelation" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def dFC(self, time_series, Fs): + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + n_regions, T = time_series.shape + + envelope = np.abs(hilbert(time_series, axis=1)) + + FCSs, TR_array = [], [] + for l in range(0, T - W_samples + 1, step): + seg = envelope[:, l : l + W_samples] + C = np.corrcoef(seg) + C[np.isnan(C)] = 0.0 + np.fill_diagonal(C, 1.0) + FCSs.append(C) + TR_array.append(int(l + W_samples // 2)) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/base_dfc_method.py b/pydfc/dfc_methods/base_dfc_method.py index af3f518..6fb35ba 100644 --- a/pydfc/dfc_methods/base_dfc_method.py +++ b/pydfc/dfc_methods/base_dfc_method.py @@ -22,6 +22,9 @@ class BaseDFCMethod: + # Required in every concrete subclass for registry discovery. + MEASURE_NAME = None + TF_methods_name_lst = ["CWT_mag", "CWT_phase_r", "CWT_phase_a", "WTC"] sw_methods_name_lst = [ @@ -255,6 +258,8 @@ def visualize_FCS( class method_name(dFC): + MEASURE_NAME = 'method_name' + def __init__(self, **params): self.FCS_ = [] self.logs_ = '' @@ -270,7 +275,7 @@ def __init__(self, **params): self.params[params_name] = None self.params['specific_param'] = value - self.params['measure_name'] = 'method_name' + self.params['measure_name'] = self.MEASURE_NAME self.params['is_state_based'] = True/False @property diff --git a/pydfc/dfc_methods/bayesian_gaussian_mixture_states.py b/pydfc/dfc_methods/bayesian_gaussian_mixture_states.py new file mode 100644 index 0000000..a94ef24 --- /dev/null +++ b/pydfc/dfc_methods/bayesian_gaussian_mixture_states.py @@ -0,0 +1,149 @@ +"""Bayesian Gaussian mixture state-based dFC.""" + +import time + +import numpy as np +from sklearn.mixture import BayesianGaussianMixture + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +def _corr(samples): + if samples.shape[0] < 2: + return np.eye(samples.shape[1], dtype=float) + cov = np.cov(samples, rowvar=False) + std = np.sqrt(np.maximum(np.diag(cov), 1e-12)) + den = np.outer(std, std) + corr = np.divide(cov, den, out=np.zeros_like(cov), where=den > 0) + corr[np.diag_indices_from(corr)] = 1.0 + return 0.5 * (corr + corr.T) + + +class BAYESIAN_GAUSSIAN_MIXTURE_STATES(BaseDFCMethod): + MEASURE_NAME = "BayesianGaussianMixtureStates" + """Sparse state emissions with Bayesian Gaussian mixture regularization.""" + + def __init__(self, **params): + self._covariance_type = "full" + self._reg_covar = 1e-6 + self._max_iter = 300 + self._train_sample_limit = 5000 + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "n_states", + "normalization", + "num_subj", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {name: params.get(name, None) for name in self.params_name_lst} + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = True + if self.params["n_states"] is None: + self.params["n_states"] = 5 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _chunks(self, time_series): + return [ + time_series.get_subj_ts(subjs_id=subj).data.T.copy() + for subj in time_series.subj_id_lst + ] + + def _fit_matrix(self, chunks): + each = max(1, int(self._train_sample_limit) // max(len(chunks), 1)) + sampled = [] + for chunk in chunks: + if chunk.shape[0] <= each: + sampled.append(chunk) + else: + idx = np.linspace(0, chunk.shape[0] - 1, each, dtype=int) + sampled.append(chunk[idx, :]) + return np.concatenate(sampled, axis=0) + + def estimate_FCS(self, time_series): + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4FCS(time_series) + tic = time.time() + + chunks = self._chunks(time_series) + fit_matrix = self._fit_matrix(chunks) + n_states = min(int(self.params["n_states"]), fit_matrix.shape[0]) + self.params["n_states"] = n_states + + self.bgmm_ = BayesianGaussianMixture( + n_components=n_states, + covariance_type=self._covariance_type, + reg_covar=self._reg_covar, + max_iter=self._max_iter, + random_state=None, + weight_concentration_prior_type="dirichlet_process", + ).fit(fit_matrix) + + labels_chunks = [self.bgmm_.predict(chunk).astype(int) for chunk in chunks] + self.Z = np.concatenate(labels_chunks, axis=0) + self.FCS_ = np.zeros( + (n_states, chunks[0].shape[1], chunks[0].shape[1]), dtype=float + ) + for state in range(n_states): + samples = [ + chunk[labels == state, :] + for chunk, labels in zip(chunks, labels_chunks) + if np.any(labels == state) + ] + self.FCS_[state, :, :] = ( + _corr(np.concatenate(samples, axis=0)) + if samples + else np.eye(chunks[0].shape[1]) + ) + + counts = np.full((n_states, n_states), 1.0, dtype=float) + for labels in labels_chunks: + for a, b in zip(labels[:-1], labels[1:]): + counts[a, b] += 1.0 + self.TPM = counts / np.maximum(np.sum(counts, axis=1, keepdims=True), 1e-12) + + self.set_mean_activity(time_series) + self.set_FCS_fit_time(time.time() - tic) + return self + + def estimate_dFC(self, time_series): + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + + features = time_series.data.T.copy() + probs = self.bgmm_.predict_proba(features) + labels = np.argmax(probs, axis=1).astype(int) + + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC( + FCSs=self.FCS_, + FCS_idx=labels, + FCS_proba=probs, + TS_info=time_series.info_dict, + TR_array=np.arange(features.shape[0], dtype=int), + ) + return dFC diff --git a/pydfc/dfc_methods/birch_states.py b/pydfc/dfc_methods/birch_states.py new file mode 100644 index 0000000..5967290 --- /dev/null +++ b/pydfc/dfc_methods/birch_states.py @@ -0,0 +1,165 @@ +"""BIRCH state-based dFC.""" + +import time + +import numpy as np +from sklearn.cluster import Birch + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +def _corr(samples): + if samples.shape[0] < 2: + return np.eye(samples.shape[1], dtype=float) + cov = np.cov(samples, rowvar=False) + std = np.sqrt(np.maximum(np.diag(cov), 1e-12)) + den = np.outer(std, std) + corr = np.divide(cov, den, out=np.zeros_like(cov), where=den > 0) + corr[np.diag_indices_from(corr)] = 1.0 + return 0.5 * (corr + corr.T) + + +def _softmax_dist(features, centers, temperature): + distances = np.sum((features[:, None, :] - centers[None, :, :]) ** 2, axis=2) + logits = -distances / max(float(temperature), 1e-6) + logits = logits - np.max(logits, axis=1, keepdims=True) + probs = np.exp(logits) + probs = probs / np.maximum(np.sum(probs, axis=1, keepdims=True), 1e-12) + return np.argmin(distances, axis=1).astype(int), probs + + +class BIRCH_STATES(BaseDFCMethod): + MEASURE_NAME = "BirchStates" + """Compact hierarchical states learned with BIRCH clustering.""" + + def __init__(self, **params): + self._train_sample_limit = 5000 + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "n_states", + "assignment_temperature", + "normalization", + "num_subj", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {name: params.get(name, None) for name in self.params_name_lst} + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = True + if self.params["n_states"] is None: + self.params["n_states"] = 5 + if self.params["assignment_temperature"] is None: + self.params["assignment_temperature"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _chunks(self, time_series): + return [ + time_series.get_subj_ts(subjs_id=subj).data.T.copy() + for subj in time_series.subj_id_lst + ] + + def _fit_matrix(self, chunks): + each = max(1, int(self._train_sample_limit) // max(len(chunks), 1)) + sampled = [] + for chunk in chunks: + if chunk.shape[0] <= each: + sampled.append(chunk) + else: + idx = np.linspace(0, chunk.shape[0] - 1, each, dtype=int) + sampled.append(chunk[idx, :]) + return np.concatenate(sampled, axis=0) + + def estimate_FCS(self, time_series): + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4FCS(time_series) + tic = time.time() + + chunks = self._chunks(time_series) + fit_matrix = self._fit_matrix(chunks) + n_states = min(int(self.params["n_states"]), fit_matrix.shape[0]) + self.params["n_states"] = n_states + + birch = Birch(n_clusters=n_states) + labels_fit = birch.fit_predict(fit_matrix).astype(int) + self.centers_ = np.zeros((n_states, fit_matrix.shape[1]), dtype=float) + fallback = np.mean(fit_matrix, axis=0) + for state in range(n_states): + mask = labels_fit == state + self.centers_[state, :] = ( + np.mean(fit_matrix[mask], axis=0) if np.any(mask) else fallback + ) + + labels_chunks, _ = zip( + *[ + _softmax_dist(chunk, self.centers_, self.params["assignment_temperature"]) + for chunk in chunks + ] + ) + self.Z = np.concatenate(labels_chunks, axis=0) + self.FCS_ = np.zeros( + (n_states, chunks[0].shape[1], chunks[0].shape[1]), dtype=float + ) + for state in range(n_states): + samples = [ + chunk[labels == state, :] + for chunk, labels in zip(chunks, labels_chunks) + if np.any(labels == state) + ] + self.FCS_[state, :, :] = ( + _corr(np.concatenate(samples, axis=0)) + if samples + else np.eye(chunks[0].shape[1]) + ) + + counts = np.full((n_states, n_states), 1.0, dtype=float) + for labels in labels_chunks: + for a, b in zip(labels[:-1], labels[1:]): + counts[a, b] += 1.0 + self.TPM = counts / np.maximum(np.sum(counts, axis=1, keepdims=True), 1e-12) + + self.set_mean_activity(time_series) + self.set_FCS_fit_time(time.time() - tic) + return self + + def estimate_dFC(self, time_series): + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + + features = time_series.data.T.copy() + labels, probs = _softmax_dist( + features, self.centers_, self.params["assignment_temperature"] + ) + + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC( + FCSs=self.FCS_, + FCS_idx=labels, + FCS_proba=probs, + TS_info=time_series.info_dict, + TR_array=np.arange(features.shape[0], dtype=int), + ) + return dFC diff --git a/pydfc/dfc_methods/cap.py b/pydfc/dfc_methods/cap.py index ed0b21c..2837e07 100644 --- a/pydfc/dfc_methods/cap.py +++ b/pydfc/dfc_methods/cap.py @@ -34,6 +34,7 @@ class CAP(BaseDFCMethod): + MEASURE_NAME = "CAP" def __init__(self, **params): self.logs_ = "" @@ -63,7 +64,7 @@ def __init__(self, **params): else: self.params[params_name] = None - self.params["measure_name"] = "CAP" + self.params["measure_name"] = self.MEASURE_NAME self.params["is_state_based"] = True @property diff --git a/pydfc/dfc_methods/changepoint_multiscale_exp_hybrid_ensemble.py b/pydfc/dfc_methods/changepoint_multiscale_exp_hybrid_ensemble.py new file mode 100644 index 0000000..2a3c71d --- /dev/null +++ b/pydfc/dfc_methods/changepoint_multiscale_exp_hybrid_ensemble.py @@ -0,0 +1,286 @@ +""" +ChangepointMultiscaleExp_hybrid_ensemble. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class CHANGEPOINT_MULTISCALE_EXP_HYBRID_ENSEMBLE(BaseDFCMethod): + """State-free hybrid of ChangepointResetWindow, MultiscaleWindow, ExponentialWindow.""" + + MEASURE_NAME = "ChangepointMultiscaleExp_hybrid_ensemble" + COMPONENTS = ['ChangepointResetWindow', 'MultiscaleWindow', 'ExponentialWindow'] + HYBRID_RULE = "ensemble_mean" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/changepoint_reset_window.py b/pydfc/dfc_methods/changepoint_reset_window.py new file mode 100644 index 0000000..0060f85 --- /dev/null +++ b/pydfc/dfc_methods/changepoint_reset_window.py @@ -0,0 +1,128 @@ +""" +Changepoint-reset dFC method. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class CHANGEPOINT_RESET_WINDOW(BaseDFCMethod): + MEASURE_NAME = "ChangepointResetWindow" + """Exponentially weighted correlation that resets after abrupt changes.""" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "half_life", + "min_periods", + "change_threshold", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + "alpha", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["min_periods"] is None: + self.params["min_periods"] = 10 + if self.params["change_threshold"] is None: + self.params["change_threshold"] = 4.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + if self.params["alpha"] is not None: + return float(self.params["alpha"]) + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + def _corr_from_cov(self, covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 0, + ) + corr[np.diag_indices_from(corr)] = 1 + corr[np.isnan(corr)] = 0 + return corr + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0) + weights_sum = np.sum(weights) + if weights_sum <= 0: + weights = np.ones(samples.shape[1], dtype=float) + weights_sum = np.sum(weights) + weights = weights / weights_sum + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + def dFC(self, time_series, Fs): + alpha = self._alpha_from_half_life(Fs) + min_periods = int(self.params["min_periods"]) + reset_start = 0 + diff_energy = np.zeros(time_series.shape[1]) + diff_energy[1:] = np.mean(np.abs(np.diff(time_series, axis=1)), axis=0) + FCSs = [] + TR_array = [] + for tr in range(1, time_series.shape[1]): + history = diff_energy[max(1, tr - 30) : tr] + if history.size == 0: + baseline = max(diff_energy[tr], 1e-8) + else: + baseline = np.median(history) + 1e-8 + if diff_energy[tr] / baseline > self.params["change_threshold"]: + reset_start = max(0, tr - 1) + if tr - reset_start + 1 >= min_periods: + age = tr - np.arange(reset_start, tr + 1) + weights = alpha * np.power(1.0 - alpha, age) + FCSs.append( + self._weighted_corr(time_series[:, reset_start : tr + 1], weights) + ) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/changepoint_robust_exp_hybrid.py b/pydfc/dfc_methods/changepoint_robust_exp_hybrid.py new file mode 100644 index 0000000..8b71748 --- /dev/null +++ b/pydfc/dfc_methods/changepoint_robust_exp_hybrid.py @@ -0,0 +1,286 @@ +""" +ChangepointRobustExp_hybrid. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class CHANGEPOINT_ROBUST_EXP_HYBRID(BaseDFCMethod): + """State-free hybrid of ChangepointResetWindow, RobustSlidingWindow, ExponentialWindow.""" + + MEASURE_NAME = "ChangepointRobustExp_hybrid" + COMPONENTS = ['ChangepointResetWindow', 'RobustSlidingWindow', 'ExponentialWindow'] + HYBRID_RULE = "change_reset_gate" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/continuous_hmm.py b/pydfc/dfc_methods/continuous_hmm.py index 7082d3d..488ab5a 100644 --- a/pydfc/dfc_methods/continuous_hmm.py +++ b/pydfc/dfc_methods/continuous_hmm.py @@ -33,6 +33,7 @@ class HMM_CONT(BaseDFCMethod): + MEASURE_NAME = "ContinuousHMM" def __init__(self, **params): self.logs_ = "" @@ -63,7 +64,7 @@ def __init__(self, **params): else: self.params[params_name] = None - self.params["measure_name"] = "ContinuousHMM" + self.params["measure_name"] = self.MEASURE_NAME self.params["is_state_based"] = True @property diff --git a/pydfc/dfc_methods/copula_tail_dependence.py b/pydfc/dfc_methods/copula_tail_dependence.py new file mode 100644 index 0000000..a7251b1 --- /dev/null +++ b/pydfc/dfc_methods/copula_tail_dependence.py @@ -0,0 +1,111 @@ +""" +Copula tail-dependence dFC method. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class COPULA_TAIL_DEPENDENCE(BaseDFCMethod): + MEASURE_NAME = "CopulaTailDependence" + """FC from online concordance of empirical upper-tail events.""" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "half_life", + "min_periods", + "tail_quantile", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + "alpha", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["min_periods"] is None: + self.params["min_periods"] = 10 + if self.params["tail_quantile"] is None: + self.params["tail_quantile"] = 0.8 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + if self.params["alpha"] is not None: + return float(self.params["alpha"]) + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + alpha = self._alpha_from_half_life(Fs) + thresholds = np.quantile(time_series, self.params["tail_quantile"], axis=1) + lower_thresholds = np.quantile( + time_series, 1.0 - self.params["tail_quantile"], axis=1 + ) + tail_rate = np.zeros(time_series.shape[0]) + tail_coincidence = np.eye(time_series.shape[0]) + FCSs = [] + TR_array = [] + for tr in range(time_series.shape[1]): + upper = time_series[:, tr] >= thresholds + lower = time_series[:, tr] <= lower_thresholds + tail = np.where(upper, 1.0, np.where(lower, -1.0, 0.0)) + active = np.abs(tail) + tail_rate = (1.0 - alpha) * tail_rate + alpha * active + tail_coincidence = (1.0 - alpha) * tail_coincidence + alpha * np.outer( + tail, tail + ) + scale = np.sqrt(np.outer(tail_rate, tail_rate)) + matrix = np.divide( + tail_coincidence, + scale, + out=np.zeros_like(tail_coincidence), + where=scale > 0, + ) + matrix = np.clip(matrix, -1, 1) + matrix[np.diag_indices_from(matrix)] = 1 + if tr >= min_periods - 1: + FCSs.append(matrix) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/curvature_correlation.py b/pydfc/dfc_methods/curvature_correlation.py new file mode 100644 index 0000000..bdecb78 --- /dev/null +++ b/pydfc/dfc_methods/curvature_correlation.py @@ -0,0 +1,108 @@ +""" +Curvature Correlation FC (CurvCorr) — novel dFC method. + +Uses the second temporal difference (curvature proxy) of each region's +BOLD signal: κ_i(t) = x_i(t+1) - 2 x_i(t) + x_i(t-1). The windowed +Pearson correlation of these curvature signals measures whether regions +share the same "acceleration" pattern — do they speed up and slow down +simultaneously, independent of direction or level? +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class CURVATURE_CORRELATION(BaseDFCMethod): + """Windowed correlation of second temporal differences (curvature) of BOLD. + + Each region's BOLD trajectory in time has a local curvature given by + its second derivative. κ_i(t) = x_i(t+1) − 2x_i(t) + x_i(t−1) is the + central second-difference approximation. Within each sliding window the + Pearson correlation of these curvature signals is computed. The measure + captures synchrony in the *rate of change of rate of change* — orthogonal + to standard correlation (which captures synchrony in levels) and to + first-difference correlation (which captures synchrony in velocity). + """ + + MEASURE_NAME = "CurvatureCorrelationFC" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for p in self.params_name_lst: + self.params[p] = params.get(p, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + @staticmethod + def _curvature(x): + """Central second difference: x[2:] - 2*x[1:-1] + x[:-2], shape [R, T-2].""" + return x[:, 2:] - 2.0 * x[:, 1:-1] + x[:, :-2] + + def dFC(self, time_series, Fs): + kappa = self._curvature(time_series) # [n_regions, T-2] + n_regions, T_k = kappa.shape + + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + + FCSs, TR_array = [], [] + for l in range(0, T_k - W_samples + 1, step): + seg = kappa[:, l : l + W_samples] + C = np.corrcoef(seg) + C[np.isnan(C)] = 0.0 + np.fill_diagonal(C, 1.0) + FCSs.append(np.clip(C, -1.0, 1.0)) + TR_array.append(int(l + 1 + W_samples // 2)) # +1 offset for 2nd diff + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/dcc_changepoint_sparse_hybrid.py b/pydfc/dfc_methods/dcc_changepoint_sparse_hybrid.py new file mode 100644 index 0000000..61180d5 --- /dev/null +++ b/pydfc/dfc_methods/dcc_changepoint_sparse_hybrid.py @@ -0,0 +1,286 @@ +""" +DccChangepointSparse_hybrid. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class DCC_CHANGEPOINT_SPARSE_HYBRID(BaseDFCMethod): + """State-free hybrid of DCCConnectivity, ChangepointResetWindow, SparseCoactivationCodeFC.""" + + MEASURE_NAME = "DccChangepointSparse_hybrid" + COMPONENTS = ['DCCConnectivity', 'ChangepointResetWindow', 'SparseCoactivationCodeFC'] + HYBRID_RULE = "change_sparse_gate" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/dcc_connectivity.py b/pydfc/dfc_methods/dcc_connectivity.py new file mode 100644 index 0000000..b4ca7e0 --- /dev/null +++ b/pydfc/dfc_methods/dcc_connectivity.py @@ -0,0 +1,142 @@ +""" +Dynamic Conditional Correlation Connectivity (DCC-FC) — from financial econometrics. + +Source: Engle (2002). Dynamic conditional correlation. Journal of Business & +Economic Statistics, 20(3), 339-350. + +BOLD signals, like asset returns, are heteroskedastic: their conditional +variance clusters over time. DCC-GARCH captures this with a per-region +EWMA variance model and a DCC update for the conditional correlation matrix, +giving a per-TR estimate R(t) that explicitly accounts for time-varying +signal volatility — unlike standard correlation which treats all timepoints +as equally noisy. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class DCC_CONNECTIVITY(BaseDFCMethod): + """Per-TR conditional correlation via EWMA-GARCH + DCC recursion. + + Step 1 — Per-region EWMA variance (RiskMetrics GARCH approximation): + σ²_i(t) = λ · σ²_i(t−1) + (1−λ) · ε²_i(t−1) + + Step 2 — Standardise residuals: z_i(t) = ε_i(t) / σ_i(t) + + Step 3 — DCC update (Engle 2002): + Q(t) = (1−a−b) Q̄ + a z(t−1)z(t−1)ᵀ + b Q(t−1) + + Step 4 — Rescale to correlation: + R_ij(t) = Q_ij(t) / √(Q_ii(t) · Q_jj(t)) + + Unlike windowed Pearson correlation, DCC up-weights low-volatility periods + and down-weights high-variance bursts, capturing "conditional" coupling. + """ + + MEASURE_NAME = "DCCConnectivity" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "garch_lambda", + "dcc_a", + "dcc_b", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for p in self.params_name_lst: + self.params[p] = params.get(p, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["garch_lambda"] is None: + self.params["garch_lambda"] = 0.94 # RiskMetrics standard + if self.params["dcc_a"] is None: + self.params["dcc_a"] = 0.05 + if self.params["dcc_b"] is None: + self.params["dcc_b"] = 0.85 + + @property + def measure_name(self): + return self.params["measure_name"] + + def dFC(self, time_series, Fs): + n_regions, T = time_series.shape + lam = float(self.params["garch_lambda"]) + a = float(self.params["dcc_a"]) + b = float(self.params["dcc_b"]) + + # Demeaned residuals + eps = time_series - time_series.mean(axis=1, keepdims=True) + + # EWMA conditional variance: σ²_i(t) = λ σ²_i(t-1) + (1-λ) ε²_i(t-1) + sigma2 = np.zeros((n_regions, T)) + sigma2[:, 0] = np.var(eps, axis=1) + for t in range(1, T): + sigma2[:, t] = lam * sigma2[:, t - 1] + (1.0 - lam) * eps[:, t - 1] ** 2 + sigma2 = np.maximum(sigma2, 1e-12) + + # Standardised residuals + Z = eps / np.sqrt(sigma2) # [R, T] + + # Unconditional correlation of standardised residuals + Q_bar = np.corrcoef(Z) + Q_bar[np.isnan(Q_bar)] = 0.0 + np.fill_diagonal(Q_bar, 1.0) + Q_bar = self._nearest_spd(Q_bar) + + # DCC recursion + Q = Q_bar.copy() + FCSs, TR_array = [], [] + for t in range(1, T): + z = Z[:, t - 1] # [R] + Q = (1.0 - a - b) * Q_bar + a * np.outer(z, z) + b * Q + d = np.sqrt(np.maximum(np.diag(Q), 1e-12)) + R = Q / np.outer(d, d) + np.fill_diagonal(R, 1.0) + FCSs.append(np.clip(R, -1.0, 1.0)) + TR_array.append(t) + + return np.array(FCSs), np.array(TR_array) + + @staticmethod + def _nearest_spd(A): + A = (A + A.T) / 2.0 + vals, vecs = np.linalg.eigh(A) + return vecs @ np.diag(np.maximum(vals, 1e-6)) @ vecs.T + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/dcc_edge_exponential_hybrid.py b/pydfc/dfc_methods/dcc_edge_exponential_hybrid.py new file mode 100644 index 0000000..52f8f19 --- /dev/null +++ b/pydfc/dfc_methods/dcc_edge_exponential_hybrid.py @@ -0,0 +1,286 @@ +""" +DccEdgeExponential_hybrid. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class DCC_EDGE_EXPONENTIAL_HYBRID(BaseDFCMethod): + """State-free hybrid of DCCConnectivity, EdgeCoactivation, ExponentialWindow.""" + + MEASURE_NAME = "DccEdgeExponential_hybrid" + COMPONENTS = ['DCCConnectivity', 'EdgeCoactivation', 'ExponentialWindow'] + HYBRID_RULE = "conditional_residual" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/dcc_kalman_volatility_hybrid_ensemble.py b/pydfc/dfc_methods/dcc_kalman_volatility_hybrid_ensemble.py new file mode 100644 index 0000000..0e535b1 --- /dev/null +++ b/pydfc/dfc_methods/dcc_kalman_volatility_hybrid_ensemble.py @@ -0,0 +1,286 @@ +""" +DccKalmanVolatility_hybrid_ensemble. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class DCC_KALMAN_VOLATILITY_HYBRID_ENSEMBLE(BaseDFCMethod): + """State-free hybrid of DCCConnectivity, KalmanCovariance, VolatilityWeightedFC.""" + + MEASURE_NAME = "DccKalmanVolatility_hybrid_ensemble" + COMPONENTS = ['DCCConnectivity', 'KalmanCovariance', 'VolatilityWeightedFC'] + HYBRID_RULE = "ensemble_mean" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/derivative_weighted_window.py b/pydfc/dfc_methods/derivative_weighted_window.py new file mode 100644 index 0000000..c6d1ad2 --- /dev/null +++ b/pydfc/dfc_methods/derivative_weighted_window.py @@ -0,0 +1,107 @@ +""" +Derivative-weighted window dFC method. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class DERIVATIVE_WEIGHTED_WINDOW(BaseDFCMethod): + MEASURE_NAME = "DerivativeWeightedWindow" + """Windowed correlation weighted toward high-amplitude temporal changes.""" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["W"] is None: + self.params["W"] = 30 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _samples(self, value, Fs, minimum=1): + return max(int(round(value * Fs)), minimum) + + def _corr_from_cov(self, covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 0, + ) + corr[np.diag_indices_from(corr)] = 1 + corr[np.isnan(corr)] = 0 + return corr + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0) + weights_sum = np.sum(weights) + if weights_sum <= 0: + weights = np.ones(samples.shape[1], dtype=float) + weights_sum = np.sum(weights) + weights = weights / weights_sum + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + def dFC(self, time_series, Fs): + window = self._samples(self.params["W"], Fs, minimum=2) + derivative_energy = np.zeros(time_series.shape[1]) + derivative_energy[1:] = np.mean(np.abs(np.diff(time_series, axis=1)), axis=0) + FCSs = [] + TR_array = [] + for tr in range(window - 1, time_series.shape[1]): + start = tr - window + 1 + weights = derivative_energy[start : tr + 1] + weights = weights + 0.1 * np.mean(weights + 1e-8) + FCSs.append(self._weighted_corr(time_series[:, start : tr + 1], weights)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/differential_coactivation.py b/pydfc/dfc_methods/differential_coactivation.py new file mode 100644 index 0000000..a26df02 --- /dev/null +++ b/pydfc/dfc_methods/differential_coactivation.py @@ -0,0 +1,100 @@ +""" +Differential Coactivation FC (DiffCoact) — novel dFC method. + +Instead of correlating BOLD amplitude levels, correlates first-order +temporal differences (dx_i(t) = x_i(t) - x_i(t-1)). Two regions are +"differentially co-activated" when they change in the same direction at +the same instant — a measure of co-derivative rather than co-level. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class DIFFERENTIAL_COACTIVATION(BaseDFCMethod): + """Windowed correlation of first temporal differences of BOLD signals. + + For each region i, compute dx_i(t) = x_i(t) - x_i(t-1). Within each + sliding window the Pearson correlation matrix of these difference signals + is the dFC estimate. The measure answers: do these regions consistently + accelerate and decelerate together, independent of their absolute levels? + """ + + MEASURE_NAME = "DifferentialCoactivationFC" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for p in self.params_name_lst: + self.params[p] = params.get(p, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def dFC(self, time_series, Fs): + # Compute first differences across the entire signal + dx = np.diff(time_series, axis=1) # [n_regions, T-1] + n_regions, T_dx = dx.shape + + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + + FCSs, TR_array = [], [] + for l in range(0, T_dx - W_samples + 1, step): + seg = dx[:, l : l + W_samples] + C = np.corrcoef(seg) + C[np.isnan(C)] = 0.0 + np.fill_diagonal(C, 1.0) + FCSs.append(np.clip(C, -1.0, 1.0)) + TR_array.append(int(l + 1 + W_samples // 2)) # +1 offset for diff + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/differential_edge_kalman_hybrid.py b/pydfc/dfc_methods/differential_edge_kalman_hybrid.py new file mode 100644 index 0000000..88e8f1a --- /dev/null +++ b/pydfc/dfc_methods/differential_edge_kalman_hybrid.py @@ -0,0 +1,286 @@ +""" +DifferentialEdgeKalman_hybrid. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class DIFFERENTIAL_EDGE_KALMAN_HYBRID(BaseDFCMethod): + """State-free hybrid of DifferentialCoactivationFC, EdgeCoactivation, KalmanCovariance.""" + + MEASURE_NAME = "DifferentialEdgeKalman_hybrid" + COMPONENTS = ['DifferentialCoactivationFC', 'EdgeCoactivation', 'KalmanCovariance'] + HYBRID_RULE = "derivative_gate" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/discrete_hmm.py b/pydfc/dfc_methods/discrete_hmm.py index bb706fd..d146572 100644 --- a/pydfc/dfc_methods/discrete_hmm.py +++ b/pydfc/dfc_methods/discrete_hmm.py @@ -41,6 +41,7 @@ class HMM_DISC(BaseDFCMethod): + MEASURE_NAME = "DiscreteHMM" def __init__(self, **params): self.logs_ = "" @@ -89,7 +90,7 @@ def __init__(self, **params): else: self.params[params_name] = None - self.params["measure_name"] = "DiscreteHMM" + self.params["measure_name"] = self.MEASURE_NAME self.params["is_state_based"] = True assert ( diff --git a/pydfc/dfc_methods/dynamic_partial_correlation.py b/pydfc/dfc_methods/dynamic_partial_correlation.py new file mode 100644 index 0000000..0fe4ded --- /dev/null +++ b/pydfc/dfc_methods/dynamic_partial_correlation.py @@ -0,0 +1,120 @@ +""" +Dynamic Partial Correlation (DyPC) dFC method. + +Reference: Smith et al. (2011). Network modelling methods for fMRI. NeuroImage, +54(2), 875-891. doi:10.1016/j.neuroimage.2010.08.063 + +Also: Varoquaux & Craddock (2013). Learning and comparing functional connectomes +across subjects. NeuroImage, 80, 405-415. doi:10.1016/j.neuroimage.2013.04.007 +""" + +import time + +import numpy as np +from sklearn.covariance import LedoitWolf + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class DYNAMIC_PARTIAL_CORRELATION(BaseDFCMethod): + """Sliding-window partial correlation via Ledoit-Wolf regularised precision. + + Within each window, the regularised covariance (Ledoit-Wolf optimal + shrinkage) is inverted to the precision matrix Θ. Partial correlation is + then obtained by symmetric normalisation: + + PartCorr_ij = −Θ_ij / √(Θ_ii · Θ_jj) + + Partial correlation controls for the linear influence of all other regions, + yielding sparser, more direct coupling estimates than full Pearson + correlation and avoiding inflated connectivity due to shared third-region + drivers. + """ + + MEASURE_NAME = "DynamicPartialCorrelation" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + @staticmethod + def _precision_to_partial_corr(precision): + """Normalise precision matrix to partial correlation scale.""" + d = np.sqrt(np.abs(np.diag(precision))) + denom = np.outer(d, d) + denom[denom == 0] = 1.0 + pcorr = -precision / denom + np.fill_diagonal(pcorr, 1.0) + return np.clip(pcorr, -1.0, 1.0) + + def dFC(self, time_series, Fs): + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + n_regions, T = time_series.shape + + lw = LedoitWolf(assume_centered=False) + FCSs, TR_array = [], [] + + for l in range(0, T - W_samples + 1, step): + seg = time_series[:, l : l + W_samples].T # [W, R] + try: + lw.fit(seg) + C = self._precision_to_partial_corr(lw.precision_) + except Exception: + C = np.zeros((n_regions, n_regions)) + np.fill_diagonal(C, 1.0) + FCSs.append(C) + TR_array.append(int(l + W_samples // 2)) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/edge_coactivation.py b/pydfc/dfc_methods/edge_coactivation.py new file mode 100644 index 0000000..6307cf6 --- /dev/null +++ b/pydfc/dfc_methods/edge_coactivation.py @@ -0,0 +1,96 @@ +""" +Edge co-activation dFC method. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class EDGE_COACTIVATION(BaseDFCMethod): + MEASURE_NAME = "EdgeCoactivation" + """Smoothed edge co-activation from instantaneous z-scored products.""" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "half_life", + "min_periods", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + "alpha", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["min_periods"] is None: + self.params["min_periods"] = 10 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + if self.params["alpha"] is not None: + return float(self.params["alpha"]) + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + def dFC(self, time_series, Fs): + alpha = self._alpha_from_half_life(Fs) + min_periods = int(self.params["min_periods"]) + mean = np.zeros(time_series.shape[0]) + var = np.ones(time_series.shape[0]) + edge_state = np.eye(time_series.shape[0]) + FCSs = [] + TR_array = [] + for tr in range(time_series.shape[1]): + sample = time_series[:, tr] + delta = sample - mean + mean = (1.0 - alpha) * mean + alpha * sample + var = (1.0 - alpha) * var + alpha * delta**2 + z_sample = (sample - mean) / np.sqrt(var + 1e-8) + edge_state = (1.0 - alpha) * edge_state + alpha * np.outer(z_sample, z_sample) + matrix = edge_state.copy() + matrix[np.diag_indices_from(matrix)] = 1 + if tr >= min_periods - 1: + FCSs.append(matrix) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/edge_kalman_exp_hybrid_ensemble.py b/pydfc/dfc_methods/edge_kalman_exp_hybrid_ensemble.py new file mode 100644 index 0000000..b89835b --- /dev/null +++ b/pydfc/dfc_methods/edge_kalman_exp_hybrid_ensemble.py @@ -0,0 +1,286 @@ +""" +EdgeKalmanExp_hybrid_ensemble. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class EDGE_KALMAN_EXP_HYBRID_ENSEMBLE(BaseDFCMethod): + """State-free hybrid of EdgeCoactivation, KalmanCovariance, ExponentialWindow.""" + + MEASURE_NAME = "EdgeKalmanExp_hybrid_ensemble" + COMPONENTS = ['EdgeCoactivation', 'KalmanCovariance', 'ExponentialWindow'] + HYBRID_RULE = "ensemble_mean" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/edge_random_reservoir_hybrid_ensemble.py b/pydfc/dfc_methods/edge_random_reservoir_hybrid_ensemble.py new file mode 100644 index 0000000..612e7d7 --- /dev/null +++ b/pydfc/dfc_methods/edge_random_reservoir_hybrid_ensemble.py @@ -0,0 +1,286 @@ +""" +EdgeRandomReservoir_hybrid_ensemble. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class EDGE_RANDOM_RESERVOIR_HYBRID_ENSEMBLE(BaseDFCMethod): + """State-free hybrid of EdgeCoactivation, RandomFourierDependence, ReservoirEchoStateFC.""" + + MEASURE_NAME = "EdgeRandomReservoir_hybrid_ensemble" + COMPONENTS = ['EdgeCoactivation', 'RandomFourierDependence', 'ReservoirEchoStateFC'] + HYBRID_RULE = "ensemble_mean" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/edge_stft_quantum_hybrid.py b/pydfc/dfc_methods/edge_stft_quantum_hybrid.py new file mode 100644 index 0000000..96202cc --- /dev/null +++ b/pydfc/dfc_methods/edge_stft_quantum_hybrid.py @@ -0,0 +1,286 @@ +""" +EdgeStftQuantum_hybrid. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class EDGE_STFT_QUANTUM_HYBRID(BaseDFCMethod): + """State-free hybrid of EdgeCoactivation, STFTCoherence, QuantumMutualInformationFC.""" + + MEASURE_NAME = "EdgeStftQuantum_hybrid" + COMPONENTS = ['EdgeCoactivation', 'Time-Freq', 'QuantumMutualInformationFC'] + HYBRID_RULE = "spectral_information_gate" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/event_synchronization.py b/pydfc/dfc_methods/event_synchronization.py new file mode 100644 index 0000000..2afc940 --- /dev/null +++ b/pydfc/dfc_methods/event_synchronization.py @@ -0,0 +1,98 @@ +""" +Event synchronization dFC method. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class EVENT_SYNCHRONIZATION(BaseDFCMethod): + MEASURE_NAME = "EventSynchronization" + """FC from co-occurring high-amplitude activity events.""" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "event_quantile", + "event_decay", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 10 + if self.params["event_quantile"] is None: + self.params["event_quantile"] = 0.85 + if self.params["event_decay"] is None: + self.params["event_decay"] = 0.97 + + @property + def measure_name(self): + return self.params["measure_name"] + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + thresholds = np.quantile( + np.abs(time_series), self.params["event_quantile"], axis=1 + ) + event_rate = np.zeros(time_series.shape[0]) + coincidence = np.eye(time_series.shape[0]) + FCSs = [] + TR_array = [] + for tr in range(time_series.shape[1]): + event = (np.abs(time_series[:, tr]) >= thresholds).astype(float) + event_rate = self.params["event_decay"] * event_rate + event + coincidence = self.params["event_decay"] * coincidence + np.outer( + event, event + ) + scale = np.sqrt(np.outer(event_rate, event_rate)) + matrix = np.divide( + coincidence, + scale, + out=np.zeros_like(coincidence), + where=scale > 0, + ) + matrix[np.diag_indices_from(matrix)] = 1 + if tr >= min_periods - 1: + FCSs.append(matrix) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/exponential_window.py b/pydfc/dfc_methods/exponential_window.py new file mode 100644 index 0000000..419c32f --- /dev/null +++ b/pydfc/dfc_methods/exponential_window.py @@ -0,0 +1,114 @@ +""" +Exponentially weighted dFC method. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class EXPONENTIAL_WINDOW(BaseDFCMethod): + MEASURE_NAME = "ExponentialWindow" + """Exponentially weighted correlation with a smooth memory decay.""" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "half_life", + "min_periods", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + "alpha", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["min_periods"] is None: + self.params["min_periods"] = 10 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + if self.params["alpha"] is not None: + return float(self.params["alpha"]) + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + def _corr_from_cov(self, covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 0, + ) + corr[np.diag_indices_from(corr)] = 1 + corr[np.isnan(corr)] = 0 + return corr + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0) + weights_sum = np.sum(weights) + if weights_sum <= 0: + weights = np.ones(samples.shape[1], dtype=float) + weights_sum = np.sum(weights) + weights = weights / weights_sum + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + alpha = self._alpha_from_half_life(Fs) + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + age = tr - np.arange(tr + 1) + weights = alpha * np.power(1.0 - alpha, age) + FCSs.append(self._weighted_corr(time_series[:, : tr + 1], weights)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/gaussian_mixture_states.py b/pydfc/dfc_methods/gaussian_mixture_states.py new file mode 100644 index 0000000..5e6cd85 --- /dev/null +++ b/pydfc/dfc_methods/gaussian_mixture_states.py @@ -0,0 +1,148 @@ +"""Gaussian mixture state-based dFC.""" + +import time + +import numpy as np +from sklearn.mixture import GaussianMixture + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +def _corr(samples): + if samples.shape[0] < 2: + return np.eye(samples.shape[1], dtype=float) + cov = np.cov(samples, rowvar=False) + std = np.sqrt(np.maximum(np.diag(cov), 1e-12)) + den = np.outer(std, std) + corr = np.divide(cov, den, out=np.zeros_like(cov), where=den > 0) + corr[np.diag_indices_from(corr)] = 1.0 + return 0.5 * (corr + corr.T) + + +class GAUSSIAN_MIXTURE_STATES(BaseDFCMethod): + MEASURE_NAME = "GaussianMixtureStates" + """Elliptical state emissions learned with a Gaussian mixture.""" + + def __init__(self, **params): + self._covariance_type = "full" + self._reg_covar = 1e-6 + self._max_iter = 300 + self._train_sample_limit = 5000 + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "n_states", + "normalization", + "num_subj", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {name: params.get(name, None) for name in self.params_name_lst} + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = True + if self.params["n_states"] is None: + self.params["n_states"] = 5 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _chunks(self, time_series): + return [ + time_series.get_subj_ts(subjs_id=subj).data.T.copy() + for subj in time_series.subj_id_lst + ] + + def _fit_matrix(self, chunks): + each = max(1, int(self._train_sample_limit) // max(len(chunks), 1)) + sampled = [] + for chunk in chunks: + if chunk.shape[0] <= each: + sampled.append(chunk) + else: + idx = np.linspace(0, chunk.shape[0] - 1, each, dtype=int) + sampled.append(chunk[idx, :]) + return np.concatenate(sampled, axis=0) + + def estimate_FCS(self, time_series): + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4FCS(time_series) + tic = time.time() + + chunks = self._chunks(time_series) + fit_matrix = self._fit_matrix(chunks) + n_states = min(int(self.params["n_states"]), fit_matrix.shape[0]) + self.params["n_states"] = n_states + + self.gmm_ = GaussianMixture( + n_components=n_states, + covariance_type=self._covariance_type, + reg_covar=self._reg_covar, + max_iter=self._max_iter, + random_state=None, + ).fit(fit_matrix) + + labels_chunks = [self.gmm_.predict(chunk).astype(int) for chunk in chunks] + self.Z = np.concatenate(labels_chunks, axis=0) + self.FCS_ = np.zeros( + (n_states, chunks[0].shape[1], chunks[0].shape[1]), dtype=float + ) + for state in range(n_states): + samples = [ + chunk[labels == state, :] + for chunk, labels in zip(chunks, labels_chunks) + if np.any(labels == state) + ] + self.FCS_[state, :, :] = ( + _corr(np.concatenate(samples, axis=0)) + if samples + else np.eye(chunks[0].shape[1]) + ) + + counts = np.full((n_states, n_states), 1.0, dtype=float) + for labels in labels_chunks: + for a, b in zip(labels[:-1], labels[1:]): + counts[a, b] += 1.0 + self.TPM = counts / np.maximum(np.sum(counts, axis=1, keepdims=True), 1e-12) + + self.set_mean_activity(time_series) + self.set_FCS_fit_time(time.time() - tic) + return self + + def estimate_dFC(self, time_series): + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + + features = time_series.data.T.copy() + probs = self.gmm_.predict_proba(features) + labels = np.argmax(probs, axis=1).astype(int) + + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC( + FCSs=self.FCS_, + FCS_idx=labels, + FCS_proba=probs, + TS_info=time_series.info_dict, + TR_array=np.arange(features.shape[0], dtype=int), + ) + return dFC diff --git a/pydfc/dfc_methods/graph_diffusion_coactivation.py b/pydfc/dfc_methods/graph_diffusion_coactivation.py new file mode 100644 index 0000000..ce1b7e5 --- /dev/null +++ b/pydfc/dfc_methods/graph_diffusion_coactivation.py @@ -0,0 +1,111 @@ +""" +Graph diffusion co-activation dFC method. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class GRAPH_DIFFUSION_COACTIVATION(BaseDFCMethod): + MEASURE_NAME = "GraphDiffusionCoactivation" + """Instantaneous co-activation propagated through a learned graph.""" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "half_life", + "min_periods", + "diffusion_rate", + "instantaneous_weight", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + "alpha", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["min_periods"] is None: + self.params["min_periods"] = 10 + if self.params["diffusion_rate"] is None: + self.params["diffusion_rate"] = 0.2 + if self.params["instantaneous_weight"] is None: + self.params["instantaneous_weight"] = 0.15 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + if self.params["alpha"] is not None: + return float(self.params["alpha"]) + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + alpha = self._alpha_from_half_life(Fs) + n_regions = time_series.shape[0] + mean = np.zeros(n_regions) + var = np.ones(n_regions) + graph = np.eye(n_regions) + FCSs = [] + TR_array = [] + for tr in range(time_series.shape[1]): + sample = time_series[:, tr] + delta = sample - mean + mean = (1.0 - alpha) * mean + alpha * sample + var = (1.0 - alpha) * var + alpha * delta**2 + z_sample = (sample - mean) / np.sqrt(var + 1e-8) + instant = np.tanh(np.outer(z_sample, z_sample)) + degree = np.sum(np.abs(graph), axis=1, keepdims=True) + 1e-8 + transition = graph / degree + diffused = transition @ graph @ transition.T + graph = (1.0 - self.params["instantaneous_weight"]) * diffused + self.params[ + "instantaneous_weight" + ] * instant + graph = (1.0 - self.params["diffusion_rate"]) * graph + ( + self.params["diffusion_rate"] * 0.5 * (graph + graph.T) + ) + graph[np.diag_indices_from(graph)] = 1 + if tr >= min_periods - 1: + FCSs.append(np.clip(graph.copy(), -1, 1)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/instantaneous_phase_coherence.py b/pydfc/dfc_methods/instantaneous_phase_coherence.py new file mode 100644 index 0000000..8087b63 --- /dev/null +++ b/pydfc/dfc_methods/instantaneous_phase_coherence.py @@ -0,0 +1,116 @@ +""" +Instantaneous Phase Coherence (IPC) dFC method. + +Reference: Glerean et al. (2012). Functional Magnetic Resonance Imaging Phase +Synchronization as a Measure of Dynamic Functional Connectivity. Brain Connectivity, +2(6), 353-365. doi:10.1089/brain.2012.0088 + +Also: Cabral et al. (2014). Exploring the network dynamics underlying brain activity +during rest. Progress in Neurobiology, 114, 102-122. +doi:10.1016/j.pneurobio.2013.12.005 +""" + +import time + +import numpy as np +from scipy.signal import butter, filtfilt, hilbert + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class INSTANTANEOUS_PHASE_COHERENCE(BaseDFCMethod): + """Frame-by-frame FC from instantaneous BOLD phase differences (windowless). + + Each region's BOLD signal is bandpass-filtered to the low-frequency + resting-state band (default 0.01–0.1 Hz) and Hilbert-transformed to + extract the instantaneous phase φ_i(t). The functional connectivity + between regions i and j at time t is + + FC_ij(t) = cos(φ_i(t) − φ_j(t)) + + which equals +1 for in-phase synchrony, −1 for anti-phase, and 0 for + quadrature relationships. No sliding window is required: every TR yields + a full symmetric FC matrix, making this the highest temporal-resolution + method in the suite. + """ + + MEASURE_NAME = "InstantaneousPhaseCoherence" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "f_low", + "f_high", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["f_low"] is None: + self.params["f_low"] = 0.01 + if self.params["f_high"] is None: + self.params["f_high"] = 0.1 + + @property + def measure_name(self): + return self.params["measure_name"] + + def dFC(self, time_series, Fs): + n_regions, T = time_series.shape + f_low = float(self.params["f_low"]) + f_high = float(self.params["f_high"]) + nyq = Fs / 2.0 + + if f_low > 0 and f_high < nyq: + b, a = butter(4, [f_low / nyq, f_high / nyq], btype="band") + filtered = filtfilt(b, a, time_series, axis=1) + else: + filtered = time_series.copy() + + phases = np.angle(hilbert(filtered, axis=1)) # [n_regions, T] + + # FC_ij(t) = cos(φ_i(t) − φ_j(t)) + # cos(a-b) = cos(a)cos(b) + sin(a)sin(b) + cos_phi = np.cos(phases) + sin_phi = np.sin(phases) + FCSs = np.einsum("it,jt->tij", cos_phi, cos_phi) + np.einsum( + "it,jt->tij", sin_phi, sin_phi + ) # [T, R, R]; diagonal is cos(0) = 1 by construction + + TR_array = np.arange(T) + return FCSs, TR_array + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/kalman_covariance.py b/pydfc/dfc_methods/kalman_covariance.py new file mode 100644 index 0000000..48d28ec --- /dev/null +++ b/pydfc/dfc_methods/kalman_covariance.py @@ -0,0 +1,112 @@ +""" +Kalman-style covariance dFC method. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class KALMAN_COVARIANCE(BaseDFCMethod): + MEASURE_NAME = "KalmanCovariance" + """Recursive covariance tracking with a Kalman-style process floor.""" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "half_life", + "min_periods", + "process_noise", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + "alpha", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["min_periods"] is None: + self.params["min_periods"] = 10 + if self.params["process_noise"] is None: + self.params["process_noise"] = 1e-4 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + if self.params["alpha"] is not None: + return float(self.params["alpha"]) + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + def _corr_from_cov(self, covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 0, + ) + corr[np.diag_indices_from(corr)] = 1 + corr[np.isnan(corr)] = 0 + return corr + + def dFC(self, time_series, Fs): + alpha = self._alpha_from_half_life(Fs) + min_periods = int(self.params["min_periods"]) + n_regions = time_series.shape[0] + mean = time_series[:, 0].copy() + covariance = np.eye(n_regions) + FCSs = [] + TR_array = [] + for tr in range(1, time_series.shape[1]): + sample = time_series[:, tr] + innovation = sample - mean + mean = mean + alpha * innovation + covariance = ( + (1.0 - alpha) * covariance + + alpha * np.outer(innovation, innovation) + + self.params["process_noise"] * np.eye(n_regions) + ) + if tr >= min_periods - 1: + FCSs.append(self._corr_from_cov(covariance)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/kalman_reservoir_multiscale_hybrid.py b/pydfc/dfc_methods/kalman_reservoir_multiscale_hybrid.py new file mode 100644 index 0000000..4e221e4 --- /dev/null +++ b/pydfc/dfc_methods/kalman_reservoir_multiscale_hybrid.py @@ -0,0 +1,286 @@ +""" +KalmanReservoirMultiscale_hybrid. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class KALMAN_RESERVOIR_MULTISCALE_HYBRID(BaseDFCMethod): + """State-free hybrid of KalmanCovariance, ReservoirEchoStateFC, MultiscaleWindow.""" + + MEASURE_NAME = "KalmanReservoirMultiscale_hybrid" + COMPONENTS = ['KalmanCovariance', 'ReservoirEchoStateFC', 'MultiscaleWindow'] + HYBRID_RULE = "state_multiscale_residual" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/lagged_kmeans_states.py b/pydfc/dfc_methods/lagged_kmeans_states.py new file mode 100644 index 0000000..53247c0 --- /dev/null +++ b/pydfc/dfc_methods/lagged_kmeans_states.py @@ -0,0 +1,179 @@ +"""Lagged k-means state-based dFC.""" + +import time + +import numpy as np +from sklearn.cluster import KMeans + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +def _lagged(features, lag): + lag = max(int(lag), 1) + if lag == 1: + return features + blocks = [] + for offset in range(lag): + if offset == 0: + blocks.append(features) + else: + pad = np.repeat(features[:1, :], offset, axis=0) + blocks.append(np.vstack((pad, features[:-offset, :]))) + return np.concatenate(blocks, axis=1) + + +def _corr(samples): + if samples.shape[0] < 2: + return np.eye(samples.shape[1], dtype=float) + cov = np.cov(samples, rowvar=False) + std = np.sqrt(np.maximum(np.diag(cov), 1e-12)) + den = np.outer(std, std) + corr = np.divide(cov, den, out=np.zeros_like(cov), where=den > 0) + corr[np.diag_indices_from(corr)] = 1.0 + return 0.5 * (corr + corr.T) + + +def _softmax_dist(features, centers, temperature): + distances = np.sum((features[:, None, :] - centers[None, :, :]) ** 2, axis=2) + logits = -distances / max(float(temperature), 1e-6) + logits = logits - np.max(logits, axis=1, keepdims=True) + probs = np.exp(logits) + probs = probs / np.maximum(np.sum(probs, axis=1, keepdims=True), 1e-12) + return np.argmin(distances, axis=1).astype(int), probs + + +class LAGGED_KMEANS_STATES(BaseDFCMethod): + MEASURE_NAME = "LaggedKMeansStates" + """State prototypes estimated on lag-augmented activity vectors.""" + + def __init__(self, **params): + self._n_init = 20 + self._train_sample_limit = 5000 + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "n_states", + "lag", + "assignment_temperature", + "normalization", + "num_subj", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {name: params.get(name, None) for name in self.params_name_lst} + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = True + if self.params["n_states"] is None: + self.params["n_states"] = 5 + if self.params["lag"] is None: + self.params["lag"] = 2 + if self.params["assignment_temperature"] is None: + self.params["assignment_temperature"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _chunks(self, time_series): + return [ + time_series.get_subj_ts(subjs_id=subj).data.T.copy() + for subj in time_series.subj_id_lst + ] + + def _fit_matrix(self, chunks): + each = max(1, int(self._train_sample_limit) // max(len(chunks), 1)) + sampled = [] + for chunk in chunks: + if chunk.shape[0] <= each: + sampled.append(chunk) + else: + idx = np.linspace(0, chunk.shape[0] - 1, each, dtype=int) + sampled.append(chunk[idx, :]) + return np.concatenate(sampled, axis=0) + + def estimate_FCS(self, time_series): + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4FCS(time_series) + tic = time.time() + + chunks = self._chunks(time_series) + feature_chunks = [_lagged(chunk, self.params["lag"]) for chunk in chunks] + fit_matrix = self._fit_matrix(feature_chunks) + n_states = min(int(self.params["n_states"]), fit_matrix.shape[0]) + self.params["n_states"] = n_states + + self.kmeans_ = KMeans( + n_clusters=n_states, + n_init=self._n_init, + random_state=None, + ).fit(fit_matrix) + self.centers_ = self.kmeans_.cluster_centers_.astype(float) + + labels_chunks, _ = zip( + *[ + _softmax_dist(chunk, self.centers_, self.params["assignment_temperature"]) + for chunk in feature_chunks + ] + ) + self.Z = np.concatenate(labels_chunks, axis=0) + self.FCS_ = np.zeros( + (n_states, chunks[0].shape[1], chunks[0].shape[1]), dtype=float + ) + for state in range(n_states): + samples = [ + chunk[labels == state, :] + for chunk, labels in zip(chunks, labels_chunks) + if np.any(labels == state) + ] + self.FCS_[state, :, :] = ( + _corr(np.concatenate(samples, axis=0)) + if samples + else np.eye(chunks[0].shape[1]) + ) + + counts = np.full((n_states, n_states), 1.0, dtype=float) + for labels in labels_chunks: + for a, b in zip(labels[:-1], labels[1:]): + counts[a, b] += 1.0 + self.TPM = counts / np.maximum(np.sum(counts, axis=1, keepdims=True), 1e-12) + + self.set_mean_activity(time_series) + self.set_FCS_fit_time(time.time() - tic) + return self + + def estimate_dFC(self, time_series): + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + features = _lagged(time_series.data.T.copy(), self.params["lag"]) + labels, probs = _softmax_dist( + features, self.centers_, self.params["assignment_temperature"] + ) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC( + FCSs=self.FCS_, + FCS_idx=labels, + FCS_proba=probs, + TS_info=time_series.info_dict, + TR_array=np.arange(features.shape[0], dtype=int), + ) + return dFC diff --git a/pydfc/dfc_methods/lagged_max_correlation.py b/pydfc/dfc_methods/lagged_max_correlation.py new file mode 100644 index 0000000..217efc5 --- /dev/null +++ b/pydfc/dfc_methods/lagged_max_correlation.py @@ -0,0 +1,108 @@ +""" +Lagged maximum correlation dFC method. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class LAGGED_MAX_CORRELATION(BaseDFCMethod): + MEASURE_NAME = "LaggedMaxCorrelation" + """Windowed FC using the strongest short-lag pairwise correlation.""" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "max_lag", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["max_lag"] is None: + self.params["max_lag"] = 2 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _samples(self, value, Fs, minimum=1): + return max(int(round(value * Fs)), minimum) + + def _corr(self, samples): + corr = np.corrcoef(samples) + corr[np.isnan(corr)] = 0 + corr[np.diag_indices_from(corr)] = 1 + return corr + + def _standardize(self, data): + centered = data - np.mean(data, axis=1, keepdims=True) + scale = np.std(centered, axis=1, keepdims=True) + return np.divide(centered, scale, out=np.zeros_like(centered), where=scale > 0) + + def _lagged_corr(self, segment): + max_lag = int(self.params["max_lag"]) + best = self._corr(segment) + best_abs = np.abs(best) + for lag in range(1, max_lag + 1): + if segment.shape[1] <= lag + 1: + break + lead = self._standardize(segment[:, lag:]) + trail = self._standardize(segment[:, :-lag]) + corr = (lead @ trail.T) / max(lead.shape[1] - 1, 1) + corr = 0.5 * (corr + corr.T) + replace = np.abs(corr) > best_abs + best[replace] = corr[replace] + best_abs[replace] = np.abs(corr[replace]) + best[np.diag_indices_from(best)] = 1 + return best + + def dFC(self, time_series, Fs): + window = self._samples(self.params["W"], Fs, minimum=2) + FCSs = [] + TR_array = [] + for tr in range(window - 1, time_series.shape[1]): + segment = time_series[:, tr - window + 1 : tr + 1] + FCSs.append(self._lagged_corr(segment)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/leading_eigenvector_dynamics.py b/pydfc/dfc_methods/leading_eigenvector_dynamics.py new file mode 100644 index 0000000..b8a3ba9 --- /dev/null +++ b/pydfc/dfc_methods/leading_eigenvector_dynamics.py @@ -0,0 +1,111 @@ +""" +Leading Eigenvector Dynamics (LED) dFC method. + +Reference: Leonardi & Van De Ville (2015). On spurious and real fluctuations of +dynamic functional connectivity during rest. NeuroImage, 114, 430-436. +doi:10.1016/j.neuroimage.2015.04.004 +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class LEADING_EIGENVECTOR_DYNAMICS(BaseDFCMethod): + """Rank-1 dFC estimate from the leading eigenvector of the windowed FC matrix. + + Within each sliding window the standard Pearson correlation matrix C is + computed, then eigendecomposed. The dFC estimate at that window centre is + the rank-1 reconstruction v vᵀ where v is the eigenvector belonging to the + largest eigenvalue. This low-rank projection retains the dominant mode of + co-fluctuation while suppressing the noise contained in smaller eigenmodes, + yielding a smoother and more robust instantaneous FC estimate. + """ + + MEASURE_NAME = "LeadingEigenvectorDynamics" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + @staticmethod + def _rank1_from_eigvec(v): + """Symmetric rank-1 correlation-scale matrix from unit eigenvector v.""" + r1 = np.outer(v, v) + diag = np.sqrt(np.abs(np.diag(r1))) + denom = np.outer(diag, diag) + denom[denom == 0] = 1.0 + r1 = r1 / denom + np.fill_diagonal(r1, 1.0) + return np.clip(r1, -1.0, 1.0) + + def dFC(self, time_series, Fs): + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + n_regions, T = time_series.shape + + FCSs, TR_array = [], [] + for l in range(0, T - W_samples + 1, step): + seg = time_series[:, l : l + W_samples] + C = np.corrcoef(seg) + C[np.isnan(C)] = 0.0 + np.fill_diagonal(C, 1.0) + vals, vecs = np.linalg.eigh(C) + v = vecs[:, -1] # eigenvector of the largest eigenvalue + FCSs.append(self._rank1_from_eigvec(v)) + TR_array.append(int(l + W_samples // 2)) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/local_jacobian_coupling.py b/pydfc/dfc_methods/local_jacobian_coupling.py new file mode 100644 index 0000000..305ad03 --- /dev/null +++ b/pydfc/dfc_methods/local_jacobian_coupling.py @@ -0,0 +1,160 @@ +""" +Local Jacobian Coupling Connectivity (LJCC) — from dynamical systems theory. + +Source domain: Takens (1981). Detecting strange attractors in turbulence. +Lecture Notes in Mathematics, 898, 366-381. +Kantz & Schreiber (2004). Nonlinear Time Series Analysis. Cambridge. + +The Jacobian J(t) of a dynamical system at time t gives the instantaneous +linear sensitivity of each state variable to all others. Reconstructing +the local Jacobian from the delay-embedded BOLD trajectory via sliding-window +least-squares regression yields a time-varying coupling matrix J_ij(t): +the instantaneous influence of region j's state on region i's next state. +Symmetrising |J_ij + J_ji|/2 gives an undirected measure of dynamical +coupling — connectivity as local phase-space sensitivity. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class LOCAL_JACOBIAN_COUPLING(BaseDFCMethod): + """Windowed local Jacobian from delay-embedded BOLD trajectory. + + Each region's BOLD is delay-embedded into an m-dimensional phase space + state. Within each sliding window the local linear map + + X(t+1) ≈ J(t) X(t) + + is fit by least squares, where X(t) is the [R·m, W−1] matrix of + delay-embedded states. The instantaneous coupling between regions i and + j is read from the top-left [R, R] block of J(t) (which captures how + region j's current activity influences region i's next activity), + and is symmetrised: + + FC[i,j] = (|J_ij| + |J_ji|) / 2, normalised to [0, 1]. + + Unlike linear correlation, the Jacobian captures the local dynamical + geometry of the BOLD attractor, sensitive to transient coupling states. + """ + + MEASURE_NAME = "LocalJacobianCouplingFC" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "embed_dim", + "embed_lag", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for p in self.params_name_lst: + self.params[p] = params.get(p, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["embed_dim"] is None: + self.params["embed_dim"] = 2 + if self.params["embed_lag"] is None: + self.params["embed_lag"] = 1 + + @property + def measure_name(self): + return self.params["measure_name"] + + @staticmethod + def _delay_embed(x, m, lag): + """Delay-embed [R, T] data into [R*m, T-(m-1)*lag] matrix.""" + R, T = x.shape + offset = (m - 1) * lag + T_emb = T - offset + if T_emb <= 0: + return x[:, :1] + X = np.zeros((R * m, T_emb)) + for k in range(m): + X[k * R : (k + 1) * R, :] = x[:, offset - k * lag : T - k * lag] + return X + + def dFC(self, time_series, Fs): + n_regions, T = time_series.shape + m = int(self.params["embed_dim"]) + lag = int(self.params["embed_lag"]) + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + + X_emb = self._delay_embed(time_series, m, lag) # [R*m, T_emb] + T_emb = X_emb.shape[1] + offset = (m - 1) * lag + + FCSs, TR_array = [], [] + for l in range(0, T_emb - W_samples + 1, step): + seg = X_emb[:, l : l + W_samples] # [R*m, W] + X0 = seg[:, :-1] # [R*m, W-1] + X1 = seg[:, 1:] # [R*m, W-1] + + # Least-squares local Jacobian: J = X1 @ pinv(X0) + try: + J = X1 @ np.linalg.pinv(X0, rcond=1e-6) # [R*m, R*m] + except np.linalg.LinAlgError: + J = np.zeros((n_regions * m, n_regions * m)) + + # Top-left [R, R] block: instantaneous coupling + J_sub = J[:n_regions, :n_regions] + + # Symmetrised absolute coupling + FC = (np.abs(J_sub) + np.abs(J_sub.T)) / 2.0 + + # Normalise to [0, 1] + upper = FC[np.triu_indices(n_regions, k=1)] + fc_max = upper.max() if len(upper) > 0 else 1.0 + if fc_max > 1e-10: + FC = FC / fc_max + np.fill_diagonal(FC, 1.0) + FC = np.clip(FC, 0.0, 1.0) + + FCSs.append(FC) + TR_array.append(int(offset + l + W_samples // 2)) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/markov_smoothed_gmm_states.py b/pydfc/dfc_methods/markov_smoothed_gmm_states.py new file mode 100644 index 0000000..c30f988 --- /dev/null +++ b/pydfc/dfc_methods/markov_smoothed_gmm_states.py @@ -0,0 +1,210 @@ +"""Markov-smoothed Gaussian mixture state-based dFC.""" + +import time + +import numpy as np +from sklearn.mixture import GaussianMixture + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +def _corr(samples): + if samples.shape[0] < 2: + return np.eye(samples.shape[1], dtype=float) + cov = np.cov(samples, rowvar=False) + std = np.sqrt(np.maximum(np.diag(cov), 1e-12)) + den = np.outer(std, std) + corr = np.divide(cov, den, out=np.zeros_like(cov), where=den > 0) + corr[np.diag_indices_from(corr)] = 1.0 + return 0.5 * (corr + corr.T) + + +def _viterbi(emission_logp, tpm, startprob): + n_time, n_states = emission_logp.shape + log_tpm = np.log(np.maximum(tpm, 1e-12)) + log_start = np.log(np.maximum(startprob, 1e-12)) + dp = np.zeros((n_time, n_states), dtype=float) + bp = np.zeros((n_time, n_states), dtype=int) + dp[0, :] = log_start + emission_logp[0, :] + for t in range(1, n_time): + scores = dp[t - 1, :, None] + log_tpm + bp[t, :] = np.argmax(scores, axis=0) + dp[t, :] = scores[bp[t, :], np.arange(n_states)] + emission_logp[t, :] + z = np.zeros((n_time,), dtype=int) + z[-1] = np.argmax(dp[-1, :]) + for t in range(n_time - 2, -1, -1): + z[t] = bp[t + 1, z[t + 1]] + return z + + +def _forward_backward(emission_logp, tpm, startprob): + """Forward-backward algorithm for soft state posteriors. + + Returns posterior probabilities P(state_t | observations) for each time point. + """ + n_time, n_states = emission_logp.shape + log_tpm = np.log(np.maximum(tpm, 1e-12)) + log_start = np.log(np.maximum(startprob, 1e-12)) + + # Forward pass: α_t(i) = P(obs[0:t], state_t=i) + alpha = np.zeros((n_time, n_states), dtype=float) + alpha[0, :] = log_start + emission_logp[0, :] + for t in range(1, n_time): + alpha[t, :] = emission_logp[t, :] + np.logaddexp.reduce( + alpha[t - 1, :, None] + log_tpm, axis=0 + ) + + # Backward pass: β_t(i) = P(obs[t:] | state_t=i) + beta = np.zeros((n_time, n_states), dtype=float) + beta[-1, :] = 0.0 # log(1) + for t in range(n_time - 2, -1, -1): + beta[t, :] = np.logaddexp.reduce( + log_tpm[:, :] + emission_logp[t + 1, None, :] + beta[t + 1, None, :], + axis=1, + ) + + # Posterior: γ_t(i) = P(state_t=i | observations) + gamma = np.exp( + alpha + beta - np.logaddexp.reduce(alpha + beta, axis=1, keepdims=True) + ) + return gamma + + +class MARKOV_SMOOTHED_GMM_STATES(BaseDFCMethod): + MEASURE_NAME = "MarkovSmoothedGMMStates" + """Gaussian mixture emissions refined by a Markov transition prior.""" + + def __init__(self, **params): + self._covariance_type = "full" + self._reg_covar = 1e-6 + self._max_iter = 300 + self._train_sample_limit = 5000 + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "n_states", + "transition_smoothing", + "normalization", + "num_subj", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {name: params.get(name, None) for name in self.params_name_lst} + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = True + if self.params["n_states"] is None: + self.params["n_states"] = 5 + if self.params["transition_smoothing"] is None: + self.params["transition_smoothing"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _chunks(self, time_series): + return [ + time_series.get_subj_ts(subjs_id=subj).data.T.copy() + for subj in time_series.subj_id_lst + ] + + def _fit_matrix(self, chunks): + each = max(1, int(self._train_sample_limit) // max(len(chunks), 1)) + sampled = [] + for chunk in chunks: + if chunk.shape[0] <= each: + sampled.append(chunk) + else: + idx = np.linspace(0, chunk.shape[0] - 1, each, dtype=int) + sampled.append(chunk[idx, :]) + return np.concatenate(sampled, axis=0) + + def estimate_FCS(self, time_series): + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4FCS(time_series) + tic = time.time() + + chunks = self._chunks(time_series) + fit_matrix = self._fit_matrix(chunks) + n_states = min(int(self.params["n_states"]), fit_matrix.shape[0]) + self.params["n_states"] = n_states + + self.gmm_ = GaussianMixture( + n_components=n_states, + covariance_type=self._covariance_type, + reg_covar=self._reg_covar, + max_iter=self._max_iter, + random_state=None, + ).fit(fit_matrix) + + base_labels = [self.gmm_.predict(chunk).astype(int) for chunk in chunks] + trans = np.full((n_states, n_states), float(self.params["transition_smoothing"])) + start = np.full((n_states,), float(self.params["transition_smoothing"])) + for labels in base_labels: + start[labels[0]] += 1.0 + for a, b in zip(labels[:-1], labels[1:]): + trans[a, b] += 1.0 + self.TPM = trans / np.maximum(np.sum(trans, axis=1, keepdims=True), 1e-12) + self.startprob_ = start / np.maximum(np.sum(start), 1e-12) + + labels_chunks = [] + for chunk in chunks: + emission = np.log(np.maximum(self.gmm_.predict_proba(chunk), 1e-12)) + labels_chunks.append(_viterbi(emission, self.TPM, self.startprob_)) + + self.Z = np.concatenate(labels_chunks, axis=0) + self.FCS_ = np.zeros( + (n_states, chunks[0].shape[1], chunks[0].shape[1]), dtype=float + ) + for state in range(n_states): + samples = [ + chunk[labels == state, :] + for chunk, labels in zip(chunks, labels_chunks) + if np.any(labels == state) + ] + self.FCS_[state, :, :] = ( + _corr(np.concatenate(samples, axis=0)) + if samples + else np.eye(chunks[0].shape[1]) + ) + + self.set_mean_activity(time_series) + self.set_FCS_fit_time(time.time() - tic) + return self + + def estimate_dFC(self, time_series): + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + features = time_series.data.T.copy() + emission = np.log(np.maximum(self.gmm_.predict_proba(features), 1e-12)) + # Use forward-backward for soft state posteriors (compositional data) + proba = _forward_backward(emission, self.TPM, self.startprob_) + labels = np.argmax(proba, axis=1).astype(int) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC( + FCSs=self.FCS_, + FCS_idx=labels, + FCS_proba=proba, + TS_info=time_series.info_dict, + TR_array=np.arange(features.shape[0], dtype=int), + ) + return dFC diff --git a/pydfc/dfc_methods/markov_smoothed_kmeans_states.py b/pydfc/dfc_methods/markov_smoothed_kmeans_states.py new file mode 100644 index 0000000..10a9710 --- /dev/null +++ b/pydfc/dfc_methods/markov_smoothed_kmeans_states.py @@ -0,0 +1,229 @@ +"""Markov-smoothed k-means state-based dFC.""" + +import time + +import numpy as np +from sklearn.cluster import KMeans + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +def _corr(samples): + if samples.shape[0] < 2: + return np.eye(samples.shape[1], dtype=float) + cov = np.cov(samples, rowvar=False) + std = np.sqrt(np.maximum(np.diag(cov), 1e-12)) + den = np.outer(std, std) + corr = np.divide(cov, den, out=np.zeros_like(cov), where=den > 0) + corr[np.diag_indices_from(corr)] = 1.0 + return 0.5 * (corr + corr.T) + + +def _softmax_logits(logits): + logits = logits - np.max(logits, axis=1, keepdims=True) + probs = np.exp(logits) + return probs / np.maximum(np.sum(probs, axis=1, keepdims=True), 1e-12) + + +def _viterbi(emission_logp, tpm, startprob): + n_time, n_states = emission_logp.shape + log_tpm = np.log(np.maximum(tpm, 1e-12)) + log_start = np.log(np.maximum(startprob, 1e-12)) + dp = np.zeros((n_time, n_states), dtype=float) + bp = np.zeros((n_time, n_states), dtype=int) + dp[0, :] = log_start + emission_logp[0, :] + for t in range(1, n_time): + scores = dp[t - 1, :, None] + log_tpm + bp[t, :] = np.argmax(scores, axis=0) + dp[t, :] = scores[bp[t, :], np.arange(n_states)] + emission_logp[t, :] + z = np.zeros((n_time,), dtype=int) + z[-1] = np.argmax(dp[-1, :]) + for t in range(n_time - 2, -1, -1): + z[t] = bp[t + 1, z[t + 1]] + return z + + +def _forward_backward(emission_logp, tpm, startprob): + """Forward-backward algorithm for soft state posteriors. + + Returns posterior probabilities P(state_t | observations) for each time point. + """ + n_time, n_states = emission_logp.shape + log_tpm = np.log(np.maximum(tpm, 1e-12)) + log_start = np.log(np.maximum(startprob, 1e-12)) + + # Forward pass: α_t(i) = P(obs[0:t], state_t=i) + alpha = np.zeros((n_time, n_states), dtype=float) + alpha[0, :] = log_start + emission_logp[0, :] + for t in range(1, n_time): + alpha[t, :] = emission_logp[t, :] + np.logaddexp.reduce( + alpha[t - 1, :, None] + log_tpm, axis=0 + ) + + # Backward pass: β_t(i) = P(obs[t:] | state_t=i) + beta = np.zeros((n_time, n_states), dtype=float) + beta[-1, :] = 0.0 # log(1) + for t in range(n_time - 2, -1, -1): + beta[t, :] = np.logaddexp.reduce( + log_tpm[:, :] + emission_logp[t + 1, None, :] + beta[t + 1, None, :], + axis=1, + ) + + # Posterior: γ_t(i) = P(state_t=i | observations) + gamma = np.exp( + alpha + beta - np.logaddexp.reduce(alpha + beta, axis=1, keepdims=True) + ) + return gamma + + +class MARKOV_SMOOTHED_KMEANS_STATES(BaseDFCMethod): + MEASURE_NAME = "MarkovSmoothedKMeansStates" + """K-means emissions refined by a Markov transition prior.""" + + def __init__(self, **params): + self._n_init = 20 + self._train_sample_limit = 5000 + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "n_states", + "assignment_temperature", + "transition_smoothing", + "normalization", + "num_subj", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {name: params.get(name, None) for name in self.params_name_lst} + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = True + if self.params["n_states"] is None: + self.params["n_states"] = 5 + if self.params["assignment_temperature"] is None: + self.params["assignment_temperature"] = 1.0 + if self.params["transition_smoothing"] is None: + self.params["transition_smoothing"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _chunks(self, time_series): + return [ + time_series.get_subj_ts(subjs_id=subj).data.T.copy() + for subj in time_series.subj_id_lst + ] + + def _fit_matrix(self, chunks): + each = max(1, int(self._train_sample_limit) // max(len(chunks), 1)) + sampled = [] + for chunk in chunks: + if chunk.shape[0] <= each: + sampled.append(chunk) + else: + idx = np.linspace(0, chunk.shape[0] - 1, each, dtype=int) + sampled.append(chunk[idx, :]) + return np.concatenate(sampled, axis=0) + + def _emission(self, features): + distances = np.sum( + (features[:, None, :] - self.centers_[None, :, :]) ** 2, axis=2 + ) + logits = -distances / max(float(self.params["assignment_temperature"]), 1e-6) + return logits, _softmax_logits(logits) + + def estimate_FCS(self, time_series): + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4FCS(time_series) + tic = time.time() + + chunks = self._chunks(time_series) + fit_matrix = self._fit_matrix(chunks) + n_states = min(int(self.params["n_states"]), fit_matrix.shape[0]) + self.params["n_states"] = n_states + + self.kmeans_ = KMeans( + n_clusters=n_states, + n_init=self._n_init, + random_state=None, + ).fit(fit_matrix) + self.centers_ = self.kmeans_.cluster_centers_.astype(float) + + base_labels = [ + np.argmin( + np.sum((chunk[:, None, :] - self.centers_[None, :, :]) ** 2, axis=2), + axis=1, + ).astype(int) + for chunk in chunks + ] + trans = np.full((n_states, n_states), float(self.params["transition_smoothing"])) + start = np.full((n_states,), float(self.params["transition_smoothing"])) + for labels in base_labels: + start[labels[0]] += 1.0 + for a, b in zip(labels[:-1], labels[1:]): + trans[a, b] += 1.0 + self.TPM = trans / np.maximum(np.sum(trans, axis=1, keepdims=True), 1e-12) + self.startprob_ = start / np.maximum(np.sum(start), 1e-12) + + labels_chunks = [] + for chunk in chunks: + logits, _ = self._emission(chunk) + labels_chunks.append(_viterbi(logits, self.TPM, self.startprob_)) + + self.Z = np.concatenate(labels_chunks, axis=0) + self.FCS_ = np.zeros( + (n_states, chunks[0].shape[1], chunks[0].shape[1]), dtype=float + ) + for state in range(n_states): + samples = [ + chunk[labels == state, :] + for chunk, labels in zip(chunks, labels_chunks) + if np.any(labels == state) + ] + self.FCS_[state, :, :] = ( + _corr(np.concatenate(samples, axis=0)) + if samples + else np.eye(chunks[0].shape[1]) + ) + + self.set_mean_activity(time_series) + self.set_FCS_fit_time(time.time() - tic) + return self + + def estimate_dFC(self, time_series): + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + features = time_series.data.T.copy() + logits, _ = self._emission(features) + # Use forward-backward for soft state posteriors (compositional data) + proba = _forward_backward(logits, self.TPM, self.startprob_) + labels = np.argmax(proba, axis=1).astype(int) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC( + FCSs=self.FCS_, + FCS_idx=labels, + FCS_proba=proba, + TS_info=time_series.info_dict, + TR_array=np.arange(features.shape[0], dtype=int), + ) + return dFC diff --git a/pydfc/dfc_methods/minibatch_kmeans_states.py b/pydfc/dfc_methods/minibatch_kmeans_states.py new file mode 100644 index 0000000..abced6a --- /dev/null +++ b/pydfc/dfc_methods/minibatch_kmeans_states.py @@ -0,0 +1,167 @@ +"""Mini-batch k-means state-based dFC.""" + +import time + +import numpy as np +from sklearn.cluster import MiniBatchKMeans + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +def _corr(samples): + if samples.shape[0] < 2: + return np.eye(samples.shape[1], dtype=float) + cov = np.cov(samples, rowvar=False) + std = np.sqrt(np.maximum(np.diag(cov), 1e-12)) + den = np.outer(std, std) + corr = np.divide(cov, den, out=np.zeros_like(cov), where=den > 0) + corr[np.diag_indices_from(corr)] = 1.0 + return 0.5 * (corr + corr.T) + + +def _softmax_dist(features, centers, temperature): + distances = np.sum((features[:, None, :] - centers[None, :, :]) ** 2, axis=2) + logits = -distances / max(float(temperature), 1e-6) + logits = logits - np.max(logits, axis=1, keepdims=True) + probs = np.exp(logits) + probs = probs / np.maximum(np.sum(probs, axis=1, keepdims=True), 1e-12) + return np.argmin(distances, axis=1).astype(int), probs + + +class MINIBATCH_KMEANS_STATES(BaseDFCMethod): + MEASURE_NAME = "MiniBatchKMeansStates" + """Streaming prototype states learned with mini-batch k-means.""" + + def __init__(self, **params): + self._n_init = 20 + self._batch_size = 256 + self._max_iter = 300 + self._train_sample_limit = 5000 + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "n_states", + "assignment_temperature", + "normalization", + "num_subj", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {name: params.get(name, None) for name in self.params_name_lst} + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = True + if self.params["n_states"] is None: + self.params["n_states"] = 5 + if self.params["assignment_temperature"] is None: + self.params["assignment_temperature"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _subject_chunks(self, time_series): + return [ + time_series.get_subj_ts(subjs_id=subj).data.T.copy() + for subj in time_series.subj_id_lst + ] + + def _fit_matrix(self, chunks): + each = max(1, int(self._train_sample_limit) // max(len(chunks), 1)) + sampled = [] + for chunk in chunks: + if chunk.shape[0] <= each: + sampled.append(chunk) + else: + idx = np.linspace(0, chunk.shape[0] - 1, each, dtype=int) + sampled.append(chunk[idx, :]) + return np.concatenate(sampled, axis=0) + + def estimate_FCS(self, time_series): + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4FCS(time_series) + tic = time.time() + + chunks = self._subject_chunks(time_series) + fit_matrix = self._fit_matrix(chunks) + n_states = min(int(self.params["n_states"]), fit_matrix.shape[0]) + self.params["n_states"] = n_states + + self.kmeans_ = MiniBatchKMeans( + n_clusters=n_states, + n_init=self._n_init, + random_state=None, + batch_size=self._batch_size, + max_iter=self._max_iter, + ).fit(fit_matrix) + self.centers_ = self.kmeans_.cluster_centers_.astype(float) + + labels_chunks, _ = zip( + *[ + _softmax_dist(chunk, self.centers_, self.params["assignment_temperature"]) + for chunk in chunks + ] + ) + self.Z = np.concatenate(labels_chunks, axis=0) + self.FCS_ = np.zeros( + (n_states, chunks[0].shape[1], chunks[0].shape[1]), dtype=float + ) + for state in range(n_states): + samples = [ + chunk[labels == state, :] + for chunk, labels in zip(chunks, labels_chunks) + if np.any(labels == state) + ] + self.FCS_[state, :, :] = ( + _corr(np.concatenate(samples, axis=0)) + if samples + else np.eye(chunks[0].shape[1]) + ) + + counts = np.full((n_states, n_states), 1.0, dtype=float) + for labels in labels_chunks: + for a, b in zip(labels[:-1], labels[1:]): + counts[a, b] += 1.0 + self.TPM = counts / np.maximum(np.sum(counts, axis=1, keepdims=True), 1e-12) + + self.set_mean_activity(time_series) + self.set_FCS_fit_time(time.time() - tic) + return self + + def estimate_dFC(self, time_series): + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + + features = time_series.data.T.copy() + labels, probs = _softmax_dist( + features, self.centers_, self.params["assignment_temperature"] + ) + + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC( + FCSs=self.FCS_, + FCS_idx=labels, + FCS_proba=probs, + TS_info=time_series.info_dict, + TR_array=np.arange(features.shape[0], dtype=int), + ) + return dFC diff --git a/pydfc/dfc_methods/multiscale_dcc_kalman_hybrid.py b/pydfc/dfc_methods/multiscale_dcc_kalman_hybrid.py new file mode 100644 index 0000000..232ffde --- /dev/null +++ b/pydfc/dfc_methods/multiscale_dcc_kalman_hybrid.py @@ -0,0 +1,286 @@ +""" +MultiscaleDccKalman_hybrid. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class MULTISCALE_DCC_KALMAN_HYBRID(BaseDFCMethod): + """State-free hybrid of MultiscaleWindow, DCCConnectivity, KalmanCovariance.""" + + MEASURE_NAME = "MultiscaleDccKalman_hybrid" + COMPONENTS = ['MultiscaleWindow', 'DCCConnectivity', 'KalmanCovariance'] + HYBRID_RULE = "multiscale_residual" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/multiscale_window.py b/pydfc/dfc_methods/multiscale_window.py new file mode 100644 index 0000000..27a7b20 --- /dev/null +++ b/pydfc/dfc_methods/multiscale_window.py @@ -0,0 +1,105 @@ +""" +Multiscale window dFC method. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class MULTISCALE_WINDOW(BaseDFCMethod): + MEASURE_NAME = "MultiscaleWindow" + """Average correlations across multiple recent temporal scales.""" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "windows", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["windows"] is None: + self.params["windows"] = [15, 30, 60] + + @property + def measure_name(self): + return self.params["measure_name"] + + def _samples(self, value, Fs, minimum=1): + return max(int(round(value * Fs)), minimum) + + def _corr_from_cov(self, covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 0, + ) + corr[np.diag_indices_from(corr)] = 1 + corr[np.isnan(corr)] = 0 + return corr + + def _corr(self, samples): + corr = np.corrcoef(samples) + corr[np.isnan(corr)] = 0 + corr[np.diag_indices_from(corr)] = 1 + return corr + + def dFC(self, time_series, Fs): + windows = [ + self._samples(window, Fs, minimum=2) for window in self.params["windows"] + ] + min_periods = min(windows) + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [] + weights = [] + for window in windows: + start = max(0, tr - window + 1) + if tr - start + 1 >= 2: + matrices.append(self._corr(time_series[:, start : tr + 1])) + weights.append(np.sqrt(tr - start + 1)) + FCSs.append(np.average(np.array(matrices), axis=0, weights=weights)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/mutual_compression.py b/pydfc/dfc_methods/mutual_compression.py new file mode 100644 index 0000000..4be3993 --- /dev/null +++ b/pydfc/dfc_methods/mutual_compression.py @@ -0,0 +1,148 @@ +""" +Mutual Compression FC (MCFC) — novel dFC method. + +Two signals share information if their joint sequence is more compressible +than their individual sequences. This method quantifies pairwise dynamic +functional coupling as the "compression savings" when encoding region i +and region j together rather than separately, using the Lempel-Ziv 1976 +complexity of binarized BOLD signals within sliding windows. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class MUTUAL_COMPRESSION(BaseDFCMethod): + """Windowed Lempel-Ziv compression savings as a dFC measure. + + For a sliding window, each region's BOLD signal is binarized at its + within-window median. The Lempel-Ziv 1976 complexity C(s) (number of + novel substrings encountered on one pass through the sequence) is then + computed for each region individually and for each pair's concatenated + sequence. The mutual compression index is: + + MCI[i,j] = (C(s_i) + C(s_j) − C(s_i ‖ s_j)) / max(C(s_i), C(s_j)) + + Positive values indicate that the joint sequence is more compressible + than its parts — regions share repeating co-activation patterns. + The measure is bounded in [0, 1], symmetric, and free of distributional + assumptions. + """ + + MEASURE_NAME = "MutualCompressionFC" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for p in self.params_name_lst: + self.params[p] = params.get(p, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + @staticmethod + def _lz76(seq): + """Lempel-Ziv 76 complexity (number of novel substrings, not normalised).""" + n = len(seq) + if n == 0: + return 1 + i, l, k, c, kmax = 0, 1, 1, 1, 1 + while l + k <= n: + if seq[i + k - 1] == seq[l + k - 1]: + k += 1 + else: + kmax = max(kmax, k) + i += 1 + if i == l: + c += 1 + l += kmax + i = 0 + k = 1 + kmax = 1 + else: + k = 1 + return c + (1 if k > 1 else 0) + + def _mci_matrix(self, data): + """[R, R] mutual compression index for [R, W] data.""" + n_regions, W = data.shape + # Binarise at median + binary = (data >= np.median(data, axis=1, keepdims=True)).astype(np.int8) + + lz = np.array([self._lz76(binary[i]) for i in range(n_regions)]) + + C = np.zeros((n_regions, n_regions)) + np.fill_diagonal(C, 1.0) + for i in range(n_regions): + for j in range(i + 1, n_regions): + joint = np.concatenate([binary[i], binary[j]]) + lz_ij = self._lz76(joint) + norm = max(lz[i], lz[j], 1) + mci = (lz[i] + lz[j] - lz_ij) / norm + mci = float(np.clip(mci, 0.0, 1.0)) + C[i, j] = mci + C[j, i] = mci + + return C + + def dFC(self, time_series, Fs): + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + _, T = time_series.shape + + FCSs, TR_array = [], [] + for l in range(0, T - W_samples + 1, step): + seg = time_series[:, l : l + W_samples] + FCSs.append(self._mci_matrix(seg)) + TR_array.append(int(l + W_samples // 2)) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/nmf_states.py b/pydfc/dfc_methods/nmf_states.py new file mode 100644 index 0000000..1bcd23e --- /dev/null +++ b/pydfc/dfc_methods/nmf_states.py @@ -0,0 +1,193 @@ +""" +Non-negative Matrix Factorization States (NMF-States) dFC method. + +Reference: Yousefi et al. (2021). Quasi-periodic patterns of intrinsic brain +activity in individuals and their relationship to global signal. +NeuroImage, 225, 117479. doi:10.1016/j.neuroimage.2020.117479 + +Also: Chai et al. (2017). Evolution of brain network dynamics in +neurodevelopment. Network Neuroscience, 1(1), 14-30. +doi:10.1162/NETN_a_00006 +""" + +import time + +import numpy as np +from sklearn.decomposition import NMF + +from ..dfc import DFC +from ..dfc_utils import SW_downsample +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class NMF_STATES(BaseDFCMethod): + """FC states discovered by Non-negative Matrix Factorization of windowed FC. + + Group-level sliding-window FC matrices are vectorised (upper triangle), + globally shifted to be non-negative, and stacked into a data matrix V. + NMF decomposes V ≈ W H where H contains the latent FC patterns (states) + and W their temporal activations. At estimation time each subject's + window is projected onto the learned components and assigned to the state + with the highest activation. NMF enforces non-negativity, yielding + additive, parts-based FC components that do not cancel each other. + """ + + MEASURE_NAME = "NMFStates" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.mean_act = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.nmf_ = None + self._v_min = 0.0 + + self.params_name_lst = [ + "measure_name", + "is_state_based", + "n_states", + "W", + "n_overlap", + "normalization", + "num_subj", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = True + + if self.params["n_states"] is None: + self.params["n_states"] = 5 + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _windowed_fc_vecs(self, data, Fs): + """Vectorised upper-triangle FC for all windows of [R, T] data.""" + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + n_regions, T = data.shape + upper_idx = np.triu_indices(n_regions, k=1) + vecs, trs = [], [] + for l in range(0, T - W_samples + 1, step): + seg = data[:, l : l + W_samples] + C = np.corrcoef(seg) + C[np.isnan(C)] = 0.0 + vecs.append(C[upper_idx]) + trs.append(int(l + W_samples // 2)) + return np.array(vecs), np.array(trs) + + @staticmethod + def _vec_to_full(vec, n_regions): + upper_idx = np.triu_indices(n_regions, k=1) + C = np.zeros((n_regions, n_regions)) + C[upper_idx] = vec + C += C.T + np.fill_diagonal(C, 1.0) + return C + + def estimate_FCS(self, time_series): + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + time_series = self.manipulate_time_series4FCS(time_series) + Fs = time_series.Fs + n_regions = time_series.n_regions + + tic = time.time() + + all_vecs = [] + for subj_id in time_series.subj_id_lst: + subj_data = time_series.get_subj_ts(subjs_id=subj_id).data + vecs, _ = self._windowed_fc_vecs(subj_data, Fs) + all_vecs.append(vecs) + V = np.vstack(all_vecs) # [total_windows, n_pairs] + + # Shift to non-negative for NMF + self._v_min = float(V.min()) + V_nn = V - self._v_min + + self.nmf_ = NMF( + n_components=int(self.params["n_states"]), + max_iter=500, + random_state=0, + ) + W_coef = self.nmf_.fit_transform(V_nn) # [windows, n_states] + + # Reconstruct state FC matrices (shift back) + H = self.nmf_.components_ # [n_states, n_pairs] + H_shifted = H + self._v_min + self.FCS_ = np.array([self._vec_to_full(h, n_regions) for h in H_shifted]) + + # Group-level Z for set_mean_activity + self.Z = W_coef.argmax(axis=1) + self._set_mean_activity_nmf(time_series) + self.set_FCS_fit_time(time.time() - tic) + return self + + def _set_mean_activity_nmf(self, time_series): + """Mean window-averaged BOLD per NMF state.""" + TS_data = None + for subject in time_series.subj_id_lst: + subj_ts = time_series.get_subj_ts(subjs_id=subject) + win_data = SW_downsample( + data=subj_ts.data.T, + Fs=time_series.Fs, + W=self.params["W"], + n_overlap=self.params["n_overlap"], + tapered_window=False, + ).T # [n_regions, n_windows] + TS_data = ( + win_data + if TS_data is None + else np.concatenate((TS_data, win_data), axis=1) + ) + + mean_act = [] + for i in np.unique(self.Z): + ids = np.array([int(s == i) for s in self.Z]) + mean_act.append(np.average(TS_data, weights=ids, axis=1)) + self.mean_act = np.array(mean_act) + + def estimate_dFC(self, time_series): + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + assert len(time_series.subj_id_lst) == 1, "one subject per call" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + + vecs, TR_array = self._windowed_fc_vecs(time_series.data, time_series.Fs) + V_nn = vecs - self._v_min + W_coef = self.nmf_.transform(np.clip(V_nn, 0, None)) # [n_win, n_states] + + Z = W_coef.argmax(axis=1) + row_sums = W_coef.sum(axis=1, keepdims=True) + Z_proba = W_coef / np.where(row_sums > 0, row_sums, 1.0) + + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC( + FCSs=self.FCS_, + FCS_idx=Z, + FCS_proba=Z_proba, + TS_info=time_series.info_dict, + TR_array=TR_array, + ) + return dFC diff --git a/pydfc/dfc_methods/oja_subspace_connectivity.py b/pydfc/dfc_methods/oja_subspace_connectivity.py new file mode 100644 index 0000000..dee0966 --- /dev/null +++ b/pydfc/dfc_methods/oja_subspace_connectivity.py @@ -0,0 +1,125 @@ +""" +Oja subspace dFC method. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class OJA_SUBSPACE_CONNECTIVITY(BaseDFCMethod): + MEASURE_NAME = "OjaSubspaceConnectivity" + """Online low-rank connectivity from Oja-style latent subspace learning.""" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "half_life", + "min_periods", + "n_components", + "learning_rate", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + "alpha", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["min_periods"] is None: + self.params["min_periods"] = 10 + if self.params["n_components"] is None: + self.params["n_components"] = 5 + if self.params["learning_rate"] is None: + self.params["learning_rate"] = 0.03 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + if self.params["alpha"] is not None: + return float(self.params["alpha"]) + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + def _corr_from_cov(self, covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 0, + ) + corr[np.diag_indices_from(corr)] = 1 + corr[np.isnan(corr)] = 0 + return corr + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + n_regions = time_series.shape[0] + n_components = min(int(self.params["n_components"]), n_regions) + rng = np.random.default_rng() + basis = rng.normal(size=(n_regions, n_components)) + basis, _ = np.linalg.qr(basis) + mean = np.zeros(n_regions) + covariance = np.eye(n_regions) + alpha = self._alpha_from_half_life(Fs) + FCSs = [] + TR_array = [] + for tr in range(time_series.shape[1]): + sample = time_series[:, tr] + mean = (1.0 - alpha) * mean + alpha * sample + centered = sample - mean + covariance = (1.0 - alpha) * covariance + alpha * np.outer(centered, centered) + scores = basis.T @ centered + reconstruction = basis @ scores + basis = basis + self.params["learning_rate"] * np.outer( + centered - reconstruction, scores + ) + basis, _ = np.linalg.qr(basis) + projection = basis @ basis.T + low_rank_covariance = projection @ covariance @ projection.T + residual_variance = np.maximum(np.diag(covariance - low_rank_covariance), 0) + reconstructed_covariance = low_rank_covariance + np.diag(residual_variance) + if tr >= min_periods - 1: + FCSs.append(self._corr_from_cov(reconstructed_covariance)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/persistent_homology.py b/pydfc/dfc_methods/persistent_homology.py new file mode 100644 index 0000000..f245f35 --- /dev/null +++ b/pydfc/dfc_methods/persistent_homology.py @@ -0,0 +1,151 @@ +""" +Persistent Homology Connectivity (PHC) — from algebraic topology / TDA. + +Source domain: Carlsson (2009). Topology and data. Bull. Amer. Math. Soc., +46(2), 255-308. Applied to brain networks in Petri et al. (2014), Nature +Communications. + +Within each window a Vietoris-Rips filtration is built over the correlation +distance matrix d_ij = 1 − |C_ij|. The H0 bottleneck distance between +nodes i and j — the maximum edge weight on the minimum-spanning-tree path +from i to j — gives the "topological connectivity": a value that rewards +strong hub-mediated pathways, not just direct pairwise correlation. +""" + +import time + +import numpy as np +import scipy.sparse +from scipy.sparse.csgraph import minimum_spanning_tree + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class PERSISTENT_HOMOLOGY(BaseDFCMethod): + """Windowed MST bottleneck distance as a topologically filtered dFC measure. + + For each sliding window the pairwise correlation distance matrix + D[i,j] = 1 − |C_ij| is computed and its minimum spanning tree (MST) + is extracted. The H0 persistent homology bottleneck distance between + nodes i and j equals the maximum edge weight on the unique MST path + connecting them. Converting to similarity: + + FC[i,j] = 1 − bottleneck(i, j) / max(D) + + This differs from raw correlation in two ways: + • Indirect hub-mediated paths elevate FC between nodes that are + weakly directly coupled but strongly hub-connected. + • Spurious weak edges (noise) are penalised because they inflate + the MST path weight. + """ + + MEASURE_NAME = "PersistentHomologyFC" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for p in self.params_name_lst: + self.params[p] = params.get(p, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + @staticmethod + def _mst_bottleneck(D): + """Bottleneck distance matrix from MST of distance matrix D.""" + n = D.shape[0] + # Build MST (scipy returns upper-triangle sparse) + mst_sparse = minimum_spanning_tree(scipy.sparse.csr_matrix(D)) + mst = mst_sparse.toarray() + mst = mst + mst.T # symmetrise + + # Initialise bottleneck matrix from MST edges + B = np.full((n, n), np.inf) + np.fill_diagonal(B, 0.0) + mask = mst > 0 + B[mask] = mst[mask] + + # Vectorised Floyd-Warshall for min-bottleneck (max-edge) paths + for k in range(n): + # Candidate via k: max(B[i,k], B[k,j]) + via_k = np.maximum(B[:, k : k + 1], B[k : k + 1, :]) + B = np.minimum(B, via_k) + + # Remaining inf means no path (shouldn't happen for complete graph) + max_d = ( + B[np.isfinite(B) & (B > 0)].max() if np.any(np.isfinite(B) & (B > 0)) else 1.0 + ) + return B, max_d + + def dFC(self, time_series, Fs): + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + n_regions, T = time_series.shape + + FCSs, TR_array = [], [] + for l in range(0, T - W_samples + 1, step): + seg = time_series[:, l : l + W_samples] + C = np.corrcoef(seg) + C[np.isnan(C)] = 0.0 + np.fill_diagonal(C, 1.0) + + D = 1.0 - np.abs(C) + np.fill_diagonal(D, 0.0) + + B, max_d = self._mst_bottleneck(D) + + FC = 1.0 - B / max(max_d, 1e-12) + FC[~np.isfinite(FC)] = 0.0 + np.fill_diagonal(FC, 1.0) + FC = np.clip(FC, 0.0, 1.0) + + FCSs.append(FC) + TR_array.append(int(l + W_samples // 2)) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/phase_amplitude_cross.py b/pydfc/dfc_methods/phase_amplitude_cross.py new file mode 100644 index 0000000..15fa046 --- /dev/null +++ b/pydfc/dfc_methods/phase_amplitude_cross.py @@ -0,0 +1,128 @@ +""" +Phase-Amplitude Cross-Region FC (PAFC) — novel dFC method. + +Standard AEC correlates amplitude envelopes; standard phase methods +correlate phases. This method crosses them: it asks whether the +instantaneous phase of region i is correlated with the amplitude +envelope of region j — a spatial generalisation of cross-frequency +phase-amplitude coupling, applied as a between-region dFC measure. +""" + +import time + +import numpy as np +from scipy.signal import hilbert + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class PHASE_AMPLITUDE_CROSS(BaseDFCMethod): + """Windowed cross-coupling between Hilbert phase and amplitude envelope. + + Within each sliding window: + • phase_i(t) = cos(∠ hilbert(x_i(t))) — instantaneous phase (as cosine) + • amp_j(t) = |hilbert(x_j(t))| — instantaneous amplitude + + A raw asymmetric coupling matrix A[i,j] = corr(phase_i, amp_j) is + computed, then symmetrised as FC[i,j] = (|A[i,j]| + |A[j,i]|) / 2. + + A large value for pair (i, j) indicates that the oscillatory phase + of region i consistently predicts the energy level of region j (and/or + vice versa), orthogonal to both amplitude-only and phase-only coupling. + """ + + MEASURE_NAME = "PhaseAmplitudeCrossFC" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for p in self.params_name_lst: + self.params[p] = params.get(p, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + @staticmethod + def _pac_matrix(data): + """[R, R] symmetrised phase-amplitude cross-coupling for [R, W] data.""" + n_regions, W = data.shape + analytic = hilbert(data, axis=1) # [R, W] + cos_phase = np.cos(np.angle(analytic)) # [R, W] + amplitude = np.abs(analytic) # [R, W] + + # Standardise each signal to zero mean, unit std + def _standardise(x): + mu = x.mean(axis=1, keepdims=True) + sd = x.std(axis=1, keepdims=True) + sd = np.where(sd > 1e-12, sd, 1.0) + return (x - mu) / sd + + cp = _standardise(cos_phase) # [R, W] + am = _standardise(amplitude) # [R, W] + + # A[i,j] = corr(phase_i, amp_j) = (cp_i · am_j) / W + A = (cp @ am.T) / max(W - 1, 1) # [R, R] + + # Symmetrise absolute values + C = (np.abs(A) + np.abs(A.T)) / 2.0 + np.fill_diagonal(C, 1.0) + return np.clip(C, 0.0, 1.0) + + def dFC(self, time_series, Fs): + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + _, T = time_series.shape + + FCSs, TR_array = [], [] + for l in range(0, T - W_samples + 1, step): + seg = time_series[:, l : l + W_samples] + FCSs.append(self._pac_matrix(seg)) + TR_array.append(int(l + W_samples // 2)) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/phase_lag_index_window.py b/pydfc/dfc_methods/phase_lag_index_window.py new file mode 100644 index 0000000..f3b2814 --- /dev/null +++ b/pydfc/dfc_methods/phase_lag_index_window.py @@ -0,0 +1,130 @@ +""" +Windowed Phase Lag Index (PLI) dFC method. + +Reference: Stam et al. (2007). Phase lag index: Assessment of functional +connectivity from multi channel EEG and MEG with diminished bias from common +sources. Hum Brain Map, 28(11), 1178-1193. doi:10.1002/hbm.20346 + +Application to sliding-window fMRI dFC: Aydore et al. (2013). A note on the +phase locking value and its properties. NeuroImage, 74, 231-244. +doi:10.1016/j.neuroimage.2013.02.008 +""" + +import time + +import numpy as np +from scipy.signal import butter, filtfilt, hilbert + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class PHASE_LAG_INDEX_WINDOW(BaseDFCMethod): + """Sliding-window Phase Lag Index (PLI) measuring consistent phase asymmetry. + + PLV (phase locking value) inflates connectivity estimates when two channels + share a common zero-lag source (e.g. volume conduction in EEG, global + signal in fMRI). PLI instead measures whether the *sign* of the imaginary + part of the cross-spectrum is consistently positive or negative: + + PLI_ij = |E_t[sign(sin(φ_j(t) − φ_i(t)))]| + + A value of 1 means the phase difference consistently falls on one side of + zero (strict leading/lagging); 0 means no consistent asymmetry. Pure + zero-lag coupling contributes nothing to PLI, making it robust to common + sources and global signal fluctuations. + """ + + MEASURE_NAME = "PhaseLagIndexWindow" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "f_low", + "f_high", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["f_low"] is None: + self.params["f_low"] = 0.01 + if self.params["f_high"] is None: + self.params["f_high"] = 0.1 + + @property + def measure_name(self): + return self.params["measure_name"] + + def dFC(self, time_series, Fs): + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + n_regions, T = time_series.shape + + f_low = float(self.params["f_low"]) + f_high = float(self.params["f_high"]) + nyq = Fs / 2.0 + + if f_low > 0 and f_high < nyq: + b, a = butter(4, [f_low / nyq, f_high / nyq], btype="band") + filtered = filtfilt(b, a, time_series, axis=1) + else: + filtered = time_series.copy() + + phases = np.angle(hilbert(filtered, axis=1)) # [R, T] + + FCSs, TR_array = [], [] + for l in range(0, T - W_samples + 1, step): + phi = phases[:, l : l + W_samples] # [R, W] + + # sin(φ_j(t) - φ_i(t)) for every pair via broadcasting + # [R, W] - [R, 1, W] broadcasting → [R, R, W] of differences + delta_phi = phi[np.newaxis, :, :] - phi[:, np.newaxis, :] # [R, R, W] + signed = np.sign(np.sin(delta_phi)) # [R, R, W] + C = np.abs(signed.mean(axis=2)) # PLI: [R, R] + np.fill_diagonal(C, 1.0) + FCSs.append(C) + TR_array.append(int(l + W_samples // 2)) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/phase_locking_window.py b/pydfc/dfc_methods/phase_locking_window.py new file mode 100644 index 0000000..f049763 --- /dev/null +++ b/pydfc/dfc_methods/phase_locking_window.py @@ -0,0 +1,82 @@ +""" +Phase-locking window dFC method. +""" + +import time + +import numpy as np +from scipy.signal import hilbert + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class PHASE_LOCKING_WINDOW(BaseDFCMethod): + MEASURE_NAME = "PhaseLockingWindow" + """Windowed phase-locking value from Hilbert analytic phases.""" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["W"] is None: + self.params["W"] = 30 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _samples(self, value, Fs, minimum=1): + return max(int(round(value * Fs)), minimum) + + def dFC(self, time_series, Fs): + window = self._samples(self.params["W"], Fs, minimum=2) + phase = np.angle(hilbert(time_series, axis=1)) + complex_phase = np.exp(1j * phase) + FCSs = [] + TR_array = [] + for tr in range(window - 1, time_series.shape[1]): + segment = complex_phase[:, tr - window + 1 : tr + 1] + plv = np.abs(segment @ np.conjugate(segment.T)) / segment.shape[1] + plv[np.diag_indices_from(plv)] = 1 + FCSs.append(plv) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/point_process_connectivity.py b/pydfc/dfc_methods/point_process_connectivity.py new file mode 100644 index 0000000..e3a5004 --- /dev/null +++ b/pydfc/dfc_methods/point_process_connectivity.py @@ -0,0 +1,114 @@ +""" +Point Process Connectivity (PPFC) dFC method. + +Reference: Tagliazucchi et al. (2012). Criticality in Large-Scale Brain fMRI +Dynamics Unveiled by a Novel Point Process Analysis. Front Physiol, 3, 15. +doi:10.3389/fphys.2012.00015 + +Also: Liu & Duyn (2013). Time-varying functional network information extracted +from brief instances of spontaneous brain activity. PNAS, 110(11), 4392-4397. +doi:10.1073/pnas.1216856110 +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class POINT_PROCESS_CONNECTIVITY(BaseDFCMethod): + """Event-driven FC from discrete BOLD threshold-crossing frames. + + At each TR where at least one region's z-scored BOLD amplitude exceeds a + positive threshold, an "event" is detected. The instantaneous FC at that + TR is the standardised co-activation matrix: z(t) zᵀ(t) with diagonal + forced to 1, where z_i = x_i / σ_i (globally standardised). TRs without + events contribute no FC estimate, yielding a sparse output. This approach + preserves the high-amplitude, high-SNR moments of the BOLD signal and + discards near-baseline TRs that contribute noise to time-averaged FC. + """ + + MEASURE_NAME = "PointProcessConnectivity" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "z_threshold", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["z_threshold"] is None: + self.params["z_threshold"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def dFC(self, time_series, Fs): + n_regions, T = time_series.shape + threshold = float(self.params["z_threshold"]) + + # Globally standardise each region (zero mean, unit variance over time) + sigma = time_series.std(axis=1, keepdims=True) + sigma = np.where(sigma > 1e-10, sigma, 1e-10) + z = (time_series - time_series.mean(axis=1, keepdims=True)) / sigma + + # Detect events: TR where max |z| across regions >= threshold + event_mask = np.max(np.abs(z), axis=0) >= threshold + event_trs = np.where(event_mask)[0] + + if len(event_trs) == 0: + # Fallback: return a single global mean FC centred at the midpoint + C = np.corrcoef(time_series) + C[np.isnan(C)] = 0.0 + np.fill_diagonal(C, 1.0) + return C[np.newaxis], np.array([T // 2]) + + FCSs, TR_array = [], [] + for t in event_trs: + frame = z[:, t] + C = np.outer(frame, frame) + C = np.clip(C, -1.0, 1.0) + np.fill_diagonal(C, 1.0) + FCSs.append(C) + TR_array.append(int(t)) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/pooled_kmeans_states.py b/pydfc/dfc_methods/pooled_kmeans_states.py new file mode 100644 index 0000000..b827f9f --- /dev/null +++ b/pydfc/dfc_methods/pooled_kmeans_states.py @@ -0,0 +1,160 @@ +"""Pooled k-means state-based dFC.""" + +import time + +import numpy as np +from sklearn.cluster import KMeans + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +def _corr(samples): + if samples.shape[0] < 2: + return np.eye(samples.shape[1], dtype=float) + cov = np.cov(samples, rowvar=False) + std = np.sqrt(np.maximum(np.diag(cov), 1e-12)) + den = np.outer(std, std) + corr = np.divide(cov, den, out=np.zeros_like(cov), where=den > 0) + corr[np.diag_indices_from(corr)] = 1.0 + return 0.5 * (corr + corr.T) + + +def _softmax_dist(features, centers, temperature): + distances = np.sum((features[:, None, :] - centers[None, :, :]) ** 2, axis=2) + logits = -distances / max(float(temperature), 1e-6) + logits = logits - np.max(logits, axis=1, keepdims=True) + probs = np.exp(logits) + probs = probs / np.maximum(np.sum(probs, axis=1, keepdims=True), 1e-12) + return np.argmin(distances, axis=1).astype(int), probs + + +class POOLED_KMEANS_STATES(BaseDFCMethod): + MEASURE_NAME = "PooledKMeansStates" + """Discretizes recurring whole-brain activity prototypes with k-means.""" + + def __init__(self, **params): + self._n_init = 20 + self._train_sample_limit = 5000 + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "n_states", + "assignment_temperature", + "normalization", + "num_subj", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {name: params.get(name, None) for name in self.params_name_lst} + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = True + if self.params["n_states"] is None: + self.params["n_states"] = 5 + if self.params["assignment_temperature"] is None: + self.params["assignment_temperature"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _subject_chunks(self, time_series): + return [ + time_series.get_subj_ts(subjs_id=subj).data.T.copy() + for subj in time_series.subj_id_lst + ] + + def _fit_matrix(self, chunks): + if self._train_sample_limit is None: + return np.concatenate(chunks, axis=0) + each = max(1, int(self._train_sample_limit) // max(len(chunks), 1)) + sampled = [] + for chunk in chunks: + if chunk.shape[0] <= each: + sampled.append(chunk) + else: + idx = np.linspace(0, chunk.shape[0] - 1, each, dtype=int) + sampled.append(chunk[idx, :]) + return np.concatenate(sampled, axis=0) + + def estimate_FCS(self, time_series): + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4FCS(time_series) + tic = time.time() + + chunks = self._subject_chunks(time_series) + fit_matrix = self._fit_matrix(chunks) + n_states = min(int(self.params["n_states"]), fit_matrix.shape[0]) + self.params["n_states"] = n_states + + self.kmeans_ = KMeans( + n_clusters=n_states, + n_init=self._n_init, + random_state=None, + ).fit(fit_matrix) + self.centers_ = self.kmeans_.cluster_centers_.astype(float) + + labels_chunks = [self.kmeans_.predict(chunk).astype(int) for chunk in chunks] + self.Z = np.concatenate(labels_chunks, axis=0) + self.FCS_ = np.zeros( + (n_states, chunks[0].shape[1], chunks[0].shape[1]), dtype=float + ) + for state in range(n_states): + samples = [ + chunk[labels == state, :] + for chunk, labels in zip(chunks, labels_chunks) + if np.any(labels == state) + ] + self.FCS_[state, :, :] = ( + _corr(np.concatenate(samples, axis=0)) + if samples + else np.eye(chunks[0].shape[1]) + ) + + counts = np.full((n_states, n_states), 1.0, dtype=float) + for labels in labels_chunks: + for a, b in zip(labels[:-1], labels[1:]): + counts[a, b] += 1.0 + self.TPM = counts / np.maximum(np.sum(counts, axis=1, keepdims=True), 1e-12) + + self.set_mean_activity(time_series) + self.set_FCS_fit_time(time.time() - tic) + return self + + def estimate_dFC(self, time_series): + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + + features = time_series.data.T.copy() + labels, probs = _softmax_dist( + features, self.centers_, self.params["assignment_temperature"] + ) + + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC( + FCSs=self.FCS_, + FCS_idx=labels, + FCS_proba=probs, + TS_info=time_series.info_dict, + TR_array=np.arange(features.shape[0], dtype=int), + ) + return dFC diff --git a/pydfc/dfc_methods/positive_negative_asymmetry.py b/pydfc/dfc_methods/positive_negative_asymmetry.py new file mode 100644 index 0000000..497063c --- /dev/null +++ b/pydfc/dfc_methods/positive_negative_asymmetry.py @@ -0,0 +1,141 @@ +""" +Positive-Negative Asymmetry FC (PNAFC) — novel dFC method. + +Standard Pearson correlation treats all BOLD deviations symmetrically. +This method asks: is the coupling between regions i and j the same when +region i is in an up-state (above its mean) as when it is in a down-state? +The dFC value captures the asymmetry between up-state and down-state +conditional connectivity — a measure of state-dependent coupling directionality. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class POSITIVE_NEGATIVE_ASYMMETRY(BaseDFCMethod): + """Windowed asymmetry of up-state vs. down-state conditional correlation. + + Within each sliding window, for each conditioning region i: + • FC_up[i,j] = Pearson corr(x_i, x_j) restricted to timepoints + where x_i > μ_i (region i above its window mean) + • FC_down[i,j] = Pearson corr(x_i, x_j) restricted to timepoints + where x_i ≤ μ_i (region i below its window mean) + + The asymmetry matrix A[i,j] = |FC_up[i,j] − FC_down[i,j]| is asymmetric + in i because the conditioning is on region i. It is symmetrised as: + + FC[i,j] = (A[i,j] + A[j,i]) / 2 + + A large value indicates that the coupling between i and j is qualitatively + different during the "up" and "down" BOLD phases of either region. + """ + + MEASURE_NAME = "PositiveNegativeAsymmetryFC" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for p in self.params_name_lst: + self.params[p] = params.get(p, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + @staticmethod + def _conditional_corr(xi, xj, mask): + """Pearson correlation of xi and xj over selected timepoints.""" + if mask.sum() < 3: + return 0.0 + a = xi[mask] + b = xj[mask] + a = a - a.mean() + b = b - b.mean() + sa = np.sqrt((a**2).sum()) + sb = np.sqrt((b**2).sum()) + if sa < 1e-12 or sb < 1e-12: + return 0.0 + return float(np.clip(np.dot(a, b) / (sa * sb), -1.0, 1.0)) + + def _pna_matrix(self, data): + """[R, R] positive-negative asymmetry matrix for [R, W] data.""" + n_regions, W = data.shape + mu = data.mean(axis=1) # [R] + + # Asymmetry conditioned on each row region + A = np.zeros((n_regions, n_regions)) + for i in range(n_regions): + up_mask = data[i] > mu[i] + dn_mask = ~up_mask + for j in range(n_regions): + if i == j: + continue + r_up = self._conditional_corr(data[i], data[j], up_mask) + r_dn = self._conditional_corr(data[i], data[j], dn_mask) + A[i, j] = abs(r_up - r_dn) + + # Symmetrise + C = (A + A.T) / 2.0 + np.fill_diagonal(C, 1.0) + return np.clip(C, 0.0, 1.0) + + def dFC(self, time_series, Fs): + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + _, T = time_series.shape + + FCSs, TR_array = [], [] + for l in range(0, T - W_samples + 1, step): + seg = time_series[:, l : l + W_samples] + FCSs.append(self._pna_matrix(seg)) + TR_array.append(int(l + W_samples // 2)) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/precision_shrinkage_window.py b/pydfc/dfc_methods/precision_shrinkage_window.py new file mode 100644 index 0000000..304456b --- /dev/null +++ b/pydfc/dfc_methods/precision_shrinkage_window.py @@ -0,0 +1,93 @@ +""" +Precision-shrinkage window dFC method. +""" + +import time + +import numpy as np +from sklearn.covariance import LedoitWolf + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class PRECISION_SHRINKAGE_WINDOW(BaseDFCMethod): + MEASURE_NAME = "PrecisionShrinkageWindow" + """Windowed partial correlations from Ledoit-Wolf covariance shrinkage.""" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["W"] is None: + self.params["W"] = 30 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _samples(self, value, Fs, minimum=1): + return max(int(round(value * Fs)), minimum) + + def _precision_corr(self, segment): + model = LedoitWolf().fit(segment.T) + precision = model.precision_ + diagonal = np.sqrt(np.diag(precision)) + scale = np.outer(diagonal, diagonal) + partial = np.divide( + -precision, + scale, + out=np.zeros_like(precision), + where=scale > 0, + ) + partial[np.diag_indices_from(partial)] = 1 + partial[np.isnan(partial)] = 0 + return partial + + def dFC(self, time_series, Fs): + window = self._samples(self.params["W"], Fs, minimum=3) + FCSs = [] + TR_array = [] + for tr in range(window - 1, time_series.shape[1]): + segment = time_series[:, tr - window + 1 : tr + 1] + FCSs.append(self._precision_corr(segment)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/quantum_multiscale_kalman_hybrid_ensemble.py b/pydfc/dfc_methods/quantum_multiscale_kalman_hybrid_ensemble.py new file mode 100644 index 0000000..f1f9fd8 --- /dev/null +++ b/pydfc/dfc_methods/quantum_multiscale_kalman_hybrid_ensemble.py @@ -0,0 +1,286 @@ +""" +QuantumMultiscaleKalman_hybrid_ensemble. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class QUANTUM_MULTISCALE_KALMAN_HYBRID_ENSEMBLE(BaseDFCMethod): + """State-free hybrid of QuantumMutualInformationFC, MultiscaleWindow, KalmanCovariance.""" + + MEASURE_NAME = "QuantumMultiscaleKalman_hybrid_ensemble" + COMPONENTS = ['QuantumMutualInformationFC', 'MultiscaleWindow', 'KalmanCovariance'] + HYBRID_RULE = "ensemble_mean" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/quantum_mutual_information.py b/pydfc/dfc_methods/quantum_mutual_information.py new file mode 100644 index 0000000..6694d7e --- /dev/null +++ b/pydfc/dfc_methods/quantum_mutual_information.py @@ -0,0 +1,158 @@ +""" +Quantum Mutual Information Connectivity (QMIC) — from quantum information theory. + +Source domain: Nielsen & Chuang (2000). Quantum Computation and Quantum +Information. Cambridge University Press. + +A normalised covariance matrix has trace 1 and non-negative eigenvalues, +making it formally identical to a quantum density matrix ρ. The quantum +mutual information I(i:j) = S(ρ_i) + S(ρ_j) − S(ρ_ij) — where S is the +von Neumann entropy −Tr(ρ log ρ) — measures how much the 2×2 joint state +of pair (i,j) differs from a product (independent) state. Unlike linear +correlation, QMIC is sensitive to off-diagonal structure regardless of sign, +captures higher-order Gaussian dependencies, and is bounded in [0, 1]. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class QUANTUM_MUTUAL_INFORMATION(BaseDFCMethod): + """Windowed von Neumann mutual information of normalised covariance pairs. + + Within each window, the sample covariance C is normalised to a density + matrix ρ = C / Tr(C). For each pair (i, j) the 2×2 marginal is: + + ρ_ij = [[ρ_ii, ρ_ij], [ρ_ji, ρ_jj]] / (ρ_ii + ρ_jj) + + The quantum mutual information is then: + + QMI(i,j) = S_product − S_joint + + where S_product = −p log p − q log q (p = ρ_ii_norm, q = ρ_jj_norm) + is the von Neumann entropy of the hypothetical product state, and + S_joint = −Σ λ_k log λ_k (eigenvalues of ρ_ij_norm) is the + actual joint entropy. The result is bounded in [0, log 2] and + normalised to [0, 1]; it equals zero iff the two regions are + uncorrelated within the window. + """ + + MEASURE_NAME = "QuantumMutualInformationFC" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for p in self.params_name_lst: + self.params[p] = params.get(p, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + @staticmethod + def _vn_entropy_2x2(mat): + """Von Neumann entropy −Σ λ log λ of a 2×2 density matrix.""" + eigvals = np.linalg.eigvalsh(mat) + eigvals = np.maximum(eigvals, 1e-12) + eigvals = eigvals / eigvals.sum() # ensure normalised + return -float(np.sum(eigvals * np.log(eigvals))) + + def _qmi_matrix(self, data): + """[R, R] QMI matrix for [R, W] window data.""" + n_regions, W = data.shape + C = np.cov(data) + tr_C = np.trace(C) + if tr_C < 1e-12: + return np.eye(n_regions) + rho = C / tr_C + + FC = np.zeros((n_regions, n_regions)) + np.fill_diagonal(FC, 1.0) + log2 = np.log(2.0) + + for i in range(n_regions): + for j in range(i + 1, n_regions): + # 2×2 marginal + rho_ij = np.array([[rho[i, i], rho[i, j]], [rho[j, i], rho[j, j]]]) + tr_ij = rho_ij[0, 0] + rho_ij[1, 1] + if tr_ij < 1e-12: + continue + rho_ij_norm = rho_ij / tr_ij + + p = rho_ij_norm[0, 0] # marginal probability for i + q = rho_ij_norm[1, 1] # marginal probability for j (= 1-p) + + # Product-state entropy (independence baseline) + p = np.clip(p, 1e-12, 1.0 - 1e-12) + q = np.clip(q, 1e-12, 1.0 - 1e-12) + s_product = -p * np.log(p) - q * np.log(q) + + # Joint von Neumann entropy + s_joint = self._vn_entropy_2x2(rho_ij_norm) + + qmi = max(0.0, s_product - s_joint) / log2 + FC[i, j] = np.clip(qmi, 0.0, 1.0) + FC[j, i] = FC[i, j] + + return FC + + def dFC(self, time_series, Fs): + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + _, T = time_series.shape + + FCSs, TR_array = [], [] + for l in range(0, T - W_samples + 1, step): + seg = time_series[:, l : l + W_samples] + FCSs.append(self._qmi_matrix(seg)) + TR_array.append(int(l + W_samples // 2)) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/quantum_random_exp_hybrid.py b/pydfc/dfc_methods/quantum_random_exp_hybrid.py new file mode 100644 index 0000000..1b86d86 --- /dev/null +++ b/pydfc/dfc_methods/quantum_random_exp_hybrid.py @@ -0,0 +1,286 @@ +""" +QuantumRandomExp_hybrid. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class QUANTUM_RANDOM_EXP_HYBRID(BaseDFCMethod): + """State-free hybrid of QuantumMutualInformationFC, RandomFourierDependence, ExponentialWindow.""" + + MEASURE_NAME = "QuantumRandomExp_hybrid" + COMPONENTS = ['QuantumMutualInformationFC', 'RandomFourierDependence', 'ExponentialWindow'] + HYBRID_RULE = "information_gate" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/random_fourier_dependence.py b/pydfc/dfc_methods/random_fourier_dependence.py new file mode 100644 index 0000000..282c9d1 --- /dev/null +++ b/pydfc/dfc_methods/random_fourier_dependence.py @@ -0,0 +1,105 @@ +""" +Random Fourier feature dFC method. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class RANDOM_FOURIER_DEPENDENCE(BaseDFCMethod): + MEASURE_NAME = "RandomFourierDependence" + """Nonlinear edge dependence from random Fourier features of node activity.""" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "half_life", + "min_periods", + "n_random_features", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + "alpha", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["min_periods"] is None: + self.params["min_periods"] = 10 + if self.params["n_random_features"] is None: + self.params["n_random_features"] = 32 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + if self.params["alpha"] is not None: + return float(self.params["alpha"]) + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + def _feature_map(self, samples): + rng = np.random.default_rng() + n_features = int(self.params["n_random_features"]) + omega = rng.normal(size=n_features) + phase = rng.uniform(0, 2 * np.pi, size=n_features) + return np.sqrt(2.0 / n_features) * np.cos(samples[:, :, None] * omega + phase) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + alpha = self._alpha_from_half_life(Fs) + features = self._feature_map(time_series) + n_regions = time_series.shape[0] + dependence = np.eye(n_regions) + FCSs = [] + TR_array = [] + for tr in range(time_series.shape[1]): + phi = features[:, tr, :] + phi = phi - np.mean(phi, axis=1, keepdims=True) + norm = np.linalg.norm(phi, axis=1, keepdims=True) + phi = np.divide(phi, norm, out=np.zeros_like(phi), where=norm > 0) + instant = phi @ phi.T + dependence = (1.0 - alpha) * dependence + alpha * instant + dependence[np.diag_indices_from(dependence)] = 1 + if tr >= min_periods - 1: + FCSs.append(dependence.copy()) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/random_fourier_kalman_hybrid.py b/pydfc/dfc_methods/random_fourier_kalman_hybrid.py new file mode 100644 index 0000000..962e4e7 --- /dev/null +++ b/pydfc/dfc_methods/random_fourier_kalman_hybrid.py @@ -0,0 +1,286 @@ +""" +RandomFourierKalman_hybrid. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class RANDOM_FOURIER_KALMAN_HYBRID(BaseDFCMethod): + """State-free hybrid of RandomFourierDependence, KalmanCovariance, ExponentialWindow.""" + + MEASURE_NAME = "RandomFourierKalman_hybrid" + COMPONENTS = ['RandomFourierDependence', 'KalmanCovariance', 'ExponentialWindow'] + HYBRID_RULE = "kernel_modulated" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/recurrence_kernel_dependence.py b/pydfc/dfc_methods/recurrence_kernel_dependence.py new file mode 100644 index 0000000..285cf59 --- /dev/null +++ b/pydfc/dfc_methods/recurrence_kernel_dependence.py @@ -0,0 +1,107 @@ +""" +Recurrence-kernel dFC method. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class RECURRENCE_KERNEL_DEPENDENCE(BaseDFCMethod): + MEASURE_NAME = "RecurrenceKernelDependence" + """State-dependent FC from samples whose whole-brain pattern recurs.""" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "kernel_width", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 10 + if self.params["kernel_width"] is None: + self.params["kernel_width"] = 1.5 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _corr_from_cov(self, covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 0, + ) + corr[np.diag_indices_from(corr)] = 1 + corr[np.isnan(corr)] = 0 + return corr + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0) + weights_sum = np.sum(weights) + if weights_sum <= 0: + weights = np.ones(samples.shape[1], dtype=float) + weights_sum = np.sum(weights) + weights = weights / weights_sum + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + history = time_series[:, : tr + 1] + state = history[:, -1:] + distances = np.mean((history - state) ** 2, axis=0) + scale = np.median(distances) + 1e-8 + weights = np.exp(-distances / (self.params["kernel_width"] * scale)) + FCSs.append(self._weighted_corr(history, weights)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/reservoir_changepoint_robust_hybrid_ensemble.py b/pydfc/dfc_methods/reservoir_changepoint_robust_hybrid_ensemble.py new file mode 100644 index 0000000..9b2c834 --- /dev/null +++ b/pydfc/dfc_methods/reservoir_changepoint_robust_hybrid_ensemble.py @@ -0,0 +1,286 @@ +""" +ReservoirChangepointRobust_hybrid_ensemble. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class RESERVOIR_CHANGEPOINT_ROBUST_HYBRID_ENSEMBLE(BaseDFCMethod): + """State-free hybrid of ReservoirEchoStateFC, ChangepointResetWindow, RobustSlidingWindow.""" + + MEASURE_NAME = "ReservoirChangepointRobust_hybrid_ensemble" + COMPONENTS = ['ReservoirEchoStateFC', 'ChangepointResetWindow', 'RobustSlidingWindow'] + HYBRID_RULE = "ensemble_mean" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/reservoir_echo_state.py b/pydfc/dfc_methods/reservoir_echo_state.py new file mode 100644 index 0000000..4543dff --- /dev/null +++ b/pydfc/dfc_methods/reservoir_echo_state.py @@ -0,0 +1,157 @@ +""" +Reservoir Echo State Connectivity (RESC) — from reservoir computing. + +Source domain: Jaeger (2001). The "echo state" approach to analysing and +training recurrent neural networks. GMD Technical Report 148. +Maass et al. (2002). Real-time computing without stable states. Neural +Computation, 14(11), 2531-2560. + +A fixed random recurrent network (the reservoir) provides a high-dimensional +nonlinear expansion with memory of the input. Driving the reservoir with +BOLD and reading out each region's contribution to the reservoir state +yields a nonlinearly filtered signal per region. Windowed correlation of +these readouts captures temporal coupling that linear correlation misses, +because the reservoir acts as a universal nonlinear temporal filter. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class RESERVOIR_ECHO_STATE(BaseDFCMethod): + """Windowed correlation of reservoir-filtered BOLD readouts. + + A fixed random recurrent network of N units is driven by the R-region + BOLD signal: + + h(t) = tanh(W_res h(t−1) + W_in x(t)) + + W_res (N×N) has spectral radius < 1 to satisfy the echo-state property; + W_in (N×R) is a random input weight matrix. The readout for region i is: + + y_i(t) = W_in[:, i] · h(t) — projection of reservoir state onto + region i's input subspace + + Within each sliding window the Pearson correlation of the readout + signals [y_1(t), …, y_R(t)] is the dFC estimate. Unlike linear + correlation on raw BOLD, the reservoir nonlinearly expands the signal + history before correlation, capturing higher-order temporal dependencies. + + The reservoir is fixed (random seed 42) for reproducibility. + """ + + MEASURE_NAME = "ReservoirEchoStateFC" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self._W_res = None + self._W_in = None + + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "reservoir_dim", + "spectral_radius", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for p in self.params_name_lst: + self.params[p] = params.get(p, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["reservoir_dim"] is None: + self.params["reservoir_dim"] = 100 + if self.params["spectral_radius"] is None: + self.params["spectral_radius"] = 0.9 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _build_reservoir(self, n_regions): + N = int(self.params["reservoir_dim"]) + rho = float(self.params["spectral_radius"]) + rng = np.random.RandomState(42) + W_res = rng.randn(N, N) * 0.1 + sr = np.max(np.abs(np.linalg.eigvals(W_res))) + if sr > 1e-10: + W_res = W_res * (rho / sr) + W_in = rng.randn(N, n_regions) * 0.1 + self._W_res = W_res + self._W_in = W_in + + def _reservoir_readout(self, time_series): + """Drive reservoir with [R, T] BOLD, return [R, T] readout signals.""" + n_regions, T = time_series.shape + if self._W_res is None or self._W_in.shape[1] != n_regions: + self._build_reservoir(n_regions) + + N = self._W_res.shape[0] + h = np.zeros(N) + Y = np.zeros((n_regions, T)) + + for t in range(T): + h = np.tanh(self._W_res @ h + self._W_in @ time_series[:, t]) + # Readout for each region: projection of h onto region i's input weights + Y[:, t] = self._W_in.T @ h # [R,] — W_in.T is [R, N] + + return Y # [R, T] + + def dFC(self, time_series, Fs): + Y = self._reservoir_readout(time_series) # [R, T] + _, T = Y.shape + + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + + FCSs, TR_array = [], [] + for l in range(0, T - W_samples + 1, step): + seg = Y[:, l : l + W_samples] + C = np.corrcoef(seg) + C[np.isnan(C)] = 0.0 + np.fill_diagonal(C, 1.0) + FCSs.append(np.clip(C, -1.0, 1.0)) + TR_array.append(int(l + W_samples // 2)) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/reservoir_edge_dcc_hybrid.py b/pydfc/dfc_methods/reservoir_edge_dcc_hybrid.py new file mode 100644 index 0000000..10e5e1c --- /dev/null +++ b/pydfc/dfc_methods/reservoir_edge_dcc_hybrid.py @@ -0,0 +1,286 @@ +""" +ReservoirEdgeDcc_hybrid. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class RESERVOIR_EDGE_DCC_HYBRID(BaseDFCMethod): + """State-free hybrid of ReservoirEchoStateFC, EdgeCoactivation, DCCConnectivity.""" + + MEASURE_NAME = "ReservoirEdgeDcc_hybrid" + COMPONENTS = ['ReservoirEchoStateFC', 'EdgeCoactivation', 'DCCConnectivity'] + HYBRID_RULE = "state_modulated" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/robust_differential_stft_hybrid.py b/pydfc/dfc_methods/robust_differential_stft_hybrid.py new file mode 100644 index 0000000..d22e51b --- /dev/null +++ b/pydfc/dfc_methods/robust_differential_stft_hybrid.py @@ -0,0 +1,286 @@ +""" +RobustDifferentialStft_hybrid. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class ROBUST_DIFFERENTIAL_STFT_HYBRID(BaseDFCMethod): + """State-free hybrid of RobustSlidingWindow, DifferentialCoactivationFC, STFTCoherence.""" + + MEASURE_NAME = "RobustDifferentialStft_hybrid" + COMPONENTS = ['RobustSlidingWindow', 'DifferentialCoactivationFC', 'Time-Freq'] + HYBRID_RULE = "robust_spectral_derivative" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/robust_sliding_window.py b/pydfc/dfc_methods/robust_sliding_window.py new file mode 100644 index 0000000..eb4a332 --- /dev/null +++ b/pydfc/dfc_methods/robust_sliding_window.py @@ -0,0 +1,114 @@ +""" +Robust Sliding Window (Spearman) dFC method. + +Reference: Pernet et al. (2013). Robust correlation analyses: false positive +and power validation using a new open source Matlab toolbox. Front Psychol, +3, 606. doi:10.3389/fpsyg.2012.00606 + +Application to dFC: discussed as a robust alternative to Pearson in +Lindquist et al. (2014). Evaluating dynamic bivariate correlations in +resting-state fMRI. NeuroImage, 101, 531-546. +doi:10.1016/j.neuroimage.2014.06.043 +""" + +import time + +import numpy as np +from scipy.stats import spearmanr + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class ROBUST_SLIDING_WINDOW(BaseDFCMethod): + """Sliding-window Spearman rank correlation as a robust dFC estimator. + + Standard Pearson correlation is sensitive to outliers and non-Gaussian + BOLD amplitude distributions. Spearman rank correlation replaces raw + values with their within-window ranks before computing correlation, + providing resistance to heavy-tailed noise and single-TR artefacts + without requiring explicit outlier removal. The output shares the same + windowed structure as the standard sliding window but with substantially + improved robustness under real fMRI noise conditions. + """ + + MEASURE_NAME = "RobustSlidingWindow" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + @staticmethod + def _spearman_matrix(data): + """Spearman correlation matrix for [n_regions, W] data.""" + # Rank each row + from scipy.stats import rankdata + + ranked = np.array([rankdata(row) for row in data]) + C = np.corrcoef(ranked) + C[np.isnan(C)] = 0.0 + np.fill_diagonal(C, 1.0) + return np.clip(C, -1.0, 1.0) + + def dFC(self, time_series, Fs): + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + n_regions, T = time_series.shape + + FCSs, TR_array = [], [] + for l in range(0, T - W_samples + 1, step): + seg = time_series[:, l : l + W_samples] + FCSs.append(self._spearman_matrix(seg)) + TR_array.append(int(l + W_samples // 2)) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/robust_sparse_edge_hybrid.py b/pydfc/dfc_methods/robust_sparse_edge_hybrid.py new file mode 100644 index 0000000..60ec416 --- /dev/null +++ b/pydfc/dfc_methods/robust_sparse_edge_hybrid.py @@ -0,0 +1,286 @@ +""" +RobustSparseEdge_hybrid. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class ROBUST_SPARSE_EDGE_HYBRID(BaseDFCMethod): + """State-free hybrid of RobustSlidingWindow, SparseCoactivationCodeFC, EdgeCoactivation.""" + + MEASURE_NAME = "RobustSparseEdge_hybrid" + COMPONENTS = ['RobustSlidingWindow', 'SparseCoactivationCodeFC', 'EdgeCoactivation'] + HYBRID_RULE = "sparse_residual" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/robust_volatility_sliding_hybrid_ensemble.py b/pydfc/dfc_methods/robust_volatility_sliding_hybrid_ensemble.py new file mode 100644 index 0000000..812456e --- /dev/null +++ b/pydfc/dfc_methods/robust_volatility_sliding_hybrid_ensemble.py @@ -0,0 +1,286 @@ +""" +RobustVolatilitySliding_hybrid_ensemble. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class ROBUST_VOLATILITY_SLIDING_HYBRID_ENSEMBLE(BaseDFCMethod): + """State-free hybrid of RobustSlidingWindow, VolatilityWeightedFC, SlidingWindow.""" + + MEASURE_NAME = "RobustVolatilitySliding_hybrid_ensemble" + COMPONENTS = ['RobustSlidingWindow', 'VolatilityWeightedFC', 'SlidingWindow'] + HYBRID_RULE = "ensemble_mean" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/sliding_phase_window_test.py b/pydfc/dfc_methods/sliding_phase_window_test.py new file mode 100644 index 0000000..bfaca7b --- /dev/null +++ b/pydfc/dfc_methods/sliding_phase_window_test.py @@ -0,0 +1,82 @@ +""" +My new dFC combined method. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class MY_NEW_METHOD(BaseDFCMethod): + """Short description of the method assumption.""" + + MEASURE_NAME = "MyNewMethod" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["min_periods"] is None: + self.params["min_periods"] = 10 + + @property + def measure_name(self): + return self.params["measure_name"] + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + FCSs = [] + TR_array = [] + + for tr in range(min_periods - 1, time_series.shape[1]): + matrix = np.corrcoef(time_series[:, : tr + 1]) + matrix[np.isnan(matrix)] = 0 + matrix[np.diag_indices_from(matrix)] = 1 + FCSs.append(matrix) + TR_array.append(tr) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC \ No newline at end of file diff --git a/pydfc/dfc_methods/sliding_volatility_dcc_hybrid.py b/pydfc/dfc_methods/sliding_volatility_dcc_hybrid.py new file mode 100644 index 0000000..7e8a191 --- /dev/null +++ b/pydfc/dfc_methods/sliding_volatility_dcc_hybrid.py @@ -0,0 +1,286 @@ +""" +SlidingVolatilityDcc_hybrid. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class SLIDING_VOLATILITY_DCC_HYBRID(BaseDFCMethod): + """State-free hybrid of SlidingWindow, VolatilityWeightedFC, DCCConnectivity.""" + + MEASURE_NAME = "SlidingVolatilityDcc_hybrid" + COMPONENTS = ['SlidingWindow', 'VolatilityWeightedFC', 'DCCConnectivity'] + HYBRID_RULE = "volatility_residual" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/sliding_window.py b/pydfc/dfc_methods/sliding_window.py index 25a8906..169d724 100644 --- a/pydfc/dfc_methods/sliding_window.py +++ b/pydfc/dfc_methods/sliding_window.py @@ -32,6 +32,7 @@ class SLIDING_WINDOW(BaseDFCMethod): + MEASURE_NAME = "SlidingWindow" def __init__(self, **params): self.logs_ = "" @@ -66,7 +67,7 @@ def __init__(self, **params): else: self.params[params_name] = None - self.params["measure_name"] = "SlidingWindow" + self.params["measure_name"] = self.MEASURE_NAME self.params["is_state_based"] = False assert ( diff --git a/pydfc/dfc_methods/sliding_window_clustr.py b/pydfc/dfc_methods/sliding_window_clustr.py index d181a2b..a16a81f 100644 --- a/pydfc/dfc_methods/sliding_window_clustr.py +++ b/pydfc/dfc_methods/sliding_window_clustr.py @@ -40,6 +40,7 @@ class SLIDING_WINDOW_CLUSTR(BaseDFCMethod): + MEASURE_NAME = "Clustering" def __init__(self, **params): @@ -86,7 +87,7 @@ def __init__(self, **params): else: self.params[params_name] = None - self.params["measure_name"] = "Clustering" + self.params["measure_name"] = self.MEASURE_NAME self.params["is_state_based"] = True if self.params["clstr_distance"] is None: diff --git a/pydfc/dfc_methods/sparse_coactivation_code.py b/pydfc/dfc_methods/sparse_coactivation_code.py new file mode 100644 index 0000000..e394787 --- /dev/null +++ b/pydfc/dfc_methods/sparse_coactivation_code.py @@ -0,0 +1,173 @@ +""" +Sparse Co-activation Code Connectivity (SCCC) — from sparse coding / CS. + +Source domain: Olshausen & Field (1996). Emergence of simple-cell receptive +field properties by learning a sparse code for natural images. Nature, 381, +607-609. Aharon et al. (2006). K-SVD: An algorithm for designing +overcomplete dictionaries for sparse representation. IEEE Trans. Signal +Process., 54(11), 4311-4322. + +The brain may encode information in sparse patterns of co-active regions. +A dictionary D of K activity atoms is learned from the group-level BOLD +data. At each timepoint the BOLD vector x(t) is sparsely reconstructed as +x̃(t) = D α(t) where α(t) is a sparse code. The sparse reconstruction +denoises the BOLD signal while preserving coherent co-activation structure. +Windowed correlation of the reconstructed signals x̃_i(t) captures coupling +that is mediated by shared dictionary atoms — connectivity as participation +in the same latent activity patterns. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class SPARSE_COACTIVATION_CODE(BaseDFCMethod): + """Group-level dictionary learning + per-subject sparse-code correlation. + + estimate_FCS (group level): + Learns a dictionary D ∈ ℝ^{K×R} from all subjects' BOLD data via + sklearn DictionaryLearning with a sparsity-inducing LASSO penalty. + + estimate_dFC (per subject): + 1. Sparse-code each TR: α(t) = argmin ½‖x(t) − Dα‖² + λ‖α‖₁ + 2. Reconstruct: x̃(t) = D α(t) (sparse denoised BOLD, same shape) + 3. Windowed Pearson correlation of x̃_i(t) across the window + + Unlike NMF_STATES (which learns a basis for FC matrices), SCCC learns a + basis for BOLD *activity patterns* and computes connectivity on the + sparse reconstruction rather than the raw signal. + """ + + MEASURE_NAME = "SparseCoactivationCodeFC" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.dictionary_ = None + + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "n_atoms", + "dict_alpha", + "normalization", + "num_subj", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for p in self.params_name_lst: + self.params[p] = params.get(p, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["n_atoms"] is None: + self.params["n_atoms"] = 20 + if self.params["dict_alpha"] is None: + self.params["dict_alpha"] = 0.1 + + @property + def measure_name(self): + return self.params["measure_name"] + + def estimate_FCS(self, time_series): + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + time_series = self.manipulate_time_series4FCS(time_series) + + tic = time.time() + + from sklearn.decomposition import DictionaryLearning + + all_data = [] + for subj_id in time_series.subj_id_lst: + subj_data = time_series.get_subj_ts(subjs_id=subj_id).data # [R, T] + all_data.append(subj_data.T) # [T, R] + X = np.vstack(all_data) # [total_T, R] + + n_atoms = int(self.params["n_atoms"]) + alpha = float(self.params["dict_alpha"]) + + dl = DictionaryLearning( + n_components=n_atoms, + alpha=alpha, + max_iter=200, + random_state=0, + fit_algorithm="cd", + transform_algorithm="lasso_cd", + n_jobs=1, + ) + dl.fit(X) + self.dictionary_ = dl.components_ # [n_atoms, R] + + self.set_FCS_fit_time(time.time() - tic) + return self + + def _sparse_reconstruct(self, data): + """Sparse-reconstruct [R, T] data → [R, T] via learned dictionary.""" + from sklearn.decomposition import SparseCoder + + if self.dictionary_ is None: + return data # fallback: no dictionary learned + + coder = SparseCoder( + dictionary=self.dictionary_, + transform_algorithm="lasso_cd", + transform_alpha=float(self.params["dict_alpha"]), + n_jobs=1, + ) + codes = coder.transform(data.T) # [T, n_atoms] + reconstruction = codes @ self.dictionary_ # [T, R] + return reconstruction.T # [R, T] + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + + recon = self._sparse_reconstruct(time_series.data) # [R, T] + n_regions, T = recon.shape + + W_samples = int(self.params["W"] * time_series.Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + + FCSs, TR_array = [], [] + for l in range(0, T - W_samples + 1, step): + seg = recon[:, l : l + W_samples] + C = np.corrcoef(seg) + C[np.isnan(C)] = 0.0 + np.fill_diagonal(C, 1.0) + FCSs.append(np.clip(C, -1.0, 1.0)) + TR_array.append(int(l + W_samples // 2)) + + self.set_dFC_assess_time(time.time() - tic) + + dFC_out = DFC(measure=self) + dFC_out.set_dFC( + FCSs=np.array(FCSs), + TR_array=np.array(TR_array), + TS_info=time_series.info_dict, + ) + return dFC_out diff --git a/pydfc/dfc_methods/sparse_dcc_exp_hybrid_ensemble.py b/pydfc/dfc_methods/sparse_dcc_exp_hybrid_ensemble.py new file mode 100644 index 0000000..8c3a19d --- /dev/null +++ b/pydfc/dfc_methods/sparse_dcc_exp_hybrid_ensemble.py @@ -0,0 +1,286 @@ +""" +SparseDccExp_hybrid_ensemble. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class SPARSE_DCC_EXP_HYBRID_ENSEMBLE(BaseDFCMethod): + """State-free hybrid of SparseCoactivationCodeFC, DCCConnectivity, ExponentialWindow.""" + + MEASURE_NAME = "SparseDccExp_hybrid_ensemble" + COMPONENTS = ['SparseCoactivationCodeFC', 'DCCConnectivity', 'ExponentialWindow'] + HYBRID_RULE = "ensemble_mean" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/spectral_similarity.py b/pydfc/dfc_methods/spectral_similarity.py new file mode 100644 index 0000000..4e54e76 --- /dev/null +++ b/pydfc/dfc_methods/spectral_similarity.py @@ -0,0 +1,121 @@ +""" +Spectral Similarity FC (SpecSimFC) — novel dFC method. + +Two brain regions are "spectrally similar" if their BOLD power spectra +have the same shape — they oscillate in the same frequency bands at the +same relative strengths. This method quantifies pairwise dynamic +connectivity as the Bhattacharyya coefficient between normalised +within-window power spectra, independent of signal amplitude. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class SPECTRAL_SIMILARITY(BaseDFCMethod): + """Windowed Bhattacharyya coefficient between power spectral densities. + + Within each sliding window, the power spectral density (PSD) of each + region is estimated via the squared FFT magnitude and normalised to + a probability distribution over frequencies. The pairwise similarity + between regions i and j is: + + FC[i,j] = Σ_f √(PSD_i(f) · PSD_j(f)) (Bhattacharyya coefficient) + + This equals 1 when spectra are identical and 0 when they have no + overlapping frequency content. Unlike coherence, spectral similarity + does not require phase alignment — two regions with identical spectral + profiles but random phase relations will have FC = 1. It therefore + captures "oscillatory repertoire coupling" rather than phase-based + synchrony. + """ + + MEASURE_NAME = "SpectralSimilarityFC" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for p in self.params_name_lst: + self.params[p] = params.get(p, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + @staticmethod + def _spectral_similarity_matrix(data): + """[R, R] Bhattacharyya coefficient matrix for [R, W] data.""" + n_regions, W = data.shape + # Power spectral density via FFT (positive frequencies only) + fft_mag = np.abs(np.fft.rfft(data, axis=1)) ** 2 # [R, W//2+1] + # Add tiny constant for numerical stability, then normalise + fft_mag += 1e-10 + psd = fft_mag / fft_mag.sum(axis=1, keepdims=True) # [R, F] probability + + # Bhattacharyya coefficient: BC[i,j] = sum_f sqrt(psd_i * psd_j) + sqrt_psd = np.sqrt(psd) # [R, F] + C = sqrt_psd @ sqrt_psd.T # [R, R] + C = np.clip(C, 0.0, 1.0) + np.fill_diagonal(C, 1.0) + return C + + def dFC(self, time_series, Fs): + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + _, T = time_series.shape + + FCSs, TR_array = [], [] + for l in range(0, T - W_samples + 1, step): + seg = time_series[:, l : l + W_samples] + FCSs.append(self._spectral_similarity_matrix(seg)) + TR_array.append(int(l + W_samples // 2)) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/spectral_states.py b/pydfc/dfc_methods/spectral_states.py new file mode 100644 index 0000000..538cb86 --- /dev/null +++ b/pydfc/dfc_methods/spectral_states.py @@ -0,0 +1,175 @@ +"""Spectral state-based dFC.""" + +import time + +import numpy as np +from sklearn.cluster import SpectralClustering + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +def _corr(samples): + if samples.shape[0] < 2: + return np.eye(samples.shape[1], dtype=float) + cov = np.cov(samples, rowvar=False) + std = np.sqrt(np.maximum(np.diag(cov), 1e-12)) + den = np.outer(std, std) + corr = np.divide(cov, den, out=np.zeros_like(cov), where=den > 0) + corr[np.diag_indices_from(corr)] = 1.0 + return 0.5 * (corr + corr.T) + + +def _softmax_dist(features, centers, temperature): + distances = np.sum((features[:, None, :] - centers[None, :, :]) ** 2, axis=2) + logits = -distances / max(float(temperature), 1e-6) + logits = logits - np.max(logits, axis=1, keepdims=True) + probs = np.exp(logits) + probs = probs / np.maximum(np.sum(probs, axis=1, keepdims=True), 1e-12) + return np.argmin(distances, axis=1).astype(int), probs + + +class SPECTRAL_STATES(BaseDFCMethod): + MEASURE_NAME = "SpectralStates" + """Manifold-aware states discovered with spectral clustering.""" + + def __init__(self, **params): + self._train_sample_limit = 2500 + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "n_states", + "n_neighbors", + "assignment_temperature", + "normalization", + "num_subj", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {name: params.get(name, None) for name in self.params_name_lst} + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = True + if self.params["n_states"] is None: + self.params["n_states"] = 5 + if self.params["n_neighbors"] is None: + self.params["n_neighbors"] = 15 + if self.params["assignment_temperature"] is None: + self.params["assignment_temperature"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _chunks(self, time_series): + return [ + time_series.get_subj_ts(subjs_id=subj).data.T.copy() + for subj in time_series.subj_id_lst + ] + + def _fit_matrix(self, chunks): + each = max(1, int(self._train_sample_limit) // max(len(chunks), 1)) + sampled = [] + for chunk in chunks: + if chunk.shape[0] <= each: + sampled.append(chunk) + else: + idx = np.linspace(0, chunk.shape[0] - 1, each, dtype=int) + sampled.append(chunk[idx, :]) + return np.concatenate(sampled, axis=0) + + def estimate_FCS(self, time_series): + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4FCS(time_series) + tic = time.time() + + chunks = self._chunks(time_series) + fit_matrix = self._fit_matrix(chunks) + n_states = min(int(self.params["n_states"]), fit_matrix.shape[0]) + self.params["n_states"] = n_states + n_neighbors = min( + int(self.params["n_neighbors"]), max(fit_matrix.shape[0] - 1, 1) + ) + + labels_fit = SpectralClustering( + n_clusters=n_states, + affinity="nearest_neighbors", + n_neighbors=n_neighbors, + assign_labels="kmeans", + random_state=None, + ).fit_predict(fit_matrix) + + self.centers_ = np.zeros((n_states, fit_matrix.shape[1]), dtype=float) + fallback = np.mean(fit_matrix, axis=0) + for state in range(n_states): + mask = labels_fit == state + self.centers_[state, :] = ( + np.mean(fit_matrix[mask], axis=0) if np.any(mask) else fallback + ) + + labels_chunks, _ = zip( + *[ + _softmax_dist(chunk, self.centers_, self.params["assignment_temperature"]) + for chunk in chunks + ] + ) + self.Z = np.concatenate(labels_chunks, axis=0) + self.FCS_ = np.zeros( + (n_states, chunks[0].shape[1], chunks[0].shape[1]), dtype=float + ) + for state in range(n_states): + samples = [ + chunk[labels == state, :] + for chunk, labels in zip(chunks, labels_chunks) + if np.any(labels == state) + ] + self.FCS_[state, :, :] = ( + _corr(np.concatenate(samples, axis=0)) + if samples + else np.eye(chunks[0].shape[1]) + ) + + counts = np.full((n_states, n_states), 1.0, dtype=float) + for labels in labels_chunks: + for a, b in zip(labels[:-1], labels[1:]): + counts[a, b] += 1.0 + self.TPM = counts / np.maximum(np.sum(counts, axis=1, keepdims=True), 1e-12) + + self.set_mean_activity(time_series) + self.set_FCS_fit_time(time.time() - tic) + return self + + def estimate_dFC(self, time_series): + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + features = time_series.data.T.copy() + labels, probs = _softmax_dist( + features, self.centers_, self.params["assignment_temperature"] + ) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC( + FCSs=self.FCS_, + FCS_idx=labels, + FCS_proba=probs, + TS_info=time_series.info_dict, + TR_array=np.arange(features.shape[0], dtype=int), + ) + return dFC diff --git a/pydfc/dfc_methods/state_space_neighborhood.py b/pydfc/dfc_methods/state_space_neighborhood.py new file mode 100644 index 0000000..9e7eb90 --- /dev/null +++ b/pydfc/dfc_methods/state_space_neighborhood.py @@ -0,0 +1,125 @@ +""" +State-Space Neighborhood FC (SSNFC) — novel dFC method. + +Rather than averaging BOLD signals over a temporal window, this method +finds the k timepoints in the entire recording whose whole-brain state +vector is most similar to the current state, then computes the +correlation matrix over those k nearest neighbors in state space. + +Each timepoint gets its own FC matrix based on *where* in state space +the brain is, not *when* the measurement was taken. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class STATE_SPACE_NEIGHBORHOOD(BaseDFCMethod): + """Per-timepoint FC computed over k nearest neighbors in BOLD state space. + + For each timepoint t, the k timepoints with the smallest Euclidean + distance ‖x(t') − x(t)‖ are found (excluding a Theiler window of + ±theiler TRs to avoid temporal autocorrelation bias). The Pearson + correlation matrix computed over these k state-space neighbors is + the instantaneous dFC estimate at t. + + Unlike windowed methods, the "averaging region" is defined by + similarity in brain state rather than proximity in time. Timepoints + that are far apart in time but share the same brain configuration will + contribute to each other's FC estimate, capturing attractor-relative + connectivity. + """ + + MEASURE_NAME = "StateSpaceNeighborhoodFC" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "k_neighbors", + "theiler", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for p in self.params_name_lst: + self.params[p] = params.get(p, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["k_neighbors"] is None: + self.params["k_neighbors"] = 20 + if self.params["theiler"] is None: + self.params["theiler"] = 5 + + @property + def measure_name(self): + return self.params["measure_name"] + + def dFC(self, time_series, Fs): + n_regions, T = time_series.shape + k = int(self.params["k_neighbors"]) + theiler = int(self.params["theiler"]) + + k = min(k, T - 2 * theiler - 1) + if k < 3: + # Not enough data for meaningful FC; return identity matrices + return (np.tile(np.eye(n_regions), (T, 1, 1)), np.arange(T)) + + # Pairwise squared Euclidean distances [T, T] + # ||x(t) - x(t')||^2 = ||x(t)||^2 + ||x(t')||^2 - 2 x(t)·x(t') + X = time_series.T # [T, R] + sq_norms = np.sum(X**2, axis=1) # [T] + D2 = sq_norms[:, None] + sq_norms[None, :] - 2 * (X @ X.T) # [T, T] + D2 = np.maximum(D2, 0.0) + + FCSs, TR_array = [], [] + for t in range(T): + # Mask out Theiler window + d = D2[t].copy() + lo = max(0, t - theiler) + hi = min(T, t + theiler + 1) + d[lo:hi] = np.inf + # k nearest neighbors + nn_idx = np.argpartition(d, k)[:k] + seg = time_series[:, nn_idx] # [R, k] + C = np.corrcoef(seg) + C[np.isnan(C)] = 0.0 + np.fill_diagonal(C, 1.0) + FCSs.append(np.clip(C, -1.0, 1.0)) + TR_array.append(t) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/stft_coherence.py b/pydfc/dfc_methods/stft_coherence.py new file mode 100644 index 0000000..915fe20 --- /dev/null +++ b/pydfc/dfc_methods/stft_coherence.py @@ -0,0 +1,131 @@ +""" +Short-Time Fourier Transform Coherence (STFT-Coh) dFC method. + +Reference: Wacker & Witte (2011). Time-frequency techniques in biomedical signal +analysis. Methods Inf Med, 50(5), 435-444. doi:10.3414/ME10-01-0083 + +Application to fMRI: Chang & Glover (2010). Time-frequency dynamics of resting- +state brain connectivity measured with fMRI. NeuroImage, 50(1), 81-98. +doi:10.1016/j.neuroimage.2009.12.011 +""" + +import time + +import numpy as np +from scipy.signal import csd, welch + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class STFT_COHERENCE(BaseDFCMethod): + """Sliding-window spectral coherence via Welch cross-spectral estimation. + + Within each window the magnitude-squared coherence between every pair of + regions is computed by Welch's method: + + Coh_ij(f) = |S_xy(f)|² / (S_xx(f) · S_yy(f)) + + and then summed over the low-frequency resting-state band (default + 0.01–0.1 Hz). Unlike correlation, coherence is frequency-specific and + normalised, capturing oscillatory coupling strength independent of + signal amplitude. Unlike continuous-wavelet coherence, STFT coherence + uses a uniform frequency resolution. + """ + + MEASURE_NAME = "STFTCoherence" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "f_low", + "f_high", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 60 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["f_low"] is None: + self.params["f_low"] = 0.01 + if self.params["f_high"] is None: + self.params["f_high"] = 0.1 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _band_coherence(self, xi, xj, Fs, f_low, f_high): + """Summed magnitude-squared coherence over [f_low, f_high].""" + nperseg = min(len(xi), max(8, len(xi) // 4)) + freqs, Pxx = welch(xi, fs=Fs, nperseg=nperseg) + _, Pyy = welch(xj, fs=Fs, nperseg=nperseg) + _, Pxy = csd(xi, xj, fs=Fs, nperseg=nperseg) + band = (freqs >= f_low) & (freqs <= f_high) + if not np.any(band): + return 0.0 + denom = Pxx[band] * Pyy[band] + coh = np.where(denom > 0, np.abs(Pxy[band]) ** 2 / denom, 0.0) + return float(coh.mean()) + + def dFC(self, time_series, Fs): + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + n_regions, T = time_series.shape + f_low = float(self.params["f_low"]) + f_high = float(self.params["f_high"]) + + FCSs, TR_array = [], [] + for l in range(0, T - W_samples + 1, step): + seg = time_series[:, l : l + W_samples] + C = np.zeros((n_regions, n_regions)) + for i in range(n_regions): + C[i, i] = 1.0 + for j in range(i + 1, n_regions): + c = self._band_coherence(seg[i], seg[j], Fs, f_low, f_high) + C[i, j] = c + C[j, i] = c + FCSs.append(C) + TR_array.append(int(l + W_samples // 2)) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/stft_edge_adaptive_hybrid_ensemble.py b/pydfc/dfc_methods/stft_edge_adaptive_hybrid_ensemble.py new file mode 100644 index 0000000..5e14272 --- /dev/null +++ b/pydfc/dfc_methods/stft_edge_adaptive_hybrid_ensemble.py @@ -0,0 +1,286 @@ +""" +StftEdgeAdaptive_hybrid_ensemble. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class STFT_EDGE_ADAPTIVE_HYBRID_ENSEMBLE(BaseDFCMethod): + """State-free hybrid of STFTCoherence, EdgeCoactivation, AdaptiveExponentialWindow.""" + + MEASURE_NAME = "StftEdgeAdaptive_hybrid_ensemble" + COMPONENTS = ['Time-Freq', 'EdgeCoactivation', 'AdaptiveExponentialWindow'] + HYBRID_RULE = "ensemble_mean" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/stft_exp_kalman_hybrid.py b/pydfc/dfc_methods/stft_exp_kalman_hybrid.py new file mode 100644 index 0000000..5712c6d --- /dev/null +++ b/pydfc/dfc_methods/stft_exp_kalman_hybrid.py @@ -0,0 +1,286 @@ +""" +StftExpKalman_hybrid. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class STFT_EXP_KALMAN_HYBRID(BaseDFCMethod): + """State-free hybrid of STFTCoherence, ExponentialWindow, KalmanCovariance.""" + + MEASURE_NAME = "StftExpKalman_hybrid" + COMPONENTS = ['STFTCoherence', 'ExponentialWindow', 'KalmanCovariance'] + HYBRID_RULE = "spectral_gate" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/stft_quantum_sparse_hybrid_ensemble.py b/pydfc/dfc_methods/stft_quantum_sparse_hybrid_ensemble.py new file mode 100644 index 0000000..87956fe --- /dev/null +++ b/pydfc/dfc_methods/stft_quantum_sparse_hybrid_ensemble.py @@ -0,0 +1,286 @@ +""" +StftQuantumSparse_hybrid_ensemble. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class STFT_QUANTUM_SPARSE_HYBRID_ENSEMBLE(BaseDFCMethod): + """State-free hybrid of STFTCoherence, QuantumMutualInformationFC, SparseCoactivationCodeFC.""" + + MEASURE_NAME = "StftQuantumSparse_hybrid_ensemble" + COMPONENTS = ['Time-Freq', 'QuantumMutualInformationFC', 'SparseCoactivationCodeFC'] + HYBRID_RULE = "ensemble_mean" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/synchrony_likelihood_window.py b/pydfc/dfc_methods/synchrony_likelihood_window.py new file mode 100644 index 0000000..7de324d --- /dev/null +++ b/pydfc/dfc_methods/synchrony_likelihood_window.py @@ -0,0 +1,174 @@ +""" +Windowed Synchrony Likelihood (SL) dFC method. + +Reference: Stam & van Dijk (2002). Synchronization likelihood: an unbiased +measure of generalized synchronization in multivariate data sets. +Physica D, 163(3-4), 236-251. doi:10.1016/S0167-2789(01)00386-4 + +Application to fMRI: Stam & van Straaten (2012). The organization of +physiological brain networks. Clin Neurophysiol, 123(6), 1067-1087. +doi:10.1016/j.clinph.2012.01.011 +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class SYNCHRONY_LIKELIHOOD_WINDOW(BaseDFCMethod): + """Sliding-window nonlinear synchrony based on joint phase-space recurrence. + + Each region's BOLD is delay-embedded into an m-dimensional phase-space + trajectory. Within each window, for every reference time t₀ and every + region i, a "hit" is counted when the embedded state at time t is within + radius r_i of the reference state (Theiler correction applied to exclude + temporal neighbours). r_i is chosen adaptively so that the average hit + probability equals ref_prob. + + Synchrony Likelihood for the pair (i, j) is: + + SL_ij = P(hit_i AND hit_j) / (P(hit_i) × P(hit_j)) + + A value near 1 indicates no more joint recurrences than expected by + chance; values > 1 indicate nonlinear generalized synchrony. The matrix + is symmetrised and mapped to [0, 1] by capping at an empirical maximum. + """ + + MEASURE_NAME = "SynchronyLikelihoodWindow" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "embed_dim", + "embed_lag", + "ref_prob", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["embed_dim"] is None: + self.params["embed_dim"] = 2 + if self.params["embed_lag"] is None: + self.params["embed_lag"] = 1 + if self.params["ref_prob"] is None: + self.params["ref_prob"] = 0.05 + + @property + def measure_name(self): + return self.params["measure_name"] + + @staticmethod + def _embed(x, m, lag): + """Delay-embed scalar time series x into m-dimensional vectors.""" + T = len(x) - (m - 1) * lag + if T <= 0: + return x[:, np.newaxis] + return np.stack([x[k * lag : k * lag + T] for k in range(m)], axis=1) + + @staticmethod + def _recurrence_matrix(X, ref_prob, theiler=1): + """Boolean recurrence matrix with adaptive radius for hit_prob = ref_prob.""" + T = X.shape[0] + # Pairwise L2 distances + diff = X[:, np.newaxis, :] - X[np.newaxis, :, :] # [T, T, m] + D = np.sqrt((diff**2).sum(axis=2)) # [T, T] + # Exclude diagonal and Theiler strip from radius estimation + mask = np.abs(np.arange(T)[:, None] - np.arange(T)[None, :]) > theiler + valid_dists = D[mask] + r = np.quantile(valid_dists, ref_prob) if len(valid_dists) else 0.0 + R = (D <= r) & mask + return R.astype(float) + + def _sl_window(self, data_window): + """Compute [R, R] SL matrix for one window of data [R, W].""" + n_regions, W = data_window.shape + m = int(self.params["embed_dim"]) + lag = int(self.params["embed_lag"]) + ref_prob = float(self.params["ref_prob"]) + theiler = max(1, lag) + + # Build per-region recurrence matrices + R_mats = [] + for i in range(n_regions): + Xi = self._embed(data_window[i], m, lag) + Ri = self._recurrence_matrix(Xi, ref_prob, theiler) + R_mats.append(Ri) + + T_emb = R_mats[0].shape[0] + n_pairs = T_emb * (T_emb - 1) # denominator (upper + lower triangle) + + # Marginal hit probabilities + p = np.array([Ri.sum() / max(n_pairs, 1) for Ri in R_mats]) + + SL = np.zeros((n_regions, n_regions)) + np.fill_diagonal(SL, 1.0) + for i in range(n_regions): + for j in range(i + 1, n_regions): + p_ij = (R_mats[i] * R_mats[j]).sum() / max(n_pairs, 1) + denom = p[i] * p[j] + sl = (p_ij / denom) if denom > 1e-12 else 0.0 + # Map to [0, 1]: SL=1 is chance; typical range [1, 1/(ref_prob)] + sl_norm = min(sl / (1.0 / ref_prob), 1.0) if sl > 0 else 0.0 + SL[i, j] = sl_norm + SL[j, i] = sl_norm + + return SL + + def dFC(self, time_series, Fs): + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + n_regions, T = time_series.shape + + FCSs, TR_array = [], [] + for l in range(0, T - W_samples + 1, step): + seg = time_series[:, l : l + W_samples] + C = self._sl_window(seg) + FCSs.append(C) + TR_array.append(int(l + W_samples // 2)) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/tangent_space_fc.py b/pydfc/dfc_methods/tangent_space_fc.py new file mode 100644 index 0000000..115896f --- /dev/null +++ b/pydfc/dfc_methods/tangent_space_fc.py @@ -0,0 +1,178 @@ +""" +Tangent Space FC (TangentFC) — novel dFC method. + +The space of symmetric positive definite (SPD) matrices is a curved +Riemannian manifold. Ordinary subtraction of FC matrices mixes static +and dynamic components in a geometrically ill-defined way. This method +maps each windowed FC matrix into the tangent space at the group-level +geometric mean FC, providing a linearised, mean-centred dFC representation +that is invariant to the dominant static connectivity pattern. +""" + +import time + +import numpy as np +from scipy.linalg import logm, sqrtm + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class TANGENT_SPACE_FC(BaseDFCMethod): + """Riemannian tangent-space projection of windowed FC matrices. + + estimate_FCS computes the group-level geometric mean connectivity M̄ + (approximated here as the Euclidean mean of windowed FC matrices, then + projected to the nearest SPD matrix). + + For each subject window with FC matrix S, the tangent-space projection is: + + T = M̄^{-½} S M̄^{-½} (whitening) + T_log = logm(T) (Riemannian log-map to flat space) + + T_log is symmetric, zero-mean across windows, and captures dynamic + deviations from the group mean in a geometrically principled way. + It is rescaled element-wise to [-1, 1] for compatibility with the + standard dFC matrix convention. + """ + + MEASURE_NAME = "TangentSpaceFC" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.mean_FC_ = None + self.sqrt_inv_mean_ = None + + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "normalization", + "num_subj", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for p in self.params_name_lst: + self.params[p] = params.get(p, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _windowed_fc(self, data, Fs): + """Compute list of FC matrices for sliding windows of [R, T] data.""" + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + _, T = data.shape + matrices, centers = [], [] + for l in range(0, T - W_samples + 1, step): + seg = data[:, l : l + W_samples] + C = np.corrcoef(seg) + C[np.isnan(C)] = 0.0 + np.fill_diagonal(C, 1.0) + matrices.append(C) + centers.append(int(l + W_samples // 2)) + return matrices, centers + + @staticmethod + def _nearest_spd(A): + """Project a symmetric matrix to the nearest SPD matrix.""" + A = (A + A.T) / 2.0 + eigvals, eigvecs = np.linalg.eigh(A) + eigvals = np.maximum(eigvals, 1e-6) + return eigvecs @ np.diag(eigvals) @ eigvecs.T + + def estimate_FCS(self, time_series): + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + time_series = self.manipulate_time_series4FCS(time_series) + Fs = time_series.Fs + + tic = time.time() + + all_matrices = [] + for subj_id in time_series.subj_id_lst: + subj_data = time_series.get_subj_ts(subjs_id=subj_id).data + mats, _ = self._windowed_fc(subj_data, Fs) + all_matrices.extend(mats) + + # Group mean FC (Euclidean approximation) + mean_FC = np.mean(all_matrices, axis=0) + self.mean_FC_ = self._nearest_spd(mean_FC) + + # Pre-compute M^{-1/2} + sqrt_mean = np.real(sqrtm(self.mean_FC_)) + try: + self.sqrt_inv_mean_ = np.linalg.inv(sqrt_mean) + except np.linalg.LinAlgError: + self.sqrt_inv_mean_ = np.eye(self.mean_FC_.shape[0]) + + self.set_FCS_fit_time(time.time() - tic) + return self + + def _project_to_tangent(self, S): + """Map windowed FC matrix S to tangent space at mean_FC_.""" + n = S.shape[0] + S_spd = self._nearest_spd(S) + T = self.sqrt_inv_mean_ @ S_spd @ self.sqrt_inv_mean_ + T = self._nearest_spd(T) + try: + T_log = np.real(logm(T)) + except Exception: + T_log = T - np.eye(n) # first-order fallback + T_log = (T_log + T_log.T) / 2.0 + # Rescale to [-1, 1] + abs_max = np.abs(T_log[np.triu_indices(n, k=1)]).max() + if abs_max > 1e-12: + T_log = T_log / abs_max + np.fill_diagonal(T_log, 1.0) + return np.clip(T_log, -1.0, 1.0) + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + matrices, centers = self._windowed_fc(time_series.data, time_series.Fs) + + if self.mean_FC_ is None: + # Fallback: use subject-level mean if estimate_FCS was not called + mean_FC = np.mean(matrices, axis=0) + self.mean_FC_ = self._nearest_spd(mean_FC) + from scipy.linalg import sqrtm as _sqrtm + + sqrt_mean = np.real(_sqrtm(self.mean_FC_)) + try: + self.sqrt_inv_mean_ = np.linalg.inv(sqrt_mean) + except np.linalg.LinAlgError: + self.sqrt_inv_mean_ = np.eye(self.mean_FC_.shape[0]) + + FCSs = np.array([self._project_to_tangent(S) for S in matrices]) + TR_array = np.array(centers) + + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/temporal_asymmetry.py b/pydfc/dfc_methods/temporal_asymmetry.py new file mode 100644 index 0000000..4b73deb --- /dev/null +++ b/pydfc/dfc_methods/temporal_asymmetry.py @@ -0,0 +1,138 @@ +""" +Temporal Asymmetry FC (TAFC) — novel dFC method. + +For each region pair (i, j) the forward lagged correlation r(i→j, τ) +and backward lagged correlation r(j→i, τ) are both computed within each +sliding window. The dFC value is |r(i→j, τ) − r(j→i, τ)|: the absolute +asymmetry of directed influence. Pairs with high asymmetry have one-way +driving dynamics; pairs near zero are bidirectionally symmetric. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class TEMPORAL_ASYMMETRY(BaseDFCMethod): + """Absolute directed-coupling asymmetry at a fixed temporal lag. + + r_fwd[i,j] = corr(x_i[0:W-τ], x_j[τ:W]) — i leads j + r_bwd[i,j] = corr(x_j[0:W-τ], x_i[τ:W]) — j leads i + + FC[i,j] = |r_fwd[i,j] − r_bwd[i,j]| + + The matrix is symmetric (|a−b| = |b−a|) and bounded in [0, 2]. + It is zero for bidirectionally symmetric coupling and maximal when + influence is entirely one-directional. Unlike Granger causality, + no autoregressive model is fit; the measure is simply the signed + difference of lagged Pearson correlations. + """ + + MEASURE_NAME = "TemporalAsymmetryFC" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "lag", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for p in self.params_name_lst: + self.params[p] = params.get(p, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["lag"] is None: + self.params["lag"] = 1 + + @property + def measure_name(self): + return self.params["measure_name"] + + @staticmethod + def _corr(a, b): + """Pearson correlation between vectors a and b.""" + a = a - a.mean() + b = b - b.mean() + sa = np.sqrt((a**2).sum()) + sb = np.sqrt((b**2).sum()) + if sa < 1e-12 or sb < 1e-12: + return 0.0 + return float(np.dot(a, b) / (sa * sb)) + + def _asymmetry_matrix(self, data): + """[R, R] asymmetry matrix for [R, W] window data.""" + n_regions, W = data.shape + lag = int(self.params["lag"]) + if lag >= W: + return np.zeros((n_regions, n_regions)) + + # forward[i,j]: i leads j by lag + r_fwd = np.zeros((n_regions, n_regions)) + for i in range(n_regions): + for j in range(n_regions): + if i != j: + r_fwd[i, j] = self._corr(data[i, : W - lag], data[j, lag:]) + + # backward is just r_fwd transposed + r_bwd = r_fwd.T + asym = np.abs(r_fwd - r_bwd) + # Already symmetric: asym[i,j] = |r_fwd[i,j] - r_bwd[i,j]| + # = |r_fwd[i,j] - r_fwd[j,i]| + # asym[j,i] = |r_fwd[j,i] - r_fwd[i,j]| = asym[i,j] ✓ + np.fill_diagonal(asym, 0.0) + return asym / 2.0 # normalise to [0, 1] + + def dFC(self, time_series, Fs): + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + _, T = time_series.shape + + FCSs, TR_array = [], [] + for l in range(0, T - W_samples + 1, step): + seg = time_series[:, l : l + W_samples] + FCSs.append(self._asymmetry_matrix(seg)) + TR_array.append(int(l + W_samples // 2)) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/temporal_derivative_multiplication.py b/pydfc/dfc_methods/temporal_derivative_multiplication.py new file mode 100644 index 0000000..f8e4154 --- /dev/null +++ b/pydfc/dfc_methods/temporal_derivative_multiplication.py @@ -0,0 +1,132 @@ +""" +Multiplication of Temporal Derivatives (MTD) dFC method. + +Reference: Shine et al. (2015). Estimation of dynamic functional connectivity +using Multiplication of Temporal Derivatives (MTD). NeuroImage, 122, 399-407. +doi:10.1016/j.neuroimage.2015.07.064 +""" + +import time + +import numpy as np +from scipy.ndimage import gaussian_filter1d + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class TEMPORAL_DERIVATIVE_MULTIPLICATION(BaseDFCMethod): + """Instantaneous FC from the outer product of z-scored BOLD temporal + derivatives, smoothed with a Gaussian kernel (Shine et al., 2015). + + For each pair of regions (i, j) the raw MTD signal at time t is + dx_i(t) * dx_j(t), where dx_i(t) = x_i(t) - x_i(t-1). After z-scoring + each regional derivative series, the stack of outer products is convolved + with a Gaussian kernel of width sigma_s seconds along the time axis and + then normalized to a correlation scale. The result is one FC matrix per TR + with no explicit window length parameter. + """ + + MEASURE_NAME = "TemporalDerivativeMultiplication" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "sigma_s", + "min_periods", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + # sigma_s: Gaussian smoothing kernel width in seconds + if self.params["sigma_s"] is None: + self.params["sigma_s"] = 3.0 + # min_periods: burn-in TRs to skip while the Gaussian kernel warms up + if self.params["min_periods"] is None: + self.params["min_periods"] = 10 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _normalize_to_corr(self, cov_stack): + """Normalize a [T, R, R] covariance stack to correlation scale.""" + R = cov_stack.shape[1] + diag = cov_stack[:, np.arange(R), np.arange(R)] # [T, R] + scale = np.sqrt(np.einsum("ti,tj->tij", diag, diag)) # [T, R, R] + scale = np.where(scale > 0, scale, 1.0) + corr = cov_stack / scale + corr = np.clip(corr, -1.0, 1.0) + corr[:, np.arange(R), np.arange(R)] = 1.0 + corr[np.isnan(corr)] = 0.0 + return corr + + def dFC(self, time_series, Fs): + sigma_s = float(self.params["sigma_s"]) + min_periods = int(self.params["min_periods"]) + # convert smoothing width from seconds to samples (floor at 0.5) + sigma_samples = max(sigma_s * Fs, 0.5) + + # first-order temporal derivatives: shape [n_regions, T-1] + dX = np.diff(time_series, axis=1).astype(float) + + # z-score each region's derivative series across time + mu = dX.mean(axis=1, keepdims=True) + sd = dX.std(axis=1, keepdims=True) + dX = (dX - mu) / np.where(sd > 1e-10, sd, 1e-10) + + # instantaneous outer-product matrices: shape [T-1, n_regions, n_regions] + C_raw = np.einsum("it,jt->tij", dX, dX) + + # Gaussian smoothing along the time axis + C_smooth = gaussian_filter1d(C_raw, sigma=sigma_samples, axis=0) + + # normalize to correlation scale + C_corr = self._normalize_to_corr(C_smooth) + + # skip early TRs where the Gaussian has not yet accumulated enough signal + start = max(min_periods - 1, 0) + FCSs = C_corr[start:] + # derivative at dX[:, t] corresponds to TR index t+1 in the original signal + TR_array = np.arange(start + 1, time_series.shape[1]) + + return FCSs, TR_array + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert ( + type(time_series) is TIME_SERIES + ), "time_series must be of TIME_SERIES class." + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/time_freq.py b/pydfc/dfc_methods/time_freq.py index e820a8c..a95c8bf 100644 --- a/pydfc/dfc_methods/time_freq.py +++ b/pydfc/dfc_methods/time_freq.py @@ -69,6 +69,7 @@ class TIME_FREQ(BaseDFCMethod): + MEASURE_NAME = "Time-Freq" def __init__(self, coi_correction=True, **params): @@ -104,7 +105,7 @@ def __init__(self, coi_correction=True, **params): else: self.params[params_name] = None - self.params["measure_name"] = "Time-Freq" + self.params["measure_name"] = self.MEASURE_NAME self.params["is_state_based"] = False self.params["coi_correction"] = coi_correction diff --git a/pydfc/dfc_methods/time_reversal_asymmetry.py b/pydfc/dfc_methods/time_reversal_asymmetry.py new file mode 100644 index 0000000..199f2c0 --- /dev/null +++ b/pydfc/dfc_methods/time_reversal_asymmetry.py @@ -0,0 +1,136 @@ +""" +Time-Reversal Asymmetry Connectivity (TRAC) — from stochastic thermodynamics. + +Source domain: Roldan & Parrondo (2010). Estimating dissipation from +single stationary trajectories. PRL, 105, 150607. + +A system in thermodynamic equilibrium is time-symmetric: its statistics +are the same forwards and backwards in time. The joint time-reversal +asymmetry A_ij = E[(x_i(t+1) − x_i(t−1)) · x_j(t)] measures how much +the velocity of region i co-varies with the position of region j — a +quantity that is zero for time-symmetric (equilibrium) dynamics and +nonzero for irreversible (non-equilibrium) processes. The symmetric +combination |A_ij + A_ji| captures the joint departure from time-symmetry. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class TIME_REVERSAL_ASYMMETRY(BaseDFCMethod): + """Windowed joint time-reversal asymmetry as a dFC measure. + + Within each window, the central-difference velocity of region i is: + v_i(t) = x_i(t+1) − x_i(t−1) + + The normalised cross-covariance A[i,j] = corr(v_i, x_j) is computed. + The symmetric joint asymmetry matrix is: + + FC[i,j] = |A[i,j] + A[j,i]| / 2 + + A[i,j] captures "how much does region i's velocity predict region j's + position?" while A[j,i] captures the reverse. Their sum is large when + both directions show asymmetric velocity-position coupling — the + signature of a coupled non-equilibrium process. The measure is bounded + in [0, 1] and zero for time-symmetric (e.g., white noise) inputs. + """ + + MEASURE_NAME = "TimeReversalAsymmetryFC" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for p in self.params_name_lst: + self.params[p] = params.get(p, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + @staticmethod + def _zscore_rows(x): + mu = x.mean(axis=1, keepdims=True) + sd = x.std(axis=1, keepdims=True) + sd = np.where(sd > 1e-12, sd, 1.0) + return (x - mu) / sd + + def _trac_matrix(self, data): + """[R, R] TRAC matrix for [R, W] window data.""" + # Central-difference velocity (reduces window by 2) + vel = data[:, 2:] - data[:, :-2] # [R, W-2] + pos = data[:, 1:-1] # [R, W-2] + W2 = vel.shape[1] + if W2 < 2: + return np.zeros((data.shape[0], data.shape[0])) + + vel_z = self._zscore_rows(vel) # [R, W-2] + pos_z = self._zscore_rows(pos) # [R, W-2] + + # A[i,j] = normalised cross-covariance: corr(vel_i, pos_j) + A = (vel_z @ pos_z.T) / max(W2 - 1, 1) # [R, R] + + # Symmetric joint asymmetry + FC = np.abs(A + A.T) / 2.0 + np.fill_diagonal(FC, 1.0) + return np.clip(FC, 0.0, 1.0) + + def dFC(self, time_series, Fs): + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + _, T = time_series.shape + + FCSs, TR_array = [], [] + for l in range(0, T - W_samples + 1, step): + seg = time_series[:, l : l + W_samples] + FCSs.append(self._trac_matrix(seg)) + TR_array.append(int(l + W_samples // 2)) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/volatility_adaptive_edge_hybrid.py b/pydfc/dfc_methods/volatility_adaptive_edge_hybrid.py new file mode 100644 index 0000000..fb345da --- /dev/null +++ b/pydfc/dfc_methods/volatility_adaptive_edge_hybrid.py @@ -0,0 +1,286 @@ +""" +VolatilityAdaptiveEdge_hybrid. + +Hybrid dFC estimator generated from selected threshold_70_filtered_methods.npy +methods. The method combines at least two selected source-method elements and +returns a standard state-free PydFC DFC object. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class VOLATILITY_ADAPTIVE_EDGE_HYBRID(BaseDFCMethod): + """State-free hybrid of VolatilityWeightedFC, AdaptiveExponentialWindow, EdgeCoactivation.""" + + MEASURE_NAME = "VolatilityAdaptiveEdge_hybrid" + COMPONENTS = ['VolatilityWeightedFC', 'AdaptiveExponentialWindow', 'EdgeCoactivation'] + HYBRID_RULE = "volatility_gate" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "min_periods", + "half_life", + "window_length", + "n_overlap", + "nonlinear_gain", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for params_name in self.params_name_lst: + self.params[params_name] = params.get(params_name, None) + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + if self.params["min_periods"] is None: + self.params["min_periods"] = 12 + if self.params["half_life"] is None: + self.params["half_life"] = 30 + if self.params["window_length"] is None: + self.params["window_length"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + if self.params["nonlinear_gain"] is None: + self.params["nonlinear_gain"] = 1.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + def _alpha_from_half_life(self, Fs): + half_life_samples = max(float(self.params["half_life"]) * Fs, 1.0) + return 1.0 - np.exp(np.log(0.5) / half_life_samples) + + @staticmethod + def _corr_from_cov(covariance): + variance = np.diag(covariance) + scale = np.sqrt(np.outer(variance, variance)) + corr = np.divide( + covariance, + scale, + out=np.zeros_like(covariance, dtype=float), + where=scale > 1e-12, + ) + corr[np.isnan(corr)] = 0.0 + corr = (corr + corr.T) / 2.0 + np.fill_diagonal(corr, 1.0) + return np.clip(corr, -1.0, 1.0) + + def _weighted_corr(self, samples, weights): + weights = np.asarray(weights, dtype=float) + weights = np.maximum(weights, 0.0) + if np.sum(weights) <= 1e-12: + weights = np.ones(samples.shape[1], dtype=float) + weights = weights / np.sum(weights) + mean = np.sum(samples * weights[None, :], axis=1, keepdims=True) + centered = samples - mean + covariance = (centered * weights[None, :]) @ centered.T + return self._corr_from_cov(covariance) + + @staticmethod + def _rank_rows(samples): + ranks = np.zeros_like(samples, dtype=float) + for i, row in enumerate(samples): + order = np.argsort(row, kind="mergesort") + rank = np.empty_like(order, dtype=float) + rank[order] = np.arange(row.size, dtype=float) + ranks[i] = rank + return ranks + + def _component_matrix(self, name, data, Fs, tr, state): + n_regions = data.shape[0] + min_periods = int(self.params["min_periods"]) + start = max(0, tr - min_periods + 1) + window = data[:, start : tr + 1] + alpha = self._alpha_from_half_life(Fs) + age = tr - np.arange(tr + 1) + + if name == "SlidingWindow": + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "ExponentialWindow": + return self._weighted_corr(data[:, : tr + 1], alpha * np.power(1.0 - alpha, age)) + if name == "AdaptiveExponentialWindow": + diff = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1))) if tr > 0 else 0.0 + adapt = diff / (diff + np.std(window) + 1e-8) + local_alpha = 0.02 + 0.33 * adapt + local_weights = local_alpha * np.power(1.0 - local_alpha, age) + return self._weighted_corr(data[:, : tr + 1], local_weights) + if name == "ChangepointResetWindow": + if tr > min_periods: + diffs = np.mean(np.abs(np.diff(data[:, max(0, tr - min_periods) : tr + 1], axis=1)), axis=0) + threshold = np.median(diffs) + 2.0 * np.std(diffs) + hits = np.where(diffs > threshold)[0] + if hits.size: + window = data[:, start + hits[-1] + 1 : tr + 1] + return self._weighted_corr(window, np.ones(window.shape[1])) + if name == "DCCConnectivity": + eps = data[:, : tr + 1] - np.mean(data[:, : tr + 1], axis=1, keepdims=True) + lam = 0.94 + sigma2 = np.var(eps, axis=1) + 1e-8 + q = state.get("dcc_q", np.eye(n_regions)) + z = eps[:, -1] / np.sqrt(sigma2) + q_bar = self._weighted_corr(eps, np.ones(eps.shape[1])) + q = 0.10 * q_bar + 0.05 * np.outer(z, z) + 0.85 * q + state["dcc_q"] = q + return self._corr_from_cov(q + (1.0 - lam) * np.diag(sigma2)) + if name == "DifferentialCoactivationFC": + if window.shape[1] < 2: + return np.eye(n_regions) + return self._weighted_corr(np.diff(window, axis=1), np.ones(window.shape[1] - 1)) + if name == "EdgeCoactivation": + mean = np.mean(window, axis=1, keepdims=True) + std = np.std(window, axis=1, keepdims=True) + 1e-8 + z = (data[:, tr : tr + 1] - mean)[:, 0] / std[:, 0] + matrix = np.outer(z, z) + scale = np.sqrt(np.outer(np.diag(matrix) ** 2, np.diag(matrix) ** 2)) + 1e-8 + return self._corr_from_cov(matrix / scale) + if name == "KalmanCovariance": + mean = state.get("kalman_mean", data[:, 0].copy()) + cov = state.get("kalman_cov", np.eye(n_regions)) + innovation = data[:, tr] - mean + mean = mean + alpha * innovation + cov = (1.0 - alpha) * cov + alpha * np.outer(innovation, innovation) + 1e-4 * np.eye(n_regions) + state["kalman_mean"] = mean + state["kalman_cov"] = cov + return self._corr_from_cov(cov) + if name == "MultiscaleWindow": + mats = [] + for scale in (0.5, 1.0, 1.5): + width = max(4, int(min_periods * scale)) + seg = data[:, max(0, tr - width + 1) : tr + 1] + mats.append(self._weighted_corr(seg, np.ones(seg.shape[1]))) + return np.mean(mats, axis=0) + if name == "QuantumMutualInformationFC": + corr = self._weighted_corr(window, np.ones(window.shape[1])) + qmi = -np.log(np.maximum(1.0 - corr**2, 1e-6)) + qmi = qmi / (np.max(qmi) + 1e-8) + return np.sign(corr) * qmi + if name == "RandomFourierDependence": + omega = np.linspace(0.5, 2.5, 16) + phase = np.linspace(0.0, np.pi, 16) + phi = np.cos(window[:, :, None] * omega + phase) + feature = np.mean(phi, axis=1) + feature -= np.mean(feature, axis=1, keepdims=True) + norm = np.linalg.norm(feature, axis=1, keepdims=True) + 1e-8 + feature = feature / norm + matrix = feature @ feature.T + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "ReservoirEchoStateFC": + reservoir = state.get("reservoir", np.zeros(n_regions)) + recurrent = state.get("reservoir_w", np.eye(n_regions) * 0.45) + reservoir = np.tanh(0.55 * data[:, tr] + recurrent @ reservoir) + state["reservoir"] = reservoir + matrix = 0.5 * self._weighted_corr(window, np.ones(window.shape[1])) + 0.5 * np.outer(reservoir, reservoir) + return self._corr_from_cov(matrix) + if name == "RobustSlidingWindow": + ranked = self._rank_rows(window) + return self._weighted_corr(ranked, np.ones(ranked.shape[1])) + if name == "SparseCoactivationCodeFC": + edge = self._component_matrix("EdgeCoactivation", data, Fs, tr, state) + threshold = np.quantile(np.abs(edge), 0.75) + sparse = np.where(np.abs(edge) >= threshold, edge, 0.0) + np.fill_diagonal(sparse, 1.0) + return sparse + if name == "STFTCoherence": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + coeff = spectrum[:, 1:] + cross = np.abs(coeff @ coeff.conj().T) + power = np.sum(np.abs(coeff) ** 2, axis=1) + denom = np.sqrt(np.outer(power, power)) + 1e-8 + matrix = np.real(cross / denom) + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, 0.0, 1.0) + if name == "Time-Freq": + centered = window - np.mean(window, axis=1, keepdims=True) + spectrum = np.fft.rfft(centered, axis=1) + if spectrum.shape[1] <= 1: + return np.eye(n_regions) + phase = spectrum[:, 1:] / (np.abs(spectrum[:, 1:]) + 1e-8) + matrix = np.real(phase @ phase.conj().T) / phase.shape[1] + np.fill_diagonal(matrix, 1.0) + return np.clip(matrix, -1.0, 1.0) + if name == "VolatilityWeightedFC": + residuals = window - np.mean(window, axis=1, keepdims=True) + weights = np.sum(residuals**2, axis=0) + return self._weighted_corr(window, weights) + raise ValueError(f"Unsupported hybrid component: {name}") + + def _combine(self, matrices): + mats = np.array(matrices, dtype=float) + rule = self.HYBRID_RULE + gain = float(self.params["nonlinear_gain"]) + if rule == "ensemble_mean": + combined = np.mean(mats, axis=0) + else: + base = mats[0] + auxiliary = np.mean(mats[1:], axis=0) + contrast = mats[1] - mats[-1] + gate = 1.0 / (1.0 + np.exp(-gain * auxiliary)) + if "residual" in rule: + combined = base + gate * contrast + elif "sparse" in rule: + threshold = np.quantile(np.abs(auxiliary), 0.65) + combined = base * (np.abs(auxiliary) >= threshold) + 0.5 * contrast + elif "kernel" in rule or "information" in rule or "spectral" in rule: + combined = np.sign(base) * np.sqrt(np.abs(base * auxiliary) + 1e-8) + 0.25 * contrast + elif "change" in rule or "derivative" in rule or "volatility" in rule: + combined = (1.0 - gate) * base + gate * (base * auxiliary) + else: + combined = (1.0 - gate) * base + gate * auxiliary + combined = (combined + combined.T) / 2.0 + combined[np.isnan(combined)] = 0.0 + np.fill_diagonal(combined, 1.0) + return np.clip(combined, -1.0, 1.0) + + def dFC(self, time_series, Fs): + min_periods = int(self.params["min_periods"]) + if time_series.shape[1] < min_periods: + raise ValueError("time_series has fewer samples than min_periods.") + state = {} + FCSs = [] + TR_array = [] + for tr in range(min_periods - 1, time_series.shape[1]): + matrices = [ + self._component_matrix(component, time_series, Fs, tr, state) + for component in self.COMPONENTS + ] + FCSs.append(self._combine(matrices)) + TR_array.append(tr) + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert ( + len(time_series.subj_id_lst) == 1 + ), "this function takes only one subject as input." + assert type(time_series) is TIME_SERIES, "time_series must be of TIME_SERIES class." + time_series = self.manipulate_time_series4dFC(time_series) + tic = time.time() + FCSs, TR_array = self.dFC(time_series=time_series.data, Fs=time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/volatility_weighted.py b/pydfc/dfc_methods/volatility_weighted.py new file mode 100644 index 0000000..5203f7b --- /dev/null +++ b/pydfc/dfc_methods/volatility_weighted.py @@ -0,0 +1,118 @@ +""" +Volatility-Weighted FC (VWC) — novel dFC method. + +Standard sliding-window correlation weights every timepoint equally. +This method weights each timepoint by its squared distance from the +window mean (its "volatility" contribution), so moments of high +co-fluctuation amplitude dominate the covariance estimate. +""" + +import time + +import numpy as np + +from ..dfc import DFC +from ..time_series import TIME_SERIES +from .base_dfc_method import BaseDFCMethod + + +class VOLATILITY_WEIGHTED(BaseDFCMethod): + """Sliding-window Pearson correlation with amplitude-deviation weighting. + + Within each window the contribution of timepoint t to the covariance + estimate is weighted by w(t) = ‖x(t) − μ‖², the squared L2 distance + of the multiregion BOLD state from the window mean. High-amplitude + co-fluctuation moments receive proportionally more weight, giving a + covariance estimate that emphasises large, coordinated deviations from + baseline. The weighted covariance is normalised to a correlation matrix. + """ + + MEASURE_NAME = "VolatilityWeightedFC" + + def __init__(self, **params): + self.logs_ = "" + self.TPM = [] + self.FCS_ = [] + self.FCS_fit_time_ = None + self.dFC_assess_time_ = None + self.params_name_lst = [ + "measure_name", + "is_state_based", + "W", + "n_overlap", + "normalization", + "num_select_nodes", + "num_time_point", + "Fs_ratio", + "noise_ratio", + "num_realization", + "session", + ] + self.params = {} + for p in self.params_name_lst: + self.params[p] = params.get(p, None) + + self.params["measure_name"] = self.MEASURE_NAME + self.params["is_state_based"] = False + + if self.params["W"] is None: + self.params["W"] = 30 + if self.params["n_overlap"] is None: + self.params["n_overlap"] = 0.0 + + @property + def measure_name(self): + return self.params["measure_name"] + + @staticmethod + def _weighted_correlation(data): + """Volatility-weighted Pearson correlation for [n_regions, W] data.""" + mu = data.mean(axis=1, keepdims=True) + residuals = data - mu # [R, W] + weights = np.sum(residuals**2, axis=0) # [W] per-TR volatility + w_sum = weights.sum() + if w_sum < 1e-12: + return np.corrcoef(data) + + weights = weights / w_sum # normalise to sum=1 + # Weighted covariance: sum_t w_t * r_i(t) * r_j(t) + wcov = (residuals * weights[np.newaxis, :]) @ residuals.T # [R, R] + # Weighted variance per region + wvar = np.diag(wcov) + denom = np.sqrt(np.outer(wvar, wvar)) + denom = np.where(denom > 1e-12, denom, 1.0) + C = wcov / denom + C[np.isnan(C)] = 0.0 + np.fill_diagonal(C, 1.0) + return np.clip(C, -1.0, 1.0) + + def dFC(self, time_series, Fs): + W_samples = int(self.params["W"] * Fs) + n_overlap = float(self.params["n_overlap"]) + step = max(int((1.0 - n_overlap) * W_samples), 1) + _, T = time_series.shape + + FCSs, TR_array = [], [] + for l in range(0, T - W_samples + 1, step): + seg = time_series[:, l : l + W_samples] + FCSs.append(self._weighted_correlation(seg)) + TR_array.append(int(l + W_samples // 2)) + + return np.array(FCSs), np.array(TR_array) + + def estimate_FCS(self, time_series): + return self + + def estimate_dFC(self, time_series): + assert len(time_series.subj_id_lst) == 1, "one subject per call" + assert type(time_series) is TIME_SERIES, "must be TIME_SERIES" + + time_series = self.manipulate_time_series4dFC(time_series) + + tic = time.time() + FCSs, TR_array = self.dFC(time_series.data, time_series.Fs) + self.set_dFC_assess_time(time.time() - tic) + + dFC = DFC(measure=self) + dFC.set_dFC(FCSs=FCSs, TR_array=TR_array, TS_info=time_series.info_dict) + return dFC diff --git a/pydfc/dfc_methods/windowless.py b/pydfc/dfc_methods/windowless.py index 2c4e722..2f3fa0c 100644 --- a/pydfc/dfc_methods/windowless.py +++ b/pydfc/dfc_methods/windowless.py @@ -36,6 +36,7 @@ class WINDOWLESS(BaseDFCMethod): + MEASURE_NAME = "Windowless" def __init__(self, **params): self.logs_ = "" @@ -65,7 +66,7 @@ def __init__(self, **params): else: self.params[params_name] = None - self.params["measure_name"] = "Windowless" + self.params["measure_name"] = self.MEASURE_NAME self.params["is_state_based"] = True @property diff --git a/pydfc/ml_utils.py b/pydfc/ml_utils.py index f183ae8..38bc429 100644 --- a/pydfc/ml_utils.py +++ b/pydfc/ml_utils.py @@ -1913,7 +1913,23 @@ def process_SB_features(X, measure_name): X_transformed = softmax(-X, tau=tau) # 2) ILR transform X_transformed = ilr_transform(X_transformed) - elif measure_name in ["ContinuousHMM", "DiscreteHMM", "Windowless"]: + elif measure_name in [ + "ContinuousHMM", + "DiscreteHMM", + "Windowless", + # New state-based methods (return probabilistic state assignments) + "PooledKMeansStates", + "MiniBatchKMeansStates", + "GaussianMixtureStates", + "BayesianGaussianMixtureStates", + "NMFStates", + "SpectralStates", + "BirchStates", + "AgglomerativeStates", + "LaggedKMeansStates", + "MarkovSmoothedKMeansStates", + "MarkovSmoothedGMMStates", + ]: X_transformed = ilr_transform(X) return X_transformed diff --git a/pydfc/multi_analysis.py b/pydfc/multi_analysis.py index 437aa2c..fa1a269 100644 --- a/pydfc/multi_analysis.py +++ b/pydfc/multi_analysis.py @@ -124,6 +124,7 @@ def create_measure_obj(self, MEASURES_name_lst, **params): MEASURES_lst = list() for MEASURES_name in MEASURES_name_lst: + measure = None ###### CAP ###### if MEASURES_name == "CAP": @@ -153,6 +154,73 @@ def create_measure_obj(self, MEASURES_name_lst, **params): if MEASURES_name == "DiscreteHMM": measure = HMM_DISC(**params) + ###### EXPONENTIAL WINDOW ###### + if MEASURES_name == "ExponentialWindow": + measure = EXPONENTIAL_WINDOW(**params) + + ###### ADAPTIVE EXPONENTIAL WINDOW ###### + if MEASURES_name == "AdaptiveExponentialWindow": + measure = ADAPTIVE_EXPONENTIAL_WINDOW(**params) + + ###### MULTISCALE WINDOW ###### + if MEASURES_name == "MultiscaleWindow": + measure = MULTISCALE_WINDOW(**params) + + ###### EDGE COACTIVATION ###### + if MEASURES_name == "EdgeCoactivation": + measure = EDGE_COACTIVATION(**params) + + ###### PHASE LOCKING WINDOW ###### + if MEASURES_name == "PhaseLockingWindow": + measure = PHASE_LOCKING_WINDOW(**params) + + ###### DERIVATIVE WEIGHTED WINDOW ###### + if MEASURES_name == "DerivativeWeightedWindow": + measure = DERIVATIVE_WEIGHTED_WINDOW(**params) + + ###### CHANGEPOINT RESET WINDOW ###### + if MEASURES_name == "ChangepointResetWindow": + measure = CHANGEPOINT_RESET_WINDOW(**params) + + ###### KALMAN COVARIANCE ###### + if MEASURES_name == "KalmanCovariance": + measure = KALMAN_COVARIANCE(**params) + + ###### LAGGED MAX CORRELATION ###### + if MEASURES_name == "LaggedMaxCorrelation": + measure = LAGGED_MAX_CORRELATION(**params) + + ###### PRECISION SHRINKAGE WINDOW ###### + if MEASURES_name == "PrecisionShrinkageWindow": + measure = PRECISION_SHRINKAGE_WINDOW(**params) + + ###### RECURRENCE KERNEL DEPENDENCE ###### + if MEASURES_name == "RecurrenceKernelDependence": + measure = RECURRENCE_KERNEL_DEPENDENCE(**params) + + ###### RANDOM FOURIER DEPENDENCE ###### + if MEASURES_name == "RandomFourierDependence": + measure = RANDOM_FOURIER_DEPENDENCE(**params) + + ###### EVENT SYNCHRONIZATION ###### + if MEASURES_name == "EventSynchronization": + measure = EVENT_SYNCHRONIZATION(**params) + + ###### COPULA TAIL DEPENDENCE ###### + if MEASURES_name == "CopulaTailDependence": + measure = COPULA_TAIL_DEPENDENCE(**params) + + ###### OJA SUBSPACE CONNECTIVITY ###### + if MEASURES_name == "OjaSubspaceConnectivity": + measure = OJA_SUBSPACE_CONNECTIVITY(**params) + + ###### GRAPH DIFFUSION COACTIVATION ###### + if MEASURES_name == "GraphDiffusionCoactivation": + measure = GRAPH_DIFFUSION_COACTIVATION(**params) + + if measure is None: + raise ValueError(f"Unknown dFC measure name: {MEASURES_name}") + MEASURES_lst.append(measure) return MEASURES_lst diff --git a/pydfc/multi_analysis_utils.py b/pydfc/multi_analysis_utils.py index 93e1c6a..759f8f0 100644 --- a/pydfc/multi_analysis_utils.py +++ b/pydfc/multi_analysis_utils.py @@ -5,48 +5,81 @@ @author: Mohammad Torabi """ +import importlib +import inspect +import pkgutil +import warnings from copy import deepcopy from joblib import Parallel, delayed +from . import dfc_methods as _dfc_pkg from .dfc_methods import * +# cache for discovered measures: map measure_name -> class +_MEASURE_REGISTRY = None + + +def _build_measure_registry(): + global _MEASURE_REGISTRY + if _MEASURE_REGISTRY is not None: + return _MEASURE_REGISTRY + + registry = {} + try: + for finder in pkgutil.iter_modules(_dfc_pkg.__path__): + mod_name = f"{_dfc_pkg.__name__}.{finder.name}" + try: + module = importlib.import_module(mod_name) + except Exception: + warnings.warn(f"Could not import module {mod_name}; skipping.") + continue + for _, obj in inspect.getmembers(module, inspect.isclass): + try: + # ensure class originates from dfc_methods package + if not obj.__module__.startswith(_dfc_pkg.__name__): + continue + from .dfc_methods.base_dfc_method import BaseDFCMethod + + if not issubclass(obj, BaseDFCMethod) or obj is BaseDFCMethod: + continue + except Exception: + continue + + # class-level method name is required for stable discovery + name = getattr(obj, "MEASURE_NAME", None) + if name: + registry[name] = obj + else: + warnings.warn( + f"{obj.__module__}.{obj.__name__} has no MEASURE_NAME; skipping." + ) + except Exception: + warnings.warn("Failed to iterate dfc_methods package for discovery.") + + _MEASURE_REGISTRY = registry + return _MEASURE_REGISTRY + + ################################# DATA_LOADER functions ###################################### def create_measure_obj(MEASURES_name_lst, **params): + """ + Auto-discover dFC method classes under `pydfc.dfc_methods` and + instantiate them with `**params` based on their `measure_name`. + """ - MEASURES_lst = list() + registry = _build_measure_registry() + MEASURES_lst = [] for MEASURES_name in MEASURES_name_lst: - - ###### CAP ###### - if MEASURES_name == "CAP": - measure = CAP(**params) - - ###### CONTINUOUS HMM ###### - if MEASURES_name == "ContinuousHMM": - measure = HMM_CONT(**params) - - ###### WINDOW_LESS ###### - if MEASURES_name == "Windowless": - measure = WINDOWLESS(**params) - - ###### SLIDING WINDOW ###### - if MEASURES_name == "SlidingWindow": - measure = SLIDING_WINDOW(**params) - - ###### TIME FREQUENCY ###### - if MEASURES_name == "Time-Freq": - measure = TIME_FREQ(**params) - - ###### SLIDING WINDOW + CLUSTERING ###### - if MEASURES_name == "Clustering": - measure = SLIDING_WINDOW_CLUSTR(**params) - - ###### DISCRETE HMM ###### - if MEASURES_name == "DiscreteHMM": - measure = HMM_DISC(**params) - + cls = registry.get(MEASURES_name) + if cls is None: + raise ValueError(f"Unknown dFC measure name: {MEASURES_name}") + try: + measure = cls(**params) + except Exception as e: + raise RuntimeError(f"Failed to instantiate measure {MEASURES_name}: {e}") MEASURES_lst.append(measure) return MEASURES_lst diff --git a/task_dFC/multi_dataset_analysis/helper_functions.py b/task_dFC/multi_dataset_analysis/helper_functions.py index 8f06217..c2f4233 100644 --- a/task_dFC/multi_dataset_analysis/helper_functions.py +++ b/task_dFC/multi_dataset_analysis/helper_functions.py @@ -637,8 +637,11 @@ def figure_dfc_matrices_window_png( cbar_label_size=11, rotate_method_labels=90, method_label_pad=18, # << controls distance between method names and images + method_label_max_chars=14, wspace=None, # << override column spacing if needed (None = auto) ): + import textwrap + import matplotlib as mpl import matplotlib.pyplot as plt import numpy as np @@ -659,6 +662,14 @@ def figure_dfc_matrices_window_png( methods = list(dfc_dict.keys()) R = next(iter(dfc_dict.values())).shape[1] + def _display_method_label(label): + label = str(label).replace("_", " ").replace("-", " ") + if method_label_max_chars is None or len(label) <= method_label_max_chars: + return label + if " " not in label: + return label[: max(1, method_label_max_chars - 3)].rstrip() + "..." + return textwrap.shorten(label, width=method_label_max_chars, placeholder="...") + idxs = _window_indices( trs, window_len=window_len, @@ -732,7 +743,7 @@ def figure_dfc_matrices_window_png( if c == 0: ax.set_ylabel( - m, + _display_method_label(m), rotation=rotate_method_labels, labelpad=method_label_pad, # << tighten/loosen here va="center", diff --git a/task_dFC/multi_dataset_analysis/ml_results.py b/task_dFC/multi_dataset_analysis/ml_results.py index f522886..61e0aca 100644 --- a/task_dFC/multi_dataset_analysis/ml_results.py +++ b/task_dFC/multi_dataset_analysis/ml_results.py @@ -3,6 +3,7 @@ import os import sys +import matplotlib.lines as mlines import matplotlib.pyplot as plt import numpy as np import pandas as pd @@ -44,6 +45,26 @@ SIMULATED_METHOD_MEDIAN_ANNOTATION_THRESHOLD = 80.0 NEUTRAL_COLOR = "#D49B9B" +NON_AIGM_METHODS = frozenset( + [ + "SlidingWindow", + "Time-Freq", + "CAP", + "ContinuousHMM", + "Windowless", + "Clustering", + "DiscreteHMM", + ] +) +_AIGM_COLOR = "#0077B6" +_NON_AIGM_COLOR = "#E63946" +_NON_AIGM_LABEL_COLOR = "#D4721A" +_METRIC_SHORT = { + "Logistic regression balanced accuracy": "LogReg BA", + "SVM balanced accuracy": "SVM BA", + "SI": "SI", +} + def parse_args(): helptext = """ @@ -231,24 +252,23 @@ def style_boxplot(ax, box_edge): line.set_zorder(1) -def overlay_method_means(ax, df_best, lower, upper): +def overlay_method_means(ax, df_best, lower, upper, lw=2.4, halfwidth=0.25): means = df_best.groupby("dFC method", observed=True)["score"].mean() - xticks = ax.get_xticks() - xticklabels = [tick.get_text() for tick in ax.get_xticklabels()] - x_positions = {label: xticks[index] for index, label in enumerate(xticklabels)} + yticks = ax.get_yticks() + yticklabels = [tick.get_text() for tick in ax.get_yticklabels()] + y_positions = {label: yticks[index] for index, label in enumerate(yticklabels)} - halfwidth = 0.1 for method, mean_score in means.items(): - if method not in x_positions or pd.isna(mean_score): + if method not in y_positions or pd.isna(mean_score): continue mean_score = min(upper, max(lower, mean_score)) - x_position = x_positions[method] - ax.hlines( + y_pos = y_positions[method] + ax.vlines( mean_score, - x_position - halfwidth, - x_position + halfwidth, + y_pos - halfwidth, + y_pos + halfwidth, colors="#050505", - lw=2.4, + lw=lw, zorder=3, ) @@ -315,10 +335,11 @@ def extract_pointplot_coordinates(ax, method_order, experiment_order, experiment y_data = np.asarray(line.get_ydata(), dtype=float) coordinates[experiment] = {} for method_index, method in enumerate(method_order): + x_value = x_data[method_index] y_value = y_data[method_index] - if np.isnan(y_value): + if np.isnan(x_value) or np.isnan(y_value): continue - coordinates[experiment][method] = (x_data[method_index], y_value) + coordinates[experiment][method] = (x_value, y_value) return coordinates @@ -408,9 +429,9 @@ def annotate_per_method_quartile( SIMULATED_METHOD_MEDIAN_ANNOTATION_THRESHOLD, metric ) - xticks = ax.get_xticks() - xticklabels = [t.get_text() for t in ax.get_xticklabels()] - method_positions = {lab: xticks[i] for i, lab in enumerate(xticklabels)} + yticks = ax.get_yticks() + yticklabels = [t.get_text() for t in ax.get_yticklabels()] + method_positions = {lab: yticks[i] for i, lab in enumerate(yticklabels)} for method in method_order: method_df = df_best[df_best["dFC method"] == method] @@ -429,7 +450,9 @@ def annotate_per_method_quartile( & (method_df["score"] >= quartile_threshold) ] - method_center = method_positions[method] + method_center = method_positions.get(method) + if method_center is None: + continue for _, row in qualify_rows.iterrows(): experiment = row["experiment"] @@ -440,21 +463,17 @@ def annotate_per_method_quartile( x_value, y_value = point_coordinates[experiment][method] - # Position text left or right based on point position - if x_value < method_center: - ha_align = "right" - x_offset = -10 - else: - ha_align = "left" - x_offset = 10 + # Annotate to the right; position text above/below based on dodge offset + va_align = "bottom" if y_value > method_center else "top" + y_offset = 3 if y_value > method_center else -3 ax.annotate( experiment, xy=(x_value, y_value), - xytext=(x_offset, 0), + xytext=(8, y_offset), textcoords="offset points", - ha=ha_align, - va="center", + ha="left", + va=va_align, fontsize=7, fontweight="bold", color="#1A1A1A", @@ -463,6 +482,47 @@ def annotate_per_method_quartile( ) +def _highlight_nonaigm_labels(ax): + """Color non-AIGM method y-tick labels in a muted orange.""" + for label in ax.get_yticklabels(): + if label.get_text() in NON_AIGM_METHODS: + label.set_color(_NON_AIGM_LABEL_COLOR) + + +def _build_experiment_legend( + ax, experiment_order, neutral_palette, colored_experiments, top_experiments +): + """Add an experiment legend inside the bottom-right of the axis.""" + top_set = set(top_experiments) + handles = [] + for exp in experiment_order: + color = neutral_palette.get(exp, NEUTRAL_COLOR) + marker = TOP_EXPERIMENT_MARKERS[0] if exp in top_set else "o" + ms = 10 if exp in top_set else 7 + handles.append( + mlines.Line2D( + [], + [], + color=color, + marker=marker, + linestyle="", + markersize=ms, + markeredgecolor="#222222", + markeredgewidth=0.8, + label=exp, + ) + ) + ax.legend( + handles=handles, + loc="lower right", + fontsize=9, + frameon=True, + framealpha=0.92, + title="Experiments", + title_fontsize=10, + ) + + def plot_best_pointplot( df_best, method_order, @@ -473,10 +533,15 @@ def plot_best_pointplot( metric, simul_or_real, ): - # Keep the original width scaling so method spacing is unchanged; - # reduce only the height to improve aspect ratio. - plot_width = max(11, 0.6 * len(method_order)) - plot_height = 5.6 + # Sort methods by median score, best at top (ascending → last item at top of y-axis) + method_medians = df_best.groupby("dFC method", observed=True)["score"].median() + method_order_sorted = ( + method_medians.reindex(method_order).sort_values(ascending=True).index.tolist() + ) + + # Fixed width; height scales with number of methods to avoid pixel-limit errors + plot_width = 10 + plot_height = max(8, 0.35 * len(method_order_sorted)) figure, ax = plt.subplots(figsize=(plot_width, plot_height)) color_threshold = convert_threshold_to_score_scale(COLOR_THRESHOLD, metric) @@ -491,7 +556,6 @@ def plot_best_pointplot( colored_experiments = set(top_experiments) label_threshold = -np.inf else: - # Identify experiments with high performance (>= COLOR_THRESHOLD) colored_experiments = get_colored_experiment_mask(df_best, color_threshold) # Create neutral palette: vibrant for high performers, neutral for others @@ -502,15 +566,16 @@ def plot_best_pointplot( box_face = to_rgba("#DE9995", 0.18) box_edge = "#730800" + # Horizontal boxplot: score on x-axis, methods on y-axis sns.boxplot( data=df_best, - x="dFC method", - y="score", - order=method_order, + x="score", + y="dFC method", + order=method_order_sorted, whis=(5, 95), fliersize=0, linewidth=1.0, - width=0.2, + width=0.5, color=box_face, ax=ax, zorder=1, @@ -520,13 +585,13 @@ def plot_best_pointplot( lower, upper = get_pointplot_limits(metric) overlay_method_means(ax, df_best, lower, upper) - # Draw pointplot with neutral palette + # Horizontal pointplot: experiments as hue, dodged vertically sns.pointplot( data=df_best, - x="dFC method", - y="score", + x="score", + y="dFC method", hue="experiment", - order=method_order, + order=method_order_sorted, hue_order=experiment_order, dodge=0.4, errorbar=None, @@ -537,17 +602,15 @@ def plot_best_pointplot( zorder=6, ) finalize_marker_edges(ax) - resize_colored_markers(ax, experiment_order, colored_experiments, method_order) + resize_colored_markers(ax, experiment_order, colored_experiments, method_order_sorted) - # Extract point coordinates from the pointplot point_coordinates = extract_pointplot_coordinates( ax, - method_order, + method_order_sorted, experiment_order, neutral_palette, ) - # Overlay shapes for top 3 experiments using vibrant palette overlay_top_experiment_shapes( ax, df_best, @@ -556,33 +619,34 @@ def plot_best_pointplot( top_experiment_shapes=TOP_EXPERIMENT_SHAPES, ) - # Annotate per-method quartile points annotate_per_method_quartile( ax, df_best, point_coordinates, - method_order, + method_order_sorted, colored_experiments=colored_experiments, metric=metric, simul_or_real=simul_or_real, score_threshold=label_threshold, ) - ax.set_xlabel("dFC method") - ax.set_ylabel(metric) + ax.set_ylabel("dFC method", fontsize=15, fontweight="bold") + ax.set_xlabel(metric, fontsize=15, fontweight="bold") if metric == "SI": - ax.set_ylim(top=1.02) + ax.set_xlim(right=1.02) else: - ax.set_ylim(0.48, 1.02) - ax.yaxis.set_major_formatter(PercentFormatter(xmax=1.0, decimals=0)) - ax.grid(True, axis="y", color="#FFFFFF", alpha=0.85, linewidth=1.1) + ax.set_xlim(0.48, 1.02) + ax.xaxis.set_major_formatter(PercentFormatter(xmax=1.0, decimals=0)) + ax.set_ylim(-0.5, len(method_order_sorted) - 0.5) + ax.grid(True, axis="x", color="#FFFFFF", alpha=0.85, linewidth=1.1) sns.despine(ax=ax, top=True, right=True) - plt.setp(ax.get_xticklabels(), rotation=35, ha="right") + plt.setp(ax.get_yticklabels(), fontweight="bold", fontsize=13) + plt.setp(ax.get_xticklabels(), fontsize=12) + _highlight_nonaigm_labels(ax) if ax.legend_: ax.legend_.remove() - boldify_axes(ax, xlabel="dFC method", ylabel=metric) figure.tight_layout() savefig_pub( @@ -591,6 +655,108 @@ def plot_best_pointplot( plt.close(figure) +def plot_lollipop_pointplot( + df_best, + method_order, + experiment_order, + experiment_palette, + output_root, + embedding, + metric, + simul_or_real, +): + method_means = df_best.groupby("dFC method", observed=True)["score"].mean() + method_order_sorted = ( + method_means.reindex(method_order).sort_values(ascending=True).index.tolist() + ) + + plot_width = 10 + plot_height = max(8, 0.20 * len(method_order_sorted)) + figure, ax = plt.subplots(figsize=(plot_width, plot_height)) + + color_threshold = convert_threshold_to_score_scale(COLOR_THRESHOLD, metric) + top_experiments = get_top_experiments_by_mean(df_best, TOP_EXPERIMENT_SHAPES) + + if metric == "SI": + colored_experiments = set(top_experiments) + else: + colored_experiments = get_colored_experiment_mask(df_best, color_threshold) + + box_edge = "#730800" + + # Lollipop: 5th–95th percentile range line + median dot per method + method_groups = df_best.groupby("dFC method", observed=True)["score"] + for i, method in enumerate(method_order_sorted): + if method not in method_groups.groups: + continue + vals = method_groups.get_group(method).dropna().values + if len(vals) == 0: + continue + lo, med, hi = np.nanpercentile(vals, [5, 50, 95]) + ax.hlines(i, lo, hi, colors=box_edge, lw=1.4, alpha=0.45, zorder=1) + ax.scatter(med, i, color=box_edge, s=28, zorder=2, linewidths=0) + + lower, upper = get_pointplot_limits(metric) + + sns.pointplot( + data=df_best, + x="score", + y="dFC method", + hue="experiment", + order=method_order_sorted, + hue_order=experiment_order, + dodge=0.35, + errorbar=None, + linestyles="", + markers="o", + palette=experiment_palette, + ax=ax, + zorder=6, + ) + finalize_marker_edges(ax) + # Only starred (top) experiments get large markers; all others stay small + resize_colored_markers( + ax, experiment_order, set(top_experiments), method_order_sorted + ) + + # Called after pointplot so yticks are populated + overlay_method_means(ax, df_best, lower, upper, lw=2.8, halfwidth=0.30) + + point_coordinates = extract_pointplot_coordinates( + ax, method_order_sorted, experiment_order, experiment_palette + ) + overlay_top_experiment_shapes( + ax, + df_best, + point_coordinates, + experiment_palette, + top_experiment_shapes=TOP_EXPERIMENT_SHAPES, + ) + + ax.set_ylabel("dFC method", fontsize=15, fontweight="bold") + ax.set_xlabel(metric, fontsize=15, fontweight="bold") + if metric == "SI": + ax.set_xlim(right=1.02) + else: + ax.set_xlim(0.48, 1.02) + ax.xaxis.set_major_formatter(PercentFormatter(xmax=1.0, decimals=0)) + ax.set_ylim(-0.5, len(method_order_sorted) - 0.5) + ax.grid(True, axis="x", color="#FFFFFF", alpha=0.85, linewidth=1.1) + sns.despine(ax=ax, top=True, right=True) + plt.setp(ax.get_yticklabels(), fontweight="bold", fontsize=11) + plt.setp(ax.get_xticklabels(), fontsize=12) + _highlight_nonaigm_labels(ax) + _build_experiment_legend( + ax, experiment_order, experiment_palette, colored_experiments, top_experiments + ) + + figure.tight_layout() + savefig_pub( + f"{output_root}/ML_scores_{embedding}_{metric}_{LEVEL}_{simul_or_real}_best_lollipop.png" + ) + plt.close(figure) + + def plot_best_heatmap( df_best, method_order, @@ -614,22 +780,24 @@ def plot_best_heatmap( ) col_order = [method for method in method_order if method in matrix_best.columns] - if simul_or_real == "real": - width = max(10, 0.65 * len(col_order)) - height = max(6.0, 0.30 * len(matrix_best.index)) - else: - width = max(11, 11 / 7 * len(col_order)) - height = max(7.0, 0.35 * len(matrix_best.index)) + # Transpose: methods on rows, experiments on columns — keeps width bounded + matrix_plot = matrix_best.loc[:, col_order].T + annot_plot = annot_best.loc[:, col_order].T + + n_methods = len(matrix_plot.index) + n_exps = len(matrix_plot.columns) + width = max(8.0, 1.2 * n_exps) + height = max(8.0, 0.30 * n_methods) figure, ax = plt.subplots(figsize=(width, height)) vmin, vmax, center = get_heatmap_limits(metric) heatmap = sns.heatmap( - matrix_best.loc[:, col_order], + matrix_plot, vmin=vmin, vmax=vmax, center=center, cmap="coolwarm", - annot=annot_best.loc[:, col_order], + annot=annot_plot, fmt="", annot_kws={"fontsize": 9, "fontweight": "bold", "linespacing": 1.15}, cbar_kws={"shrink": 0.7, "pad": 0.02}, @@ -639,9 +807,9 @@ def plot_best_heatmap( colorbar.set_label(metric, fontsize=10, fontweight="bold") colorbar.ax.tick_params(labelsize=9) - boldify_axes(ax, xlabel="dFC method", ylabel="Experiment", rotate_xticks=35) - ax.set_xlabel("dFC method") - ax.set_ylabel("Experiment") + boldify_axes(ax, xlabel="Experiment", ylabel="dFC method", rotate_xticks=35) + ax.set_xlabel("Experiment") + ax.set_ylabel("dFC method") plt.setp(ax.get_xticklabels(), fontweight="bold", rotation=35, ha="right") plt.setp(ax.get_yticklabels(), fontweight="bold") sns.despine(ax=ax, top=True, right=True) @@ -699,18 +867,25 @@ def plot_across_heatmap( task_to_experiment, ) col_order = [method for method in method_order if method in matrix_across.columns] - width = max(9.0, 11 / 7 * len(col_order)) - height = max(7.0, 7 / 20 * len(matrix_across.index)) + + # Transpose: methods on rows, experiments on columns — keeps width bounded + matrix_plot = matrix_across.loc[:, col_order].T + annot_plot = annot_across.loc[:, col_order].T + + n_methods = len(matrix_plot.index) + n_exps = len(matrix_plot.columns) + width = max(8.0, 1.5 * n_exps) + height = max(8.0, 0.35 * n_methods) figure, ax = plt.subplots(figsize=(width, height)) vmin, vmax, center = get_heatmap_limits(metric) heatmap = sns.heatmap( - matrix_across.loc[:, col_order], + matrix_plot, vmin=vmin, vmax=vmax, center=center, cmap="coolwarm", - annot=annot_across.loc[:, col_order], + annot=annot_plot, fmt="", annot_kws={"fontsize": 9, "fontweight": "bold", "linespacing": 1.15}, cbar_kws={"shrink": 0.7, "pad": 0.02}, @@ -719,9 +894,9 @@ def plot_across_heatmap( colorbar = heatmap.collections[0].colorbar colorbar.set_label(metric, fontsize=10, fontweight="bold") - boldify_axes(ax, xlabel="dFC method", ylabel="Experiment", rotate_xticks=35) - ax.set_xlabel("dFC method") - ax.set_ylabel("Experiment") + boldify_axes(ax, xlabel="Experiment", ylabel="dFC method", rotate_xticks=35) + ax.set_xlabel("Experiment") + ax.set_ylabel("dFC method") plt.setp(ax.get_xticklabels(), fontweight="bold", rotation=35, ha="right") plt.setp(ax.get_yticklabels(), fontweight="bold") sns.despine(ax=ax, top=True, right=True) @@ -732,6 +907,106 @@ def plot_across_heatmap( plt.close(figure) +def plot_aigm_comparison( + df_best, + output_root, + embedding, + metric, + simul_or_real, +): + """ + Horizontal boxplot + scatter comparing AIGM vs non-AIGM method scores. + One point per (method × experiment). Matches plot_best_pointplot style. + """ + from scipy.stats import mannwhitneyu + + df_best = df_best.copy() + df_best["group"] = df_best["dFC method"].apply( + lambda m: "Non-AIGM" if m in NON_AIGM_METHODS else "AIGM" + ) + + group_order = ["AIGM", "Non-AIGM"] + n_aigm = df_best[df_best["group"] == "AIGM"]["dFC method"].nunique() + n_non_aigm = df_best[df_best["group"] == "Non-AIGM"]["dFC method"].nunique() + group_labels = [f"AIGM\n(n={n_aigm} methods)", f"Non-AIGM\n(n={n_non_aigm} methods)"] + df_best["group_label"] = df_best["group"].map(dict(zip(group_order, group_labels))) + + fig, ax = plt.subplots(figsize=(9, 4)) + + box_face = to_rgba("#DE9995", 0.18) + box_edge = "#730800" + + sns.boxplot( + data=df_best, + x="score", + y="group_label", + order=group_labels, + whis=(5, 95), + fliersize=0, + linewidth=1.0, + width=0.5, + color=box_face, + ax=ax, + zorder=1, + ) + style_boxplot(ax, box_edge) + + # One point per (method × experiment), colored by group + rng = np.random.default_rng(42) + group_colors = {"AIGM": _AIGM_COLOR, "Non-AIGM": _NON_AIGM_COLOR} + for i, (group, label) in enumerate(zip(group_order, group_labels)): + vals = df_best[df_best["group"] == group]["score"].dropna().values + y_jit = i + rng.uniform(-0.18, 0.18, len(vals)) + ax.scatter( + vals, + y_jit, + color=group_colors[group], + alpha=0.35, + s=18, + linewidths=0.0, + zorder=4, + ) + + # Mann-Whitney p-value + aigm_vals = df_best[df_best["group"] == "AIGM"]["score"].dropna().values + non_aigm_vals = df_best[df_best["group"] == "Non-AIGM"]["score"].dropna().values + if len(aigm_vals) >= 2 and len(non_aigm_vals) >= 2: + _, pval = mannwhitneyu(aigm_vals, non_aigm_vals, alternative="two-sided") + pstr = "p<0.001" if pval < 0.001 else f"p={pval:.3f}" + ax.text( + 0.97, + 0.97, + pstr, + transform=ax.transAxes, + ha="right", + va="top", + fontsize=11, + fontweight="bold", + bbox=dict(boxstyle="round,pad=0.3", fc="white", ec="#BBBBBB", alpha=0.9), + ) + + lower, upper = get_pointplot_limits(metric) + if metric == "SI": + ax.set_xlim(right=1.02) + else: + ax.set_xlim(lower, 1.02) + ax.xaxis.set_major_formatter(PercentFormatter(xmax=1.0, decimals=0)) + + ax.set_xlabel(metric, fontsize=15, fontweight="bold") + ax.set_ylabel("", fontsize=15) + ax.set_ylim(-0.5, len(group_order) - 0.5) + ax.grid(True, axis="x", color="#FFFFFF", alpha=0.85, linewidth=1.1) + sns.despine(ax=ax, top=True, right=True) + plt.setp(ax.get_yticklabels(), fontweight="bold", fontsize=13) + plt.setp(ax.get_xticklabels(), fontsize=12) + + fig.tight_layout() + savefig_pub( + f"{output_root}/ML_scores_AIGM_vs_nonAIGM_{embedding}_{metric}_{LEVEL}_{simul_or_real}.png" + ) + plt.close(fig) + + def generate_all_plots(all_ml_scores, tasks_to_include, output_root, simul_or_real): sns.set_context("paper", font_scale=1.0, rc={"lines.linewidth": 1.2}) sns.set_style("darkgrid") @@ -783,6 +1058,23 @@ def generate_all_plots(all_ml_scores, tasks_to_include, output_root, simul_or_re metric, simul_or_real, ) + plot_aigm_comparison( + df_best, + output_root, + embedding, + metric, + simul_or_real, + ) + plot_lollipop_pointplot( + df_best, + method_order, + experiment_order, + experiment_palette, + output_root, + embedding, + metric, + simul_or_real, + ) def main(): diff --git a/task_dFC/run_scripts_slurm/achillev@narval.alliancecan b/task_dFC/run_scripts_slurm/achillev@narval.alliancecan new file mode 100644 index 0000000..3247942 --- /dev/null +++ b/task_dFC/run_scripts_slurm/achillev@narval.alliancecan @@ -0,0 +1,32 @@ +#!/bin/bash +# +#SBATCH --job-name=assess_dfc_job +#SBATCH --output=logs/dfc_out_%A_%a.txt +#SBATCH --error=logs/dfc_err_%A_%a.txt +#SBATCH --time=24:00:00 +#SBATCH --mem=32G +#SBATCH --requeue + +SUBJECT_LIST="./subj_list.txt" +DATASET_INFO="./dataset_info.json" +METHODS_CONFIG="./methods_config.json" + +echo "Number subjects found: $(cat $SUBJECT_LIST | wc -l)" + +SUBJECT_ID=$(sed -n "${SLURM_ARRAY_TASK_ID}p" $SUBJECT_LIST) +echo "Subject ID: $SUBJECT_ID" + +# ---- Cluster configuration (set these for your system) ---- +VENV_PATH="/home/achillev/projects/def-jbpoline/achillev/pydfc_env/bin/activate" +PYDFC_CODE_DIR="/home/achillev/scratch/Git_repo" +# ----------------------------------------------------------- + +# Activate virtual environment +source "$VENV_PATH" + +python "$PYDFC_CODE_DIR/task_dFC/dFC_assessment.py" \ +--dataset_info $DATASET_INFO \ +--methods_config $METHODS_CONFIG \ +--participant_id $SUBJECT_ID + +deactivate diff --git a/task_dFC/run_scripts_slurm/dataset_info.json b/task_dFC/run_scripts_slurm/dataset_info.json index b01dbda..ca61229 100644 --- a/task_dFC/run_scripts_slurm/dataset_info.json +++ b/task_dFC/run_scripts_slurm/dataset_info.json @@ -1,27 +1,36 @@ { - "dataset" : "", - "main_root" : "/path/to/your/data/{dataset}", - "bids_root" : "/path/to/your/data/{dataset}/bids", - "fmriprep_root" : "/path/to/your/data/{dataset}/derivatives/fmriprep/23.1.3/output", + "dataset" : "ds003465", + "main_root" : "/home/achillev/scratch/ds003465/{dataset}", + "bids_root" : null, + "fmriprep_root" : null, "roi_root" : "{main_root}/derivatives/ROI_timeseries", "fitted_measures_root" : "{main_root}/derivatives/fitted_MEASURES", "dFC_root" : "{main_root}/derivatives/dFC_assessed", "ML_root" : "{main_root}/derivatives/ML", "reports_root" : "{main_root}/derivatives/reports", "bold_suffix" : "_space-MNI152NLin2009cAsym_res-2_desc-preproc_bold.nii.gz", - "SESSIONS" : [ - "ses-1" - ], - "TASKS" : [ - "task-A" - ], - "RUNS" : { - "task-A": ["run-01", "run-02", "run-03", "run-04", "run-05", "run-06"] - }, - "trial_type_label" : { - "task-A": "trial_type" - }, - "rest_labels" : { - "task-A": ["rest", "Rest"] - } + "SESSIONS" : [ + "ses-wave1bas" + ], + "TASKS" : [ + "task-Axcpt", "task-Cuedts", "task-Stern", "task-Stroop" + ], + "RUNS" : { + "task-Axcpt": ["run-1", "run-2"], + "task-Cuedts": ["run-1", "run-2"], + "task-Stern": ["run-1", "run-2"], + "task-Stroop": ["run-1", "run-2"] + }, + "trial_type_label" : { + "task-Axcpt": "trial_type", + "task-Cuedts": "trial_type", + "task-Stern": "trial_type", + "task-Stroop": "trial_type" + }, + "rest_labels" : { + "task-Axcpt": ["rest", "Rest"], + "task-Cuedts": ["rest", "Rest"], + "task-Stern": ["rest", "Rest"], + "task-Stroop": ["rest", "Rest"] + } } diff --git a/task_dFC/run_scripts_slurm/methods_config.json b/task_dFC/run_scripts_slurm/methods_config.json index 722b4ff..ea411eb 100644 --- a/task_dFC/run_scripts_slurm/methods_config.json +++ b/task_dFC/run_scripts_slurm/methods_config.json @@ -1,40 +1,87 @@ { - "params_methods" : { - "W": 44, - "n_overlap": 1.0, - "sw_method": "pear_corr", - "tapered_window": true, - "TF_method": "WTC", - "clstr_base_measure": "SlidingWindow", - "clstr_distance": "manhattan", - "hmm_iter": 20, - "dhmm_obs_state_ratio": 0.666, - "n_states": 5, - "n_subj_clstrs": 10, - "verbose": 0, - "n_jobs_sw": 8, - "backend_sw": "threading", - "n_jobs_tf": 2, - "backend_tf": "loky", - "n_jobs_swc": null, - "backend_swc": null, - "normalization": true, - "num_subj": null, - "num_time_point": null - }, - "MEASURES_name_lst" : [ - "SlidingWindow", - "Time-Freq", - "CAP", - "ContinuousHMM", - "Windowless", - "Clustering", - "DiscreteHMM" - ], - "alter_hparams" : [], - "params_multi_analysis" : { - "n_jobs": 8, - "verbose": 0, - "backend": "loky" - } + "params_methods": { + "W": 44, + "n_overlap": 1.0, + "sw_method": "pear_corr", + "tapered_window": true, + "window_std": null, + "TF_method": "WTC", + "clstr_base_measure": "SlidingWindow", + "clstr_distance": "manhattan", + "hmm_iter": 20, + "dhmm_obs_state_ratio": 0.666, + "n_states": 5, + "n_subj_clstrs": 10, + "verbose": 0, + "n_jobs_sw": 16, + "backend_sw": "threading", + "n_jobs_tf": 16, + "backend_tf": "loky", + "n_jobs_swc": null, + "backend_swc": null, + "normalization": true, + "num_subj": null, + "num_time_point": null, + "half_life": 30, + "min_periods": 10, + "alpha": null, + "alpha_min": 0.02, + "alpha_max": 0.35, + "windows": [15, 30, 60], + "process_noise": 0.0001, + "max_lag": 2, + "kernel_width": 1.5, + "n_random_features": 128, + "event_quantile": 0.85, + "event_decay": 0.97, + "tail_quantile": 0.8, + "n_components": 10, + "learning_rate": 0.03, + "diffusion_rate": 0.2, + "instantaneous_weight": 0.15, + "assignment_temperature": 1.0, + "transition_smoothing": 1.0, + "lag": 2, + "n_neighbors": 15, + "n_atoms": 20, + "dict_alpha": 0.1 + }, + "MEASURES_name_lst": [ + "ADAPTIVE_DCC_RANDOM_HYBRID_ENSEMBLE", + "ADAPTIVE_EDGE_KALMAN_HYBRID", + "ADAPTIVE_QUANTUM_RESERVOIR_HYBRID", + "ADAPTIVE_RANDOM_SPARSE_HYBRID", + "ADAPTIVE_ROBUST_DIFFERENTIAL_HYBRID_ENSEMBLE", + "CHANGEPOINT_MULTISCALE_EXP_HYBRID_ENSEMBLE", + "CHANGEPOINT_ROBUST_EXP_HYBRID", + "DCC_CHANGEPOINT_SPARSE_HYBRID", + "DCC_EDGE_EXPONENTIAL_HYBRID", + "DCC_KALMAN_VOLATILITY_HYBRID_ENSEMBLE", + "DIFFERENTIAL_EDGE_KALMAN_HYBRID", + "EDGE_KALMAN_EXP_HYBRID_ENSEMBLE", + "EDGE_RANDOM_RESERVOIR_HYBRID_ENSEMBLE", + "EDGE_STFT_QUANTUM_HYBRID", + "KALMAN_RESERVOIR_MULTISCALE_HYBRID", + "MULTISCALE_DCC_KALMAN_HYBRID", + "QUANTUM_MULTISCALE_KALMAN_HYBRID_ENSEMBLE", + "QUANTUM_RANDOM_EXP_HYBRID", + "RANDOM_FOURIER_KALMAN_HYBRID", + "RESERVOIR_CHANGEPOINT_ROBUST_HYBRID_ENSEMBLE", + "RESERVOIR_EDGE_DCC_HYBRID", + "ROBUST_DIFFERENTIAL_STFT_HYBRID", + "ROBUST_SPARSE_EDGE_HYBRID", + "ROBUST_VOLATILITY_SLIDING_HYBRID_ENSEMBLE", + "SLIDING_VOLATILITY_DCC_HYBRID", + "SPARSE_DCC_EXP_HYBRID_ENSEMBLE", + "STFT_EDGE_ADAPTIVE_HYBRID_ENSEMBLE", + "STFT_EXP_KALMAN_HYBRID", + "STFT_QUANTUM_SPARSE_HYBRID_ENSEMBLE", + "VOLATILITY_ADAPTIVE_EDGE_HYBRID" + ], + "alter_hparams": [], + "params_multi_analysis": { + "n_jobs": 8, + "verbose": 0, + "backend": "loky" + } } diff --git a/task_dFC/run_scripts_slurm/multi_dataset_info.json b/task_dFC/run_scripts_slurm/multi_dataset_info.json index de0cf2b..e0411df 100644 --- a/task_dFC/run_scripts_slurm/multi_dataset_info.json +++ b/task_dFC/run_scripts_slurm/multi_dataset_info.json @@ -1,38 +1,22 @@ { - "output_root": "/path/to/your/data/multi_dataset_analysis/results", + "output_root": "/home/achillev/scratch/ds003465/multi_dataset_analysis/results", "real_data": { - "main_root": "/path/to/your/data/openneuro", + "main_root": "/home/achillev/scratch/ds003465/", "DATASETS": [ - "ds001242", "ds002236", "ds002647", - "ds002843", "ds002994", - "ds003465", "ds003612", "ds003823", - "ds004044", "ds004349", "ds004359", - "ds004556", "ds004746", "ds004791", - "ds004848", "ds005038" + "ds003465" ], "TASKS_to_include": [ - "task-arithmetic", "task-AudSem", "task-Axcpt", - "task-Cuedts", "task-emotionRegulation", "task-execution","task-expo", - "task-fearlearning", "task-feedback", "task-fribBids", "task-IHG", - "task-imagery", "task-itc", "task-localiser", "task-Localizer", - "task-matching", "task-motor", "task-paingen", "task-ppalocalizer", - "task-recall", "task-risk", "task-ST", "task-Stern", - "task-Stroop", "task-VisRhyme", "task-VisSem", "task-VisSpell", - "task-vswm" + "task-Axcpt", + "task-Cuedts", + "task-Stern", + "task-Stroop" ] }, "simulated_data": { - "main_root": "/path/to/your/data/simulated", + "main_root": "/home/achillev/scratch/ds003465/", "DATASETS": [ - "ds000001", "ds000002", "ds000003", "ds000004", "ds000005", "ds000006" ], "TASKS_to_include": [ - "task-Axcpt", "task-Cuedts", "task-Stern", "task-Stroop", - "task-lowFreqLongRest", "task-lowFreqShortRest", "task-lowFreqShortTask", - "task-imagery", "task-execution", - "task-itc", "task-risk", - "task-Localizer", - "task-ppalocalizer" ] } } diff --git a/task_dFC/run_scripts_slurm/run_FCS.sh b/task_dFC/run_scripts_slurm/run_FCS.sh index a0d27bf..d6d3a30 100644 --- a/task_dFC/run_scripts_slurm/run_FCS.sh +++ b/task_dFC/run_scripts_slurm/run_FCS.sh @@ -1,11 +1,12 @@ -#!/bin/sh +#!/bin/bash # -#SBATCH --job-name=fit_fcs_job # Optional: Name of your job -#SBATCH --output=logs/fcs_out.txt # Standard output log -#SBATCH --error=logs/fcs_err.txt # Standard error log -#SBATCH --time=7-00:00:00 # Walltime for each task (7 days) -#SBATCH --cpus-per-task=8 # Number of CPU cores per task -#SBATCH --mem=64G # Memory request per node +#SBATCH --job-name=fit_fcs_job +#SBATCH --output=logs/fcs_out_%A_%a.txt +#SBATCH --error=logs/fcs_err_%A_%a.txt +#SBATCH --time=7-00:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=64G +#SBATCH --requeue DATASET_INFO="./dataset_info.json" METHODS_CONFIG="./methods_config.json" @@ -16,8 +17,8 @@ export OPENBLAS_NUM_THREADS=1 export NUMEXPR_NUM_THREADS=1 # ---- Cluster configuration (set these for your system) ---- -VENV_PATH="/path/to/your/venv/bin/activate" -PYDFC_CODE_DIR="/path/to/pydfc" +VENV_PATH="/home/achillev/projects/def-jbpoline/achillev/pydfc_env/bin/activate" +PYDFC_CODE_DIR="/home/achillev/scratch/Git_repo" # ----------------------------------------------------------- # Activate virtual environment diff --git a/task_dFC/run_scripts_slurm/run_ML.sh b/task_dFC/run_scripts_slurm/run_ML.sh index 623d298..b16c2a2 100644 --- a/task_dFC/run_scripts_slurm/run_ML.sh +++ b/task_dFC/run_scripts_slurm/run_ML.sh @@ -1,15 +1,17 @@ -#!/bin/sh +#!/bin/bash # -#SBATCH --cpus-per-task=8 # Number of CPU cores per task -#SBATCH --output=logs/ML_out.txt # Standard output log -#SBATCH --error=logs/ML_err.txt # Standard error log -#SBATCH --mem=128G # Memory request per node +#SBATCH --cpus-per-task=8 +#SBATCH --output=logs/ML_out_%A_%a.txt # %A = array job ID, %a = task ID +#SBATCH --error=logs/ML_err_%A_%a.txt +#SBATCH --time=24:00:00 +#SBATCH --mem=128G +#SBATCH --requeue DATASET_INFO="./dataset_info.json" # ---- Cluster configuration (set these for your system) ---- -VENV_PATH="/path/to/your/venv/bin/activate" -PYDFC_CODE_DIR="/path/to/pydfc" +VENV_PATH="/home/achillev/projects/def-jbpoline/achillev/pydfc_env/bin/activate" +PYDFC_CODE_DIR="/home/achillev/scratch/Git_repo" # ----------------------------------------------------------- # Activate virtual environment diff --git a/task_dFC/run_scripts_slurm/run_across_dataset_analysis.sh b/task_dFC/run_scripts_slurm/run_across_dataset_analysis.sh index 1d150bb..23d9169 100644 --- a/task_dFC/run_scripts_slurm/run_across_dataset_analysis.sh +++ b/task_dFC/run_scripts_slurm/run_across_dataset_analysis.sh @@ -9,11 +9,12 @@ # #SBATCH --chdir=/path/to/multi_dataset_analysis/codes # ---- Cluster configuration (set these for your system) ---- -VENV_PATH="/path/to/your/venv/bin/activate" -PYDFC_CODE_DIR="/path/to/pydfc" +VENV_PATH="/home/achillev/projects/def-jbpoline/achillev/pydfc_env/bin/activate" +PYDFC_CODE_DIR="/home/achillev/scratch/Git_repo" # ----------------------------------------------------------- set -euo pipefail +trap 'echo "ERROR: Script failed at line $LINENO with exit code $?" >&2' ERR mkdir -p logs source "$VENV_PATH" diff --git a/task_dFC/run_scripts_slurm/run_dFC.sh b/task_dFC/run_scripts_slurm/run_dFC.sh index f8ec43e..3247942 100644 --- a/task_dFC/run_scripts_slurm/run_dFC.sh +++ b/task_dFC/run_scripts_slurm/run_dFC.sh @@ -1,23 +1,24 @@ -#!/bin/sh +#!/bin/bash # -#SBATCH --job-name=assess_dfc_job # Optional: Name of your job -#SBATCH --output=logs/dfc_out.txt # Standard output log -#SBATCH --error=logs/dfc_err.txt # Standard error log -#SBATCH --time=24:00:00 # Walltime for each task (24 hours) -#SBATCH --mem=32G # Memory request per node +#SBATCH --job-name=assess_dfc_job +#SBATCH --output=logs/dfc_out_%A_%a.txt +#SBATCH --error=logs/dfc_err_%A_%a.txt +#SBATCH --time=24:00:00 +#SBATCH --mem=32G +#SBATCH --requeue SUBJECT_LIST="./subj_list.txt" DATASET_INFO="./dataset_info.json" METHODS_CONFIG="./methods_config.json" -echo "Number subjects found: `cat $SUBJECT_LIST | wc -l`" +echo "Number subjects found: $(cat $SUBJECT_LIST | wc -l)" -SUBJECT_ID=`sed -n "${SLURM_ARRAY_TASK_ID}p" $SUBJECT_LIST` +SUBJECT_ID=$(sed -n "${SLURM_ARRAY_TASK_ID}p" $SUBJECT_LIST) echo "Subject ID: $SUBJECT_ID" # ---- Cluster configuration (set these for your system) ---- -VENV_PATH="/path/to/your/venv/bin/activate" -PYDFC_CODE_DIR="/path/to/pydfc" +VENV_PATH="/home/achillev/projects/def-jbpoline/achillev/pydfc_env/bin/activate" +PYDFC_CODE_DIR="/home/achillev/scratch/Git_repo" # ----------------------------------------------------------- # Activate virtual environment diff --git a/task_dFC/run_scripts_slurm/run_report.sh b/task_dFC/run_scripts_slurm/run_report.sh index 12c6ebc..0ee92f1 100644 --- a/task_dFC/run_scripts_slurm/run_report.sh +++ b/task_dFC/run_scripts_slurm/run_report.sh @@ -10,8 +10,8 @@ DATASET_INFO="./dataset_info.json" SUBJ_LIST="./subj_list.txt" # ---- Cluster configuration (set these for your system) ---- -VENV_PATH="/path/to/your/venv/bin/activate" -PYDFC_CODE_DIR="/path/to/pydfc" +VENV_PATH="/home/achillev/projects/def-jbpoline/achillev/pydfc_env/bin/activate" +PYDFC_CODE_DIR="/home/achillev/scratch/Git_repo" # ----------------------------------------------------------- # Activate virtual environment diff --git a/tests/test_validation/QUICKSTART.md b/tests/test_validation/QUICKSTART.md new file mode 100644 index 0000000..f4701bd --- /dev/null +++ b/tests/test_validation/QUICKSTART.md @@ -0,0 +1,212 @@ +# DFC Validation Framework - Quick Start Guide + +## 5-Minute Setup + +### Installation + +The validation framework requires only standard dependencies already in the pydfc environment: + +```bash +# Required (usually already installed): +pip install numpy scipy scikit-learn joblib +``` + +No additional installation needed! The framework is self-contained in `/tests/test_validation/`. + +## Quick Examples + +### 1. Run Default Validation (30 seconds) + +```bash +cd /path/to/dfc/repo/tests/test_validation +python validate_dfc.py +``` + +This generates synthetic data and tests all available methods with default settings. + +### 1b. List Methods Without Running + +```bash +python -m tests.test_validation.validate_dfc --list-methods +``` + +This prints registered method IDs, aliases, and availability reasons. + +### 2. Run with Custom Settings (1 minute) + +```bash +python validate_dfc.py \ + --n-subjects 50 \ + --n-regions 100 \ + --n-timepoints 600 \ + --pass-threshold 0.90 \ + --output-dir ./my_results +``` + +### 3. Test Specific Methods Only + +```bash +python validate_dfc.py --methods SlidingWindow_W30 DummyMethod +``` + +Default selection note: +- If `--methods` is omitted, only runnable methods in the current environment are selected. + +### 4. Run from Python + +```python +from test_validation import ( + SyntheticDataGenerator, + ZeroAndPerfectCorrTest, + StepChangeTest, + ValidationRunner, + Reporter, + DummyMethod, +) + +# Generate synthetic data +gen = SyntheticDataGenerator(n_subjects=50, n_regions=100, n_timepoints=600) +timeseries, ground_truth = gen.generate() + +# Create tests +tests = [ZeroAndPerfectCorrTest(), StepChangeTest()] + +# Run validation +runner = ValidationRunner() +results = runner.run( + methods={"DummyMethod": DummyMethod()}, + test_cases=tests, + timeseries=timeseries, + ground_truth=ground_truth, +) + +# Report results +reporter = Reporter() +reporter.generate_report(results) +``` + +## Adding Your Method + +### Step 1: Create a Wrapper + +```python +from test_validation import DFCMethodWrapper + +class MyMethodWrapper(DFCMethodWrapper): + def __init__(self, param1=value1): + super().__init__(name="MyMethod_param1") + self.method = MyDFCClass(param1=param1) + + def run(self, timeseries): + """ + Input: timeseries [n_subjects, n_timepoints, n_regions] + Output: dfc_output [n_subjects, n_timepoints, n_regions, n_regions] + """ + n_subjects, n_timepoints, n_regions = timeseries.shape + dfc_output = np.zeros((n_subjects, n_timepoints, n_regions, n_regions)) + + for subj in range(n_subjects): + # Run your method on subject data + result = self.method.run(timeseries[subj]) + # Store result (ensure shape is [n_timepoints, n_regions, n_regions]) + dfc_output[subj] = result + + return dfc_output +``` + +### Step 2: Register Your Method + +Add to `dfc_method_wrappers.py` in the `get_available_methods()` function: + +```python +def get_available_methods(): + methods = { + "DummyMethod": DummyMethod(), + "SlidingWindow_W30": SlidingWindowWrapper(W=30), + "MyMethod_param1": MyMethodWrapper(param1=value1), # Add this + } + return methods +``` + +### Step 3: Test Your Method + +```bash +python validate_dfc.py --methods MyMethod_param1 +``` + +## Understanding Results + +### Summary Table + +``` +Method | ZeroAndPerfectCorr | StepChange | TOTAL +────────────────┼────────────────────┼────────────┼────── +MyMethod_param1 | PASS (0.92) | PASS (0.89)| 2/2 +``` + +- **PASS**: Score > pass_threshold (default 0.90) +- **FAIL**: Score ≤ pass_threshold +- **Score**: Rank-biserial correlation [-1, 1], higher is better + +### Failure Analysis + +If your method fails: + +1. **Check per-subject scores:** High variance suggests numerical instability +2. **Verify output shape:** Must be `[n_subjects, n_timepoints, n_regions, n_regions]` +3. **Use absolute values:** Tests compare absolute connectivity (take `np.abs()`) +4. **Check temporal resolution:** Method must have sufficient timepoint resolution + +## File Structure + +``` +test_validation/ +├── README.md # Full documentation +├── QUICKSTART.md # This file +├── synthetic_data.py # Generate synthetic fMRI +├── test_cases.py # Test definitions (ZeroAndPerfectCorr, StepChange) +├── dfc_method_wrappers.py # Method wrappers +├── runner_reporter.py # Test execution +├── validate_dfc.py # Entry point (CLI) +└── example_validation.py # Examples +``` + +## Troubleshooting + +The validation reporter now uses only the Python standard library. If you see a +formatting issue in the summary table, check the local validation code rather +than an external package install. + +### Method Run Fails + +1. Check that `run()` returns shape `[n_subjects, n_timepoints, n_regions, n_regions]` +2. Ensure time axis is index 1 (not 0 or 2) +3. Verify connectivity values are numeric (not NaN or inf) + +### All Tests Fail + +- Your method might genuinely not match the synthetic structure +- Try with a relaxed `--pass-threshold 0.70` +- Check if method needs different parameter tuning + +### Results Saved Where? + +Default: `./validation_results/validation_results_YYYYMMDD_HHMMSS.json` + +Specify with: `--output-dir /path/to/results` + +## Next Steps + +1. **Read full docs:** See `README.md` for detailed information +2. **Run examples:** `python example_validation.py` +3. **Add your method:** Follow "Adding Your Method" above +4. **Explore outputs:** Check saved JSON for detailed per-subject scores +5. **Customize tests:** Extend `test_cases.py` for domain-specific validation + +## Support + +For issues or questions: +1. Check `README.md` for detailed documentation +2. Review examples in `example_validation.py` +3. Inspect validation output and per-subject scores +4. Check method wrapper implementation diff --git a/tests/test_validation/README.md b/tests/test_validation/README.md new file mode 100644 index 0000000..23bcef6 --- /dev/null +++ b/tests/test_validation/README.md @@ -0,0 +1,83 @@ +# dFC Validation — API Conformance Checks + +Runs a 6-sub-check structural smoke test on any registered dFC method to verify it satisfies the PydFC base-class contract. Each sub-check is isolated: a failure in one does not prevent the rest from running. + +## Sub-checks + +| # | Name | What it verifies | +|---|------|-----------------| +| 1 | `registry_instantiation` | Class resolves from its `MEASURE_NAME` via `create_measure_obj`; constructor does not raise | +| 2 | `estimate_FCS_returns_self` | `estimate_FCS(group_ts)` returns the method object itself | +| 3 | `estimate_dFC_returns_DFC` | `estimate_dFC(subj_ts)` returns a `pydfc.dfc.DFC` instance | +| 4 | `dfc_mat_shape` | `dfc.get_dFC_mat()` returns a 3-D array of shape `[n_time, R, R]` | +| 5 | `symmetry` | Each FC matrix equals its own transpose (up to 1e-10) | +| 6 | `finite_values` | No NaN or Inf in any element of `get_dFC_mat()` | + +**Failure meanings:** + +- *Sub-check 1 fails*: method cannot be instantiated at all — check imports, `MEASURE_NAME`, and constructor. +- *Sub-check 2 fails*: `estimate_FCS` does not return `self` — state-based chaining will silently break. +- *Sub-check 3 fails*: no valid `DFC` object returned — sub-checks 4–6 are skipped. +- *Sub-checks 4–6 fail*: output is malformed (wrong shape / asymmetric / non-finite). + +## Command-line usage + +```bash +# Check all registered methods +python -W ignore -m tests.test_validation.validate_dfc + +# Check specific methods (by name or alias) +python -W ignore -m tests.test_validation.validate_dfc --methods SlidingWindow aec led + +# List all registered methods and aliases +python -W ignore -m tests.test_validation.validate_dfc --list-methods + +# Save output to a file +python -W ignore -m tests.test_validation.validate_dfc > results.txt 2>&1 + +# Verbose sub-check detail +python -W ignore -m tests.test_validation.validate_dfc --verbose 2 +``` + +## Python API + +```python +from tests.test_validation import APIConformanceCheck, Reporter + +checker = APIConformanceCheck(n_regions=12, n_timepoints=80, n_subjects=3, Fs=1.0) +sub_results = checker.run("SlidingWindow") +for r in sub_results: + print(r.name, "PASS" if r.passed else f"FAIL: {r.error}") +``` + +## JSON output + +Results are saved to `./validation_results/validation_results_.json`: + +```json +{ + "timestamp": "20260603_120000", + "api_conformance": { + "SlidingWindow": [ + {"name": "registry_instantiation", "passed": true, "error": ""}, + {"name": "estimate_FCS_returns_self", "passed": true, "error": ""}, + {"name": "estimate_dFC_returns_DFC", "passed": true, "error": ""}, + {"name": "dfc_mat_shape", "passed": true, "error": ""}, + {"name": "symmetry", "passed": true, "error": ""}, + {"name": "finite_values", "passed": true, "error": ""} + ] + } +} +``` + +## Directory structure + +``` +tests/test_validation/ +├── __init__.py # Package exports +├── api_checks.py # 6-sub-check conformance suite +├── dfc_method_wrappers.py # CLI alias map; discovery delegates to pydfc auto-scan +├── runner_reporter.py # Reporter (print + save JSON) +├── validate_dfc.py # CLI entry point +└── README.md # This file +``` diff --git a/tests/test_validation/__init__.py b/tests/test_validation/__init__.py new file mode 100644 index 0000000..ae9e12f --- /dev/null +++ b/tests/test_validation/__init__.py @@ -0,0 +1,20 @@ +"""The :mod:`tests.test_validation` package — dFC method API conformance checks.""" + +from .api_checks import APIConformanceCheck, SubCheckResult +from .dfc_method_wrappers import ( + get_method_availability, + get_method_catalog, + list_registered_methods, + resolve_method_requests, +) +from .runner_reporter import Reporter + +__all__ = [ + "APIConformanceCheck", + "SubCheckResult", + "get_method_availability", + "get_method_catalog", + "list_registered_methods", + "resolve_method_requests", + "Reporter", +] diff --git a/tests/test_validation/api_checks.py b/tests/test_validation/api_checks.py new file mode 100644 index 0000000..204b151 --- /dev/null +++ b/tests/test_validation/api_checks.py @@ -0,0 +1,268 @@ +""" +API Conformance Checks for dFC Methods + +Structural contract verification that runs before scientific test cases. +Each sub-check is independent: a failure in one does not prevent the others +from running, and every sub-check result is recorded separately. + +The six sub-checks, in order: + 1. registry_instantiation — class resolves from MEASURE_NAME via _build_measure_registry + 2. estimate_FCS_returns_self — group-level call returns the same object, no exception + 3. estimate_dFC_returns_DFC — per-subject call returns a pydfc.dfc.DFC instance + 4. dfc_mat_shape — get_dFC_mat() yields a 3-D [n_time, R, R] array + 5. symmetry — each sampled FC matrix equals its own transpose + 6. finite_values — no NaN or Inf in any entry of get_dFC_mat() +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import List + +import numpy as np + +# ────────────────────────────────────────────────────────────────────────────── +# Data container +# ────────────────────────────────────────────────────────────────────────────── + + +@dataclass +class SubCheckResult: + """Result of a single API conformance sub-check.""" + + name: str + passed: bool + error: str = "" # empty string when passed + + +# ────────────────────────────────────────────────────────────────────────────── +# Conformance checker +# ────────────────────────────────────────────────────────────────────────────── + + +class APIConformanceCheck: + """ + Run six structural sub-checks on a dFC method class identified by its + MEASURE_NAME string. + + Designed to be called once per method before any scientific test case. + A broad set of default parameters is supplied at instantiation time so + that most methods can be initialised without extra configuration; methods + that require parameters not in the default set will record a failure in + sub-check 1 and skip the remaining checks. + + Parameters + ---------- + n_regions : int + Number of brain regions in the synthetic TIME_SERIES objects. + n_timepoints : int + Number of time points per subject. + n_subjects : int + Number of subjects in the group TIME_SERIES used for estimate_FCS. + Fs : float + Sampling frequency (Hz). + """ + + # Broad parameter set that covers the most common method requirements. + # Methods ignore keys they do not recognise (all use params.get() or + # key-in-dict lookup), so extra keys are always safe. + _DEFAULT_PARAMS: dict = dict( + # SlidingWindow-specific + sw_method="pear_corr", + n_overlap=0, + tapered_window=False, + # window size used by windowed methods + W=10, + # state-based methods + n_states=3, + n_subj_clstrs=2, + hmm_iter=2, + # DiscreteHMM / SlidingWindowClustr + clstr_base_measure="SlidingWindow", + clstr_distance="manhattan", + dhmm_obs_state_ratio=2, + # TimeFreq + TF_method="WTC", + # shared preprocessing + normalization=True, + num_select_nodes=None, + num_time_point=None, + Fs_ratio=None, + noise_ratio=None, + num_realization=None, + session=None, + # MTD and exponential/min_periods methods + sigma_s=2.0, + min_periods=5, + half_life=10, + ) + + def __init__( + self, + n_regions: int = 12, + n_timepoints: int = 80, + n_subjects: int = 3, + Fs: float = 1.0, + ) -> None: + self.n_regions = n_regions + self.n_timepoints = n_timepoints + self.n_subjects = n_subjects + self.Fs = Fs + + # ── helpers ─────────────────────────────────────────────────────────────── + + def _make_time_series(self): + """ + Return (group_ts, subj_ts): a multi-subject TIME_SERIES for + estimate_FCS and a single-subject one for estimate_dFC. + """ + from pydfc.time_series import TIME_SERIES + + rng = np.random.RandomState(0) + locs = np.zeros((self.n_regions, 3), dtype=float) + node_labels = [f"r{i:02d}" for i in range(self.n_regions)] + + def _ts(subj_id: str): + return TIME_SERIES( + data=rng.randn(self.n_regions, self.n_timepoints), + subj_id=subj_id, + Fs=self.Fs, + locs=locs, + node_labels=node_labels, + ) + + group_ts = _ts("sub_000") + for si in range(1, self.n_subjects): + group_ts.append_ts( + new_time_series=rng.randn(self.n_regions, self.n_timepoints), + subj_id=f"sub_{si:03d}", + ) + + subj_ts = _ts("sub_000") + return group_ts, subj_ts + + # ── public API ──────────────────────────────────────────────────────────── + + def run(self, measure_name: str) -> List[SubCheckResult]: + """ + Execute all six sub-checks for *measure_name* and return one + SubCheckResult per check. Checks are independent: an exception in + one is caught and recorded; subsequent checks that depend on an + earlier result are marked as skipped with an explanatory message. + + Parameters + ---------- + measure_name : str + The MEASURE_NAME class attribute of the target dFC method. + + Returns + ------- + List[SubCheckResult] + Always exactly six items, one per sub-check. + """ + results: List[SubCheckResult] = [] + SKIP = "Skipped: earlier required sub-check failed" + + # ── sub-check 1: registry instantiation ─────────────────────────────── + method = None + try: + from pydfc.multi_analysis_utils import create_measure_obj + + (method,) = create_measure_obj([measure_name], **self._DEFAULT_PARAMS) + results.append(SubCheckResult("registry_instantiation", True)) + except Exception as exc: + results.append(SubCheckResult("registry_instantiation", False, str(exc))) + for name in ( + "estimate_FCS_returns_self", + "estimate_dFC_returns_DFC", + "dfc_mat_shape", + "symmetry", + "finite_values", + ): + results.append(SubCheckResult(name, False, SKIP)) + return results + + # ── build TIME_SERIES (required for checks 2-6) ─────────────────────── + try: + group_ts, subj_ts = self._make_time_series() + except Exception as exc: + msg = f"TIME_SERIES construction failed: {exc}" + for name in ( + "estimate_FCS_returns_self", + "estimate_dFC_returns_DFC", + "dfc_mat_shape", + "symmetry", + "finite_values", + ): + results.append(SubCheckResult(name, False, msg)) + return results + + # ── sub-check 2: estimate_FCS returns self ──────────────────────────── + try: + ret = method.estimate_FCS(time_series=group_ts) + if ret is not method: + raise AssertionError( + f"estimate_FCS returned {type(ret).__name__!r}, expected self" + ) + results.append(SubCheckResult("estimate_FCS_returns_self", True)) + except Exception as exc: + results.append(SubCheckResult("estimate_FCS_returns_self", False, str(exc))) + # estimate_dFC may still work independently; do not skip + + # ── sub-check 3: estimate_dFC returns a DFC object ──────────────────── + dfc_obj = None + try: + from pydfc.dfc import DFC + + dfc_obj = method.estimate_dFC(time_series=subj_ts) + if not isinstance(dfc_obj, DFC): + raise AssertionError( + f"estimate_dFC returned {type(dfc_obj).__name__!r}, expected DFC" + ) + results.append(SubCheckResult("estimate_dFC_returns_DFC", True)) + except Exception as exc: + results.append(SubCheckResult("estimate_dFC_returns_DFC", False, str(exc))) + for name in ("dfc_mat_shape", "symmetry", "finite_values"): + results.append(SubCheckResult(name, False, SKIP)) + return results + + # ── sub-check 4: get_dFC_mat shape ──────────────────────────────────── + mat = None + try: + mat = dfc_obj.get_dFC_mat() + if mat.ndim != 3: + raise AssertionError(f"Expected 3-D array, got shape {mat.shape}") + if mat.shape[1] != self.n_regions or mat.shape[2] != self.n_regions: + raise AssertionError( + f"Expected [..., {self.n_regions}, {self.n_regions}], " + f"got {mat.shape}" + ) + results.append(SubCheckResult("dfc_mat_shape", True)) + except Exception as exc: + results.append(SubCheckResult("dfc_mat_shape", False, str(exc))) + + if mat is None: + for name in ("symmetry", "finite_values"): + results.append(SubCheckResult(name, False, SKIP)) + return results + + # ── sub-check 5: symmetry ───────────────────────────────────────────── + try: + n_check = min(mat.shape[0], 10) + for t in range(n_check): + if not np.allclose(mat[t], mat[t].T, atol=1e-10): + raise AssertionError(f"Frame {t} is not symmetric") + results.append(SubCheckResult("symmetry", True)) + except Exception as exc: + results.append(SubCheckResult("symmetry", False, str(exc))) + + # ── sub-check 6: finite values ──────────────────────────────────────── + try: + n_nonfinite = int(np.sum(~np.isfinite(mat))) + if n_nonfinite > 0: + raise AssertionError(f"Output contains {n_nonfinite} non-finite value(s)") + results.append(SubCheckResult("finite_values", True)) + except Exception as exc: + results.append(SubCheckResult("finite_values", False, str(exc))) + + return results diff --git a/tests/test_validation/dfc_method_wrappers.py b/tests/test_validation/dfc_method_wrappers.py new file mode 100644 index 0000000..f575a23 --- /dev/null +++ b/tests/test_validation/dfc_method_wrappers.py @@ -0,0 +1,133 @@ +"""Registry shim for dFC validation — delegates discovery to pydfc auto-scan.""" + +from __future__ import annotations + +from typing import Dict, List, Tuple + +# CLI shorthand aliases: MEASURE_NAME -> [alias, ...] +_ALIASES: Dict[str, List[str]] = { + "SlidingWindow": ["sw"], + "Time-Freq": ["tf", "wtc"], + "ExponentialWindow": ["ew"], + "AdaptiveExponentialWindow": ["aew"], + "MultiscaleWindow": ["msw"], + "EdgeCoactivation": ["eca"], + "PhaseLockingWindow": ["plv"], + "DerivativeWeightedWindow": ["dww"], + "TemporalDerivativeMultiplication": ["tdm"], + "ChangepointResetWindow": ["crw"], + "KalmanCovariance": ["kalman", "kcv"], + "LaggedMaxCorrelation": ["lmc"], + "PrecisionShrinkageWindow": ["psw"], + "RecurrenceKernelDependence": ["rkd"], + "RandomFourierDependence": ["rfd"], + "EventSynchronization": ["event"], + "CopulaTailDependence": ["ctd"], + "OjaSubspaceConnectivity": ["oja"], + "GraphDiffusionCoactivation": ["gdc"], + "CAP": ["cap"], + "ContinuousHMM": ["chmm"], + "DiscreteHMM": ["dhmm"], + "Windowless": ["windowless"], + "SlidingWindowClustr": ["swc"], + "PooledKMeansStates": ["pkms"], + "MiniBatchKMeansStates": ["mbkms"], + "GaussianMixtureStates": ["gms"], + "BayesianGaussianMixtureStates": ["bgms"], + "BirchStates": ["birchstates"], + "AgglomerativeStates": ["aggstates"], + "SpectralStates": ["specstates"], + "LaggedKMeansStates": ["lagkm"], + "MarkovSmoothedKMeansStates": ["mskms"], + "MarkovSmoothedGMMStates": ["msgms"], + "AmplitudeEnvelopeCorrelation": ["aec"], + "LeadingEigenvectorDynamics": ["led", "leida"], + "InstantaneousPhaseCoherence": ["ipc"], + "DynamicPartialCorrelation": ["dypc"], + "PhaseLagIndexWindow": ["pliw", "pli"], + "PointProcessConnectivity": ["ppfc"], + "STFTCoherence": ["stft", "stftcoh"], + "RobustSlidingWindow": ["rsw"], + "SynchronyLikelihoodWindow": ["slw"], + "NMFStates": ["nmf"], + "DifferentialCoactivationFC": ["diffcoact"], + "CurvatureCorrelationFC": ["curvcorr"], + "VolatilityWeightedFC": ["vwfc"], + "TemporalAsymmetryFC": ["tafc"], + "PhaseAmplitudeCrossFC": ["pafc"], + "MutualCompressionFC": ["mcfc"], + "TangentSpaceFC": ["tsfc"], + "SpectralSimilarityFC": ["ssfc"], + "PositiveNegativeAsymmetryFC": ["pnafc"], + "StateSpaceNeighborhoodFC": ["ssnfc", "knnfc"], + "DCCConnectivity": ["dcc"], + "PersistentHomologyFC": ["phfc", "tda"], + "TimeReversalAsymmetryFC": ["trac"], + "QuantumMutualInformationFC": ["qmic"], + "ReservoirEchoStateFC": ["resc"], + "LocalJacobianCouplingFC": ["ljcc"], + "SparseCoactivationCodeFC": ["sccc"], +} + + +def _normalize(name: str) -> str: + return "".join(c.lower() for c in name if c.isalnum()) + + +def _registry() -> Dict[str, object]: + from pydfc.multi_analysis_utils import _build_measure_registry + + return _build_measure_registry() + + +def list_registered_methods() -> List[str]: + return list(_registry().keys()) + + +def get_method_catalog() -> List[dict]: + return [ + { + "id": idx, + "key": name, + "aliases": _ALIASES.get(name, []), + "available": True, + "reason": "", + } + for idx, name in enumerate(_registry(), start=1) + ] + + +def get_method_availability() -> Tuple[List[str], Dict[str, str]]: + """Return (available_measure_names, {}). + + Methods that fail to import are absent from pydfc auto-discovery and will + simply not appear in the returned list. The empty dict signals no tracked + unavailability — import failures surface via the registry_instantiation + sub-check in api_checks.py. + """ + return list(_registry().keys()), {} + + +def resolve_method_requests( + requested: List[str], +) -> Tuple[List[str], List[str], Dict[str, str]]: + """Resolve CLI names / aliases to MEASURE_NAMEs. + + Returns (found, missing, unavailable). `unavailable` is always empty + because availability is determined by the conformance check itself. + """ + alias_map: Dict[str, str] = {} + for name in _registry(): + alias_map[_normalize(name)] = name + for alias in _ALIASES.get(name, []): + alias_map.setdefault(_normalize(alias), name) + + found: List[str] = [] + missing: List[str] = [] + for req in requested: + canonical = alias_map.get(_normalize(req)) + if canonical: + found.append(canonical) + else: + missing.append(req) + return found, missing, {} diff --git a/tests/test_validation/example_validation.py b/tests/test_validation/example_validation.py new file mode 100644 index 0000000..505b3fc --- /dev/null +++ b/tests/test_validation/example_validation.py @@ -0,0 +1,299 @@ +""" +Example: Using the dFC Validation Framework + +This script demonstrates how to use the dFC validation framework with a simple +example using the DummyMethod (for testing the framework itself). + +To run: + python example_validation.py + +To use with a real dFC method, modify the methods dict and ensure the wrapper +is correctly implemented. +""" + +from pathlib import Path + +import numpy as np + +# Import validation framework components +from test_validation import ( + DummyMethod, + Reporter, + StepChangeTest, + SyntheticDataGenerator, + ValidationRunner, + ZeroAndPerfectCorrTest, +) + + +def example_basic_usage(): + """Basic example: Generate data, run dummy method, validate.""" + print("\n" + "=" * 80) + print("EXAMPLE 1: Basic Usage") + print("=" * 80) + + # Step 1: Generate synthetic dataset + print("\n1. Generating synthetic dataset...") + generator = SyntheticDataGenerator( + n_subjects=10, # Small dataset for quick demo + n_regions=50, + n_timepoints=300, + segment_n_blocks=[3, 4, 5], + noise_floor=0.01, + random_seed=42, + ) + + timeseries, ground_truth = generator.generate() + print(f" Timeseries shape: {timeseries.shape}") + print(f" Segments: {len(ground_truth.segments)}") + + # Step 2: Create tests + print("\n2. Creating test cases...") + tests = [ + ZeroAndPerfectCorrTest(pass_threshold=0.85), + StepChangeTest(pass_threshold=0.85), + ] + print(f" Tests: {[t.name for t in tests]}") + + # Step 3: Create method (using DummyMethod for demonstration) + print("\n3. Setting up dFC method...") + methods = { + "DummyMethod": DummyMethod(), + } + print(f" Methods: {list(methods.keys())}") + + # Step 4: Run validation + print("\n4. Running validation...") + runner = ValidationRunner(verbose=1) + results = runner.run( + methods=methods, + test_cases=tests, + timeseries=timeseries, + ground_truth=ground_truth, + ) + + # Step 5: Report results + print("\n5. Generating report...") + reporter = Reporter(output_dir="./example_results") + reporter.print_summary(results) + reporter.print_details(results) + + return results + + +def example_custom_configuration(): + """Example: Using custom parameters and configurations.""" + print("\n" + "=" * 80) + print("EXAMPLE 2: Custom Configuration") + print("=" * 80) + + # Generate larger dataset with custom settings + print("\n1. Generating custom dataset...") + generator = SyntheticDataGenerator( + n_subjects=20, + n_regions=100, + n_timepoints=600, + segment_n_blocks=[6, 8, 10], # More blocks + noise_floor=0.02, # Higher noise + random_seed=123, + signal_type="gaussian_smooth", + ) + + timeseries, ground_truth = generator.generate() + + # Inspect ground truth + print("\n Dataset details:") + print(f" - Shape: {timeseries.shape}") + print(f" - Noise floor: {ground_truth.noise_floor}") + print(" - Segments:") + for i, seg in enumerate(ground_truth.segments): + print( + f" Segment {i}: blocks={seg.n_blocks}, " + f"eval_window=[{seg.eval_start}:{seg.eval_end}]" + ) + + # Create tests with custom thresholds + print("\n2. Creating tests with custom thresholds...") + tests = [ + ZeroAndPerfectCorrTest(pass_threshold=0.80), # Relaxed + StepChangeTest(pass_threshold=0.75), # More relaxed + ] + + # Run with dummy method + methods = {"DummyMethod": DummyMethod()} + + print("\n3. Running validation...") + runner = ValidationRunner(verbose=1) + results = runner.run( + methods=methods, + test_cases=tests, + timeseries=timeseries, + ground_truth=ground_truth, + ) + + # Report with JSON output + print("\n4. Generating report with JSON...") + reporter = Reporter(output_dir="./example_results_custom") + reporter.print_summary(results) + json_file = reporter.save_json(results) + print(f" Results saved: {json_file}") + + return results + + +def example_inspect_synthetic_data(): + """Example: Inspect properties of synthetic data.""" + print("\n" + "=" * 80) + print("EXAMPLE 3: Inspecting Synthetic Data") + print("=" * 80) + + # Generate small dataset for inspection + generator = SyntheticDataGenerator( + n_subjects=5, + n_regions=20, + n_timepoints=200, + segment_n_blocks=[2, 3], + random_seed=42, + ) + + timeseries, ground_truth = generator.generate() + + print( + f"\nDataset: {timeseries.shape[0]} subjects, " + f"{timeseries.shape[1]} timepoints, {timeseries.shape[2]} regions" + ) + + # Analyze first segment + seg0 = ground_truth.segments[0] + print("\nSegment 0 Analysis:") + print(f" Timepoint range: [{seg0.start}:{seg0.end}]") + print(f" Evaluation window: [{seg0.eval_start}:{seg0.eval_end}]") + print(f" Number of blocks: {seg0.n_blocks}") + print(f" Block assignments: {seg0.region_block_ids}") + + # Check block sizes + unique_blocks, counts = np.unique(seg0.region_block_ids, return_counts=True) + print(f" Block sizes: {dict(zip(unique_blocks, counts))}") + + # Verify correlation structure + n_perfect = np.sum(seg0.perfect_corr_mask) // 2 # Divide by 2 (symmetric) + n_zero = np.sum(seg0.zero_corr_mask) // 2 + print(f" Perfect-corr region pairs: {n_perfect}") + print(f" Zero-corr region pairs: {n_zero}") + + # Check actual correlations in synthetic data + print("\nActual correlations in first subject, first segment:") + subject_segment_ts = timeseries[0, seg0.start : seg0.eval_end, :] + actual_corr = np.corrcoef(subject_segment_ts.T) + + # Within-block correlation + within_block_indices = np.where( + seg0.perfect_corr_mask + & ( + np.arange(seg0.perfect_corr_mask.shape[0])[:, None] + != np.arange(seg0.perfect_corr_mask.shape[0])[None, :] + ) + ) + within_block_corrs = actual_corr[within_block_indices] + print( + f" Within-block corr: mean={np.mean(within_block_corrs):.3f}, " + f"std={np.std(within_block_corrs):.3f}" + ) + + # Between-block correlation + between_block_indices = np.where(seg0.zero_corr_mask) + between_block_corrs = actual_corr[between_block_indices] + print( + f" Between-block corr: mean={np.mean(between_block_corrs):.3f}, " + f"std={np.std(between_block_corrs):.3f}" + ) + + +def example_comparing_methods(): + """Example: Compare multiple methods (extended).""" + print("\n" + "=" * 80) + print("EXAMPLE 4: Comparing Multiple Methods") + print("=" * 80) + + # Generate dataset + print("\n1. Generating dataset...") + generator = SyntheticDataGenerator( + n_subjects=15, + n_regions=80, + n_timepoints=400, + segment_n_blocks=[4, 5, 6], + ) + timeseries, ground_truth = generator.generate() + + # Create tests + tests = [ + ZeroAndPerfectCorrTest(), + StepChangeTest(), + ] + + # Compare multiple method configurations + methods = { + "DummyMethod_v1": DummyMethod(), + "DummyMethod_v2": DummyMethod(), # Could be different variant + } + + print(f"\n2. Testing {len(methods)} methods...") + runner = ValidationRunner(verbose=1) + results = runner.run( + methods=methods, + test_cases=tests, + timeseries=timeseries, + ground_truth=ground_truth, + ) + + # Generate comparison report + print("\n3. Comparison Results:") + reporter = Reporter() + reporter.print_summary(results) + + # Analyze which method performed better + print("\nDetailed Comparison:") + for method_name in methods.keys(): + method_results = [r for r in results if r.method_name == method_name] + avg_score = np.mean([r.score for r in method_results]) + n_pass = sum(1 for r in method_results if r.passed) + print(f" {method_name}:") + print(f" - Average score: {avg_score:.3f}") + print(f" - Tests passed: {n_pass}/{len(tests)}") + + +if __name__ == "__main__": + print("\n" + "=" * 80) + print("DFC VALIDATION FRAMEWORK - EXAMPLES") + print("=" * 80) + + # Run all examples + print("\nRunning examples...\n") + + # Example 1 + try: + example_basic_usage() + except Exception as e: + print(f"ERROR in example 1: {e}") + + # Example 2 + try: + example_custom_configuration() + except Exception as e: + print(f"ERROR in example 2: {e}") + + # Example 3 + try: + example_inspect_synthetic_data() + except Exception as e: + print(f"ERROR in example 3: {e}") + + # Example 4 + try: + example_comparing_methods() + except Exception as e: + print(f"ERROR in example 4: {e}") + + print("\n" + "=" * 80) + print("Examples completed!") + print("=" * 80) diff --git a/tests/test_validation/runner_reporter.py b/tests/test_validation/runner_reporter.py new file mode 100644 index 0000000..a32bc6a --- /dev/null +++ b/tests/test_validation/runner_reporter.py @@ -0,0 +1,96 @@ +"""Reporter for dFC API conformance results.""" + +import json +from datetime import datetime +from pathlib import Path +from typing import Dict, List + +# Short display names for the 6 sub-checks (column headers) +_SUBCHECK_LABELS = { + "registry_instantiation": "instantiation", + "estimate_FCS_returns_self": "FCS→self", + "estimate_dFC_returns_DFC": "dFC→DFC", + "dfc_mat_shape": "shape", + "symmetry": "symmetric", + "finite_values": "finite", +} + + +def _format_table(headers: List[str], rows: List[List[str]]) -> str: + widths = [len(h) for h in headers] + for row in rows: + for i, cell in enumerate(row): + widths[i] = max(widths[i], len(cell)) + sep = "-+-".join("-" * w for w in widths) + fmt = lambda row: " | ".join(str(c).ljust(widths[i]) for i, c in enumerate(row)) + lines = [fmt(headers), sep] + [fmt(r) for r in rows] + return "\n".join(lines) + + +class Reporter: + """Print and save API conformance results.""" + + def __init__(self, output_dir: str = "./validation_results"): + self.output_dir = Path(output_dir) + self.output_dir.mkdir(parents=True, exist_ok=True) + self.timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + + def print_summary(self, api_conformance: Dict) -> None: + """Print a table of all methods × sub-checks, then list any errors.""" + if not api_conformance: + print("No results to report.") + return + + # Collect ordered sub-check names from the first result + first = next(iter(api_conformance.values())) + subcheck_names = [r.name for r in first] + col_headers = [_SUBCHECK_LABELS.get(n, n) for n in subcheck_names] + + headers = ["Method"] + col_headers + ["TOTAL"] + rows = [] + failures = [] # (method, subcheck_name, error) + + for method, sub_results in api_conformance.items(): + n_pass = sum(r.passed for r in sub_results) + n_total = len(sub_results) + cells = [] + for r in sub_results: + cells.append("PASS" if r.passed else "FAIL") + if not r.passed: + failures.append((method, r.name, r.error or "")) + rows.append([method] + cells + [f"{n_pass}/{n_total}"]) + + print("\n" + "=" * 80) + print("API CONFORMANCE RESULTS") + print("=" * 80) + print(_format_table(headers, rows)) + + if failures: + print("\nFailure details:") + for method, name, error in failures: + err_str = f" — {error}" if error else "" + print(f" {method} | {name}{err_str}") + + def save_json(self, api_conformance: Dict) -> str: + """Save conformance results as JSON. Returns the output file path.""" + results_dict = { + "timestamp": self.timestamp, + "api_conformance": { + method_name: [ + {"name": r.name, "passed": r.passed, "error": r.error} + for r in sub_results + ] + for method_name, sub_results in api_conformance.items() + }, + } + output_file = self.output_dir / f"validation_results_{self.timestamp}.json" + with open(output_file, "w") as f: + json.dump(results_dict, f, indent=2) + print(f"\nResults saved to: {output_file}") + return str(output_file) + + def generate_report(self, api_conformance: Dict, save_json: bool = True) -> None: + """Print summary table and optionally save to JSON.""" + self.print_summary(api_conformance) + if save_json: + self.save_json(api_conformance) diff --git a/tests/test_validation/validate_dfc.py b/tests/test_validation/validate_dfc.py new file mode 100644 index 0000000..fbd8eeb --- /dev/null +++ b/tests/test_validation/validate_dfc.py @@ -0,0 +1,114 @@ +""" +dFC Methods — API Conformance Checks + +Runs the 6-sub-check API conformance suite on registered dFC methods. +Each check is isolated: a failure in one does not prevent the rest from running. + +Usage +----- +python -m tests.test_validation.validate_dfc [--methods METHOD1 METHOD2 ...] +python -m tests.test_validation.validate_dfc --list-methods + +Example +------- +python -W ignore -m tests.test_validation.validate_dfc --methods SlidingWindow aec +python -W ignore -m tests.test_validation.validate_dfc > results.txt 2>&1 +""" + +from __future__ import annotations + +import argparse +import sys + +from .api_checks import APIConformanceCheck +from .dfc_method_wrappers import ( + get_method_availability, + get_method_catalog, + list_registered_methods, + resolve_method_requests, +) +from .runner_reporter import Reporter + + +def main(args=None): + parser = argparse.ArgumentParser( + description="Run API conformance checks on dFC methods" + ) + parser.add_argument( + "--methods", + nargs="+", + default=None, + help="Methods to check by name or alias (default: all available)", + ) + parser.add_argument( + "--output-dir", + type=str, + default="./validation_results", + help="Directory for JSON output", + ) + parser.add_argument( + "--verbose", + type=int, + default=1, + choices=[0, 1, 2], + help="0=silent, 1=per-method status, 2=per-sub-check detail", + ) + parser.add_argument( + "--list-methods", + action="store_true", + help="List registered methods with aliases and exit", + ) + + parsed_args = parser.parse_args(args) + + print("=" * 80) + print("DFC VALIDATION — API CONFORMANCE CHECKS") + print("=" * 80) + + if parsed_args.list_methods: + print("\nRegistered methods:") + for entry in get_method_catalog(): + aliases = ", ".join(entry["aliases"]) if entry["aliases"] else "-" + print(f" [{entry['id']}] {entry['key']}") + print(f" aliases: {aliases}") + return 0 + + all_methods, _ = get_method_availability() + + if parsed_args.methods: + methods_to_test, missing, _ = resolve_method_requests(parsed_args.methods) + if missing: + print(f"\nWARNING: Methods not found: {missing}") + print(f" Registered methods: {list_registered_methods()}") + else: + methods_to_test = all_methods + + print(f"\nMethods to test: {methods_to_test}") + + if not methods_to_test: + print("ERROR: No runnable methods selected.") + return 2 + + # Run API conformance checks + print("\nRunning API conformance checks...") + api_checker = APIConformanceCheck() + api_conformance_results = {} + + for measure_name in methods_to_test: + if parsed_args.verbose >= 2: + print(f" checking {measure_name}...") + api_conformance_results[measure_name] = api_checker.run(measure_name) + + # Report + reporter = Reporter(output_dir=parsed_args.output_dir) + reporter.generate_report(api_conformance_results, save_json=True) + + all_passed = all( + all(r.passed for r in sub_results) + for sub_results in api_conformance_results.values() + ) + return 0 if all_passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_validation/visualize_dfc.py b/tests/test_validation/visualize_dfc.py new file mode 100644 index 0000000..d17bfb8 --- /dev/null +++ b/tests/test_validation/visualize_dfc.py @@ -0,0 +1,350 @@ +"""Visualize outputs from the experimental state-free dFC methods.""" + +import sys +import types +import warnings +from importlib import import_module +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np + +_PACKAGE_ROOT = Path(__file__).resolve().parents[2] + + +def _ensure_pydfc_namespace(): + pydfc_path = str(_PACKAGE_ROOT / "pydfc") + pydfc_methods_path = str(_PACKAGE_ROOT / "pydfc" / "dfc_methods") + + if "pydfc" not in sys.modules: + pydfc_module = types.ModuleType("pydfc") + pydfc_module.__path__ = [pydfc_path] + sys.modules["pydfc"] = pydfc_module + + if "pydfc.dfc_methods" not in sys.modules: + pydfc_methods_module = types.ModuleType("pydfc.dfc_methods") + pydfc_methods_module.__path__ = [pydfc_methods_path] + sys.modules["pydfc.dfc_methods"] = pydfc_methods_module + + +def _load_pydfc_class(module_name, class_name): + _ensure_pydfc_namespace() + module = import_module(module_name) + return getattr(module, class_name) + + +_ensure_pydfc_namespace() +data_loader = import_module("pydfc.data_loader") + +SLIDING_WINDOW = _load_pydfc_class("pydfc.dfc_methods.sliding_window", "SLIDING_WINDOW") +EXPONENTIAL_WINDOW = _load_pydfc_class( + "pydfc.dfc_methods.exponential_window", "EXPONENTIAL_WINDOW" +) +ADAPTIVE_EXPONENTIAL_WINDOW = _load_pydfc_class( + "pydfc.dfc_methods.adaptive_exponential_window", "ADAPTIVE_EXPONENTIAL_WINDOW" +) +MULTISCALE_WINDOW = _load_pydfc_class( + "pydfc.dfc_methods.multiscale_window", "MULTISCALE_WINDOW" +) +EDGE_COACTIVATION = _load_pydfc_class( + "pydfc.dfc_methods.edge_coactivation", "EDGE_COACTIVATION" +) +PHASE_LOCKING_WINDOW = _load_pydfc_class( + "pydfc.dfc_methods.phase_locking_window", "PHASE_LOCKING_WINDOW" +) +DERIVATIVE_WEIGHTED_WINDOW = _load_pydfc_class( + "pydfc.dfc_methods.derivative_weighted_window", "DERIVATIVE_WEIGHTED_WINDOW" +) +CHANGEPOINT_RESET_WINDOW = _load_pydfc_class( + "pydfc.dfc_methods.changepoint_reset_window", "CHANGEPOINT_RESET_WINDOW" +) +KALMAN_COVARIANCE = _load_pydfc_class( + "pydfc.dfc_methods.kalman_covariance", "KALMAN_COVARIANCE" +) +LAGGED_MAX_CORRELATION = _load_pydfc_class( + "pydfc.dfc_methods.lagged_max_correlation", "LAGGED_MAX_CORRELATION" +) +PRECISION_SHRINKAGE_WINDOW = _load_pydfc_class( + "pydfc.dfc_methods.precision_shrinkage_window", "PRECISION_SHRINKAGE_WINDOW" +) +RECURRENCE_KERNEL_DEPENDENCE = _load_pydfc_class( + "pydfc.dfc_methods.recurrence_kernel_dependence", "RECURRENCE_KERNEL_DEPENDENCE" +) +RANDOM_FOURIER_DEPENDENCE = _load_pydfc_class( + "pydfc.dfc_methods.random_fourier_dependence", "RANDOM_FOURIER_DEPENDENCE" +) +EVENT_SYNCHRONIZATION = _load_pydfc_class( + "pydfc.dfc_methods.event_synchronization", "EVENT_SYNCHRONIZATION" +) +COPULA_TAIL_DEPENDENCE = _load_pydfc_class( + "pydfc.dfc_methods.copula_tail_dependence", "COPULA_TAIL_DEPENDENCE" +) +OJA_SUBSPACE_CONNECTIVITY = _load_pydfc_class( + "pydfc.dfc_methods.oja_subspace_connectivity", "OJA_SUBSPACE_CONNECTIVITY" +) +GRAPH_DIFFUSION_COACTIVATION = _load_pydfc_class( + "pydfc.dfc_methods.graph_diffusion_coactivation", "GRAPH_DIFFUSION_COACTIVATION" +) + + +warnings.simplefilter("ignore") + +OUTPUT_DIR = Path("validation_results") / "visualize_dfc" +OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + +# Keep this subset modest so every method is fast and the matrices remain readable. +NUM_SELECT_NODES = 50 + + +def load_demo_bold(): + return data_loader.nifti2timeseries( + nifti_file=( + "examples/sample_data/sub-0001_task-restingstate_acq-mb3_" + "space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz" + ), + n_rois=100, + Fs=1 / 0.75, + subj_id="sub-0001", + confound_strategy="no_motion", # no_motion, no_motion_no_gsr, or none + standardize=False, + TS_name=None, + session=None, + ) + + +def representative_trs(tr_array, n_samples=8): + """Pick a compact, evenly spaced subset of available dFC samples.""" + tr_array = np.asarray(tr_array, dtype=int) + if len(tr_array) <= n_samples: + return tr_array + sample_idx = np.linspace(0, len(tr_array) - 1, n_samples, dtype=int) + return tr_array[sample_idx] + + +def method_specs(): + base_params = { + "normalization": True, + "num_select_nodes": NUM_SELECT_NODES, + } + + return [ + ( + "00_sliding_window", + SLIDING_WINDOW, + { + **base_params, + "W": 44, + "n_overlap": 0.5, + "sw_method": "pear_corr", + "tapered_window": True, + "window_std": None, + "n_jobs_sw": 1, + "backend_sw": "threading", + }, + ), + ( + "01_exponential_window", + EXPONENTIAL_WINDOW, + { + **base_params, + "half_life": 30, + "min_periods": 12, + }, + ), + ( + "02_adaptive_exponential_window", + ADAPTIVE_EXPONENTIAL_WINDOW, + { + **base_params, + "min_periods": 12, + "alpha_min": 0.02, + "alpha_max": 0.35, + }, + ), + ( + "03_multiscale_window", + MULTISCALE_WINDOW, + { + **base_params, + "windows": [15, 30, 60], + }, + ), + ( + "04_edge_coactivation", + EDGE_COACTIVATION, + { + **base_params, + "half_life": 20, + "min_periods": 12, + }, + ), + ( + "05_phase_locking_window", + PHASE_LOCKING_WINDOW, + { + **base_params, + "W": 44, + }, + ), + ( + "06_derivative_weighted_window", + DERIVATIVE_WEIGHTED_WINDOW, + { + **base_params, + "W": 44, + }, + ), + ( + "07_changepoint_reset_window", + CHANGEPOINT_RESET_WINDOW, + { + **base_params, + "half_life": 20, + "min_periods": 12, + "change_threshold": 4.0, + }, + ), + ( + "08_kalman_covariance", + KALMAN_COVARIANCE, + { + **base_params, + "half_life": 25, + "min_periods": 12, + "process_noise": 1e-4, + }, + ), + ( + "09_lagged_max_correlation", + LAGGED_MAX_CORRELATION, + { + **base_params, + "W": 44, + "max_lag": 2, + }, + ), + ( + "10_precision_shrinkage_window", + PRECISION_SHRINKAGE_WINDOW, + { + **base_params, + "W": 60, + }, + ), + ( + "11_recurrence_kernel_dependence", + RECURRENCE_KERNEL_DEPENDENCE, + { + **base_params, + "min_periods": 20, + "kernel_width": 1.5, + }, + ), + ( + "12_random_fourier_dependence", + RANDOM_FOURIER_DEPENDENCE, + { + **base_params, + "half_life": 25, + "min_periods": 20, + "n_random_features": 128, + "random_seed": 42, + }, + ), + ( + "13_event_synchronization", + EVENT_SYNCHRONIZATION, + { + **base_params, + "min_periods": 20, + "event_quantile": 0.85, + "event_decay": 0.97, + }, + ), + ( + "14_copula_tail_dependence", + COPULA_TAIL_DEPENDENCE, + { + **base_params, + "half_life": 25, + "min_periods": 20, + "tail_quantile": 0.8, + }, + ), + ( + "15_oja_subspace_connectivity", + OJA_SUBSPACE_CONNECTIVITY, + { + **base_params, + "half_life": 25, + "min_periods": 20, + "n_components": 10, + "learning_rate": 0.03, + "random_seed": 42, + }, + ), + ( + "16_graph_diffusion_coactivation", + GRAPH_DIFFUSION_COACTIVATION, + { + **base_params, + "half_life": 20, + "min_periods": 20, + "diffusion_rate": 0.2, + "instantaneous_weight": 0.15, + }, + ), + ] + + +def validate_method_params(method_name, method_cls, params): + """Catch unsupported hyperparameters before running a method.""" + method = method_cls(**params) + unsupported = sorted(set(params) - set(method.params_name_lst)) + if unsupported: + raise ValueError( + f"{method_name} received unsupported parameter(s): {unsupported}. " + f"Supported parameters are: {method.params_name_lst}" + ) + return method + + +def plot_connectivity_strength(dfc_obj, method_name): + """Save a compact time-course summary for each dFC output.""" + mats = dfc_obj.get_dFC_mat(TRs=dfc_obj.TR_array) + upper = np.triu_indices(mats.shape[1], k=1) + mean_abs_connectivity = np.nanmean(np.abs(mats[:, upper[0], upper[1]]), axis=1) + + plt.figure(figsize=(10, 3)) + plt.plot(dfc_obj.TR_array, mean_abs_connectivity, linewidth=1.5) + plt.xlabel("TR") + plt.ylabel("Mean |connectivity|") + plt.title(method_name) + plt.tight_layout() + plt.savefig(OUTPUT_DIR / f"{method_name}_mean_abs_connectivity.png", dpi=150) + plt.close() + + +def main(): + BOLD = load_demo_bold() + + for method_name, method_cls, params in method_specs(): + print(f"Running {method_name}...") + measure = validate_method_params(method_name, method_cls, params) + dFC = measure.estimate_dFC(time_series=BOLD) + TRs = representative_trs(dFC.TR_array, n_samples=8) + + dFC.visualize_dFC( + TRs=TRs, + normalize=False, + fix_lim=False, + save_image=True, + output_root=str(OUTPUT_DIR / f"{method_name}_"), + ) + plot_connectivity_strength(dFC, method_name) + + print(f"Saved visualization outputs to: {OUTPUT_DIR}") + + +if __name__ == "__main__": + main() diff --git a/threshold_70_filtered_methods.npy b/threshold_70_filtered_methods.npy new file mode 100644 index 0000000..9da8620 Binary files /dev/null and b/threshold_70_filtered_methods.npy differ