eval_utils.py

import itertools
import os
import time
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from runners import runner_agd, runner_gd
import math_utils  # Assumed to provide mish, sigmoid, uniform_initialization, normal_initialization


algorithms_runners = {
    "Nesterov Accelerated Gradient Descent": runner_agd
    #  "Gradient Descent": runner_gd
}

def convert_configuration_to_textual_description(configuration):
    """
    Converts a configuration tuple into a textual description to be used in the plot title.
    Assumes the configuration tuple has the following elements (in the given order):
        - dim_hidden_layer
        - activation_function
        - regularization_lambda
        - initialization_strategy
    """
    return (
        f"Dim hidden layer: {configuration[0]}, "
        f"Activation function: {configuration[1].__name__}, "
        f"Regularization lambda: {configuration[2]}, "
        f"Initialization strategy: {configuration[3].__name__}"
    )


def hyperparameter_evaluation(path_file_info_data):
    """
    Takes as input the path to a file containing the information about the problems. Expects a csv with the following columns:
    uci_id, name, task, num_of_features, num_of_instances, characteristics, feature_types, sparsity
    """
    #  Read the info data. 
    info_data = pd.read_csv(path_file_info_data)

    # Create output directory if it does not exist
    output_dir = "plots_hyperparameters_new"
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    # Dictionary to record evaluation data.
    hyperparameter_evaluation_data = {
        "problem_uci_id": [],
        "problem_name": [],
        "problem_task": [],
        "problem_num_of_input_features": [],
        "problem_num_of_output_features": [],
        "problem_num_of_instances": [],
        "problem_characteristics": [],
        "problem_feature_types": [],
        "problem_sparsity": [],
        "algorithm_name": [],
        "dim_hidden_layer": [],
        "activation_function": [],
        "regularization_lambda": [],
        "kappas": [],
        "algorithm_iterations": [],
        "theory_iterations": [],
        "algorithm_minimum": [],
        "iteration_algorithm_minimum": [],
        "total_time": [],
        "k_time": [],
        "mean_iteration_time": [],
        "H_computation_time": []

    }

    print("Starting hyperparameter evaluation...")

    # Define ranges for each hyperparameter.
    dims_hidden_layer = [5, 100, 500]
    activation_functions = [math_utils.mish_function]
    regularization_lambdas = [10**-6, 10**-2, 10]

    initialization_strategies = [math_utils.uniform_initialization]



    # Generate all hyperparameter configurations.
    configs = list(itertools.product(dims_hidden_layer, activation_functions, regularization_lambdas, initialization_strategies))
    # For demonstration purposes we use a slice of the configurations.
    configs_to_use = configs

    # Progress tracking variables.
    total_iterations = len(info_data) * len(algorithms_runners) * len(configs_to_use)
    iteration = 0
    start_time = time.time()


    try:
        c = 0
        for index, row in info_data.iloc[::-1].iterrows():
            for algorithm_name, runner in algorithms_runners.items():
                for config in configs_to_use:

                    try:
                        config_description = convert_configuration_to_textual_description(config)
                        print(f"Running {algorithm_name} on problem {row['uci_id']} with configuration:")
                        print(f"    {config_description}")


                        # Try running the algorithm.
                        try:
                            k, algorithm_iterations, theory_iterations, algorithm_errors, theory_errors, H_computation_time, k_time, mean_iteration_time, total_time = runner(row['uci_id'], config)
                        except Exception as e:
                            print(f"Error while running {algorithm_name} on problem {row['uci_id']} with configuration {config}: {e}")
                            continue

                        minimum_relative_gap = np.log(np.min(algorithm_errors))
                        iteration_minimum_relative_gap = np.argmin(algorithm_errors)+1
                        print(f"Algorithm iterations: {algorithm_iterations}")
                        print(f"Algorithm minimum relative gap: {minimum_relative_gap}")

                        # Record the results.
                        hyperparameter_evaluation_data["problem_uci_id"].append(row['uci_id'])
                        hyperparameter_evaluation_data["problem_name"].append(row['name'])
                        hyperparameter_evaluation_data["problem_task"].append(row['task'])
                        hyperparameter_evaluation_data["problem_num_of_input_features"].append(row['num_of_input_features'])
                        hyperparameter_evaluation_data["problem_num_of_output_features"].append(row['num_of_output_features'])
                        hyperparameter_evaluation_data["problem_num_of_instances"].append(row['num_of_instances'])
                        hyperparameter_evaluation_data["problem_characteristics"].append(row['characteristics'])
                        hyperparameter_evaluation_data["problem_feature_types"].append(row['feature_types'])
                        hyperparameter_evaluation_data["problem_sparsity"].append(row['sparsity'])
                        hyperparameter_evaluation_data["algorithm_name"].append(algorithm_name)
                        hyperparameter_evaluation_data["dim_hidden_layer"].append(config[0])
                        hyperparameter_evaluation_data["activation_function"].append(config[1].__name__)
                        hyperparameter_evaluation_data["regularization_lambda"].append(config[2])
                        hyperparameter_evaluation_data["kappas"].append(k)
                        hyperparameter_evaluation_data["algorithm_iterations"].append(algorithm_iterations)
                        hyperparameter_evaluation_data["theory_iterations"].append(theory_iterations)
                        hyperparameter_evaluation_data["algorithm_minimum"].append(minimum_relative_gap)
                        hyperparameter_evaluation_data["iteration_algorithm_minimum"].append(iteration_minimum_relative_gap)
                        hyperparameter_evaluation_data["total_time"].append(total_time)
                        hyperparameter_evaluation_data["k_time"].append(k_time)
                        hyperparameter_evaluation_data["mean_iteration_time"].append(mean_iteration_time)
                        hyperparameter_evaluation_data["H_computation_time"].append(H_computation_time)

                        # Try plotting and saving the figure.
                        try:
                            plt.figure(figsize=(15, 6))
                            number = iteration_minimum_relative_gap
                            if iteration_minimum_relative_gap < 5000 and len(algorithm_errors) in [4999, 5000, 4998, 4997]:
                                number = iteration_minimum_relative_gap + 100
                            plt.plot(range(number), np.log(algorithm_errors[0:number]), color='orangered', linewidth=2, label="Algorithm Learning Curve")
                            plt.plot(range(number), np.log(theory_errors[0:number]), color='blueviolet', linewidth=2, label="Theoretical Bound")

                            plt.xlabel("Epochs", fontsize=16)
                            plt.ylabel("Relative Gap", fontsize=16)

                            plt.xticks(fontsize=14)
                            plt.yticks(fontsize=14)

                            plt.legend(fontsize=14, loc='upper right', frameon=True)
                            plt.grid(True, which='both', linestyle='--', linewidth=0.7)

                            filename = (
                                f"{output_dir}/problem_{row['uci_id']}_{algorithm_name.replace(' ', '_')}_"
                                f"config_{config[0]}_{config[1].__name__}_{config[2]}_{config[3].__name__}.png"
                            )
                            plt.savefig(filename)
                            plt.close()
                            print(f"Saved plot to {filename}\n")

                        except Exception as e:
                            print(f"Error while plotting/saving for {algorithm_name} on problem {row['uci_id']} with configuration {config}: {e}")
                            # Continue even if plotting fails.
                            continue

                    finally:
                        # Update progress tracking regardless of success or error.
                        iteration += 1
                        elapsed_time = time.time() - start_time
                        avg_time = elapsed_time / iteration
                        remaining_iterations = total_iterations - iteration
                        estimated_remaining = avg_time * remaining_iterations
                        print(f"[Progress] {iteration}/{total_iterations} iterations completed. "
                              f"Elapsed time: {elapsed_time:.2f}s, Estimated remaining time: {estimated_remaining:.2f}s.")

    except KeyboardInterrupt:
        print("Execution interrupted by user. Saving current progress to CSV.")
        pd.DataFrame(hyperparameter_evaluation_data).to_csv("hyperparameter_evaluation_data.csv", index=False)
        raise  # Re-raise the interrupt so the user sees it.
    except Exception as e:
        print(f"An unexpected error occurred: {e}")
        pd.DataFrame(hyperparameter_evaluation_data).to_csv("hyperparameter_evaluation_data.csv", index=False)
        raise
    finally:
        print("Finished")
        # In any case, write out the data collected so far.
        pd.DataFrame(hyperparameter_evaluation_data).to_csv("hyperparameter_evaluation_data.csv", index=False)

    print("Hyperparameter evaluation completed.")
    return hyperparameter_evaluation_data



if __name__ == "__main__":
    hyperparameter_evaluation("info_data.csv")
    #  algorithms_evaluation()