From 7a638f28c6508794eefe457bce2f995cb65eb11b Mon Sep 17 00:00:00 2001 From: valosekj Date: Tue, 21 Jan 2025 17:06:17 -0500 Subject: [PATCH 01/26] fix `ValueError: y_true and y_pred contain different number of classes` --- study/metrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/study/metrics.py b/study/metrics.py index 1185a51..b9fbd37 100644 --- a/study/metrics.py +++ b/study/metrics.py @@ -19,7 +19,8 @@ def clean_val_for_db(val): def sk_log_loss(manager: OptunaModelManager, x: BaseDataManager, y: BaseDataManager): # Log Loss py = manager.predict_proba(x.as_array()) - return log_loss(y.as_array(), py) + y_labels = [f"v{i}" for i in range(py.shape[1])] + return log_loss(y.as_array(), py, labels=y_labels) def sk_balanced_accuracy(manager: OptunaModelManager, x: BaseDataManager, y: BaseDataManager): # Balanced Accuracy From 0977831fdafa10ec4ebaedf65daf2ecf55c9845f Mon Sep 17 00:00:00 2001 From: Jan Valosek <39456460+valosekj@users.noreply.github.com> Date: Wed, 22 Jan 2025 16:15:16 -0500 Subject: [PATCH 02/26] use number instead of str --- study/metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/study/metrics.py b/study/metrics.py index b9fbd37..5ba6b7e 100644 --- a/study/metrics.py +++ b/study/metrics.py @@ -19,7 +19,7 @@ def clean_val_for_db(val): def sk_log_loss(manager: OptunaModelManager, x: BaseDataManager, y: BaseDataManager): # Log Loss py = manager.predict_proba(x.as_array()) - y_labels = [f"v{i}" for i in range(py.shape[1])] + y_labels = [i for i in range(py.shape[1])] return log_loss(y.as_array(), py, labels=y_labels) def sk_balanced_accuracy(manager: OptunaModelManager, x: BaseDataManager, y: BaseDataManager): @@ -142,4 +142,4 @@ def incorrect_samples(manager: OptunaModelManager, x: BaseDataManager, y: BaseDa # Strip quotation marks from the result so the DB backend doesn't explode bad_samples = clean_val_for_db(bad_samples) - return bad_samples \ No newline at end of file + return bad_samples From e2f501ad322f12794f62fec3c86dc3c5e2c2bd6c Mon Sep 17 00:00:00 2001 From: valosekj Date: Wed, 12 Feb 2025 15:47:52 -0500 Subject: [PATCH 03/26] Set default LogisticRegression solver to saga to make it working for RFE --- data/hooks/feature_selection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/hooks/feature_selection.py b/data/hooks/feature_selection.py index aad1690..9b59d96 100644 --- a/data/hooks/feature_selection.py +++ b/data/hooks/feature_selection.py @@ -203,7 +203,7 @@ def from_config(cls, config: dict, logger: Logger = Logger.root) -> Self: def tune(self, trial: Trial): self.prop_tuner.tune(trial) # Generate the new backing model based on this setup - new_lor = LogisticRegression() + new_lor = LogisticRegression(solver='saga') self.backing_rfe = RFE(estimator=new_lor, n_features_to_select=self.prop_tuner.value) def tunable_params(self) -> list[TunableParam]: From b5cc4d161b942524eb5084ba808296b87de1dd8b Mon Sep 17 00:00:00 2001 From: valosekj Date: Sat, 15 Feb 2025 17:43:28 -0500 Subject: [PATCH 04/26] Add a script to inspect and plot output db --- inspect_output_db.py | 265 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 265 insertions(+) create mode 100644 inspect_output_db.py diff --git a/inspect_output_db.py b/inspect_output_db.py new file mode 100644 index 0000000..0a2f9e8 --- /dev/null +++ b/inspect_output_db.py @@ -0,0 +1,265 @@ +""" +Script for inspecting and analyzing the output database. + +This script: +- Reads tables from an SQLite database containing ML trial results. +- Extracts feature importance values and model performance metrics. +- Computes weighted statistics (mean and standard deviation) using model performance as weights. +- Identifies the best models based on a specified metric. +- Saves feature importance and model performance data as CSV files. +- Generates plots to visualize model performance across replicates and trials. + +Author: Jan Valosek, Kalum Ost +""" + +import os +import re + +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import seaborn as sns +from sqlite3 import connect + +from sqlalchemy.dialects.mssql.information_schema import columns + +#target='AIS_change_bin' +target='UEMS_change_bin' +#target='LEMS_change_bin' +#target='AIS_change_bin_gt0' +#target='SNL_Class_initial_bin' +num_of_trials = 100 + + +def read_db(target): + """ + Read tables from the database as dataframes + :param target: target variable + :return: dictionary with the tables (dataframes) from the database + """ + con = connect(f'testing/output/output_{target}_{num_of_trials}_trials.db') + tables = pd.read_sql( + "SELECT * FROM sqlite_master", + con=con + ).loc[:, 'name'] + tables_dict = {} + for t in tables: + # Pull the dataframe from the database + try: + df = pd.read_sql( + f"SELECT * FROM {t}", + con=con + ) + tables_dict[t] = df + except: + print(f"Failed to read table {t}, ignoring it") + continue + con.close() + + return tables_dict + +def weighted_std(values: np.ndarray, weights: np.ndarray) -> float: + """ + Compute the weighted standard deviation. + :param values: Array of feature importance values. + :param weights: Array of weights (e.g., model performance scores). + :return: Weighted standard deviation. + """ + # Compute the weighted mean + weighted_mean = np.average(values, weights=weights) + # Compute the weighted variance + weighted_variance = np.average((values - weighted_mean) ** 2, weights=weights) + + # Take the square root to obtain the weighted standard deviation + return np.sqrt(weighted_variance) + +def compute_weighted_feature_importance(best_models, metric): + """ + Compute the weighted average feature importance using `importance_by_permutation (test)` + with `balanced_accuracy (test)` as the weight. + + :param best_models: DataFrame with best models selected for each replicate + :param metric: Performance metric used as weight (e.g., 'balanced_accuracy (test)') + :return: DataFrame with weighted feature importance + """ + + # Extract importance and performance metric + best_models = best_models[['model_name', 'replicate', 'trial', metric, 'importance_by_permutation (test)']] + + # Convert 'importance_by_permutation (test)' from str to dict using re + pattern = r'([\w\s\(\)<\-]+): ([\d\.]+)' # Works with names containing spaces, (), <-, and _ + best_models['importance_by_permutation (test)'] = best_models['importance_by_permutation (test)'].apply( + lambda x: {match[0].strip(): float(match[1]) for match in re.findall(pattern, x)}) + + # Convert the dictionaries contained with the feature_col dicts into dataframes which can be stacked + raw_dfs = [] + weighted_dfs = [] + for r in best_models.iterrows(): + rvals = r[1] + tmp_df = pd.DataFrame.from_dict({k: [v] for k, v in rvals['importance_by_permutation (test)'].items()}) + raw_dfs.append(tmp_df) + + # Stack the dataframes + raw_feature_imps = pd.concat(raw_dfs).fillna(0) + + # Query the weights list + weights = best_models[metric].astype('float64') + + # For each feature, calculate our desired statistics + return_cols = ['Mean', 'STD', 'Weighted Mean', 'Weighted STD'] + return_df_dict = {} + for c in raw_feature_imps.columns: + # Single query of the dataframe, as pandas can be slow w/ repeated queries + samples = raw_feature_imps[c] + # Raw Mean + c_mean = np.mean(samples) + # Raw STD + c_std = np.std(samples) + # Weighted mean + c_mean_weighted = np.average(samples, weights=weights) + # Weighted STD + c_std_weighted = weighted_std(samples, weights) + # Stack them into a list and store it in the dictionary + return_df_dict[c] = [c_mean, c_std, c_mean_weighted, c_std_weighted] + + weighted_importance_df = pd.DataFrame.from_dict(return_df_dict, columns=return_cols, orient='index') + # Sort by 'Weighted Mean' + weighted_importance_df = weighted_importance_df.sort_values('Weighted Mean', ascending=False) + + return weighted_importance_df + +def get_df_for_plotting(tables_dict, metric): + """ + Iterate over the dataframes in tables_dict and merge them into a single dataframe for plotting + :param tables_dict: dictionary with the tables (dataframes) from the database + :param metric: metric to plot; e.g., 'balanced_accuracy (test)' + :return: dataframe for plotting + """ + + os.makedirs('testing/output/csv', exist_ok=True) + fname_out = f'testing/output/csv/{target}_{metric}_best_models' + + df_plotting = pd.DataFrame(columns=['replicate', 'trial']) + + # Loop over individual models + for model_name, df in tables_dict.items(): + df_temp = df[['replicate', 'trial', metric]] + # Rename balanced_accuracy to model_name + df_temp = df_temp.rename(columns={metric: model_name}) + # Add df_temp to df_plotting based on 'replicate' and 'trial'; do not replicate the 'replicate' and 'trial' columns + df_plotting = pd.merge(df_plotting, df_temp, on=['replicate', 'trial'], how='outer') + + # Get the best model (trial) for each replicate + best_models = df.sort_values(metric, ascending=True).groupby('replicate').tail(1) + # Sort by best_models by replicate + best_models = best_models.sort_values('replicate') + + # Save metric and 'importance_by_permutation (test)' into a XLSX file; append models to the same file + # include model name as the first column + best_models.insert(0, 'model_name', model_name) + # Save the best models to a CSV file + best_models[['model_name', 'replicate', 'trial', metric, 'importance_by_permutation (test)']].to_csv( + f'{fname_out}.csv', mode='a', index=False, header=True) + + # Compute weighted average of `importance_by_permutation (test)` features, with the weight being the model's + # performance (e.g., `balanced_accuracy (test)`) + weighted_importance_df = compute_weighted_feature_importance(best_models, metric) + weighted_importance_df.insert(0, 'model_name', model_name) + # Save the weighted feature importance to a CSV file + weighted_importance_df.to_csv(f'{fname_out}_weighted_feature_importance.csv', + mode='a', index=True, header=True) + + print(f"Saved best models to {fname_out}.csv") + print(f"Saved weighted feature importance to {fname_out}_weighted_feature_importance.csv") + + # Some additional cleaning for plotting + # Sort by 'replicate' and 'trial' + df_plotting = df_plotting.sort_values(['replicate', 'trial']) + # Shorten column names (first two columns are 'replicate' and 'trial') + for column in df_plotting.columns[2:]: + df_plotting.rename(columns={column: column.replace(f'{target}__LogisticRegression__', '')}, inplace=True) + + return df_plotting + +def plotting(df_plotting, metric): + """ + Plot the balanced accuracy across replicates and trials for each model + :param df_plotting: dataframe for plotting + :param metric: metric to plot; e.g., 'balanced_accuracy (test)' + """ + + metric_title = metric.replace('_', ' ').title() # e.g., Balanced Accuracy (Test) + metric_fname = metric.replace(' ', '_').replace('(', '').replace(')', '') # e.g., balanced_accuracy_test + + # Melt the dataframe to a long format for easier plotting + df_long = df_plotting.melt( + id_vars=['replicate', 'trial'], + value_vars=df_plotting.columns[2:], # Skip 'replicate' and 'trial' + var_name='model_name', + value_name=metric + ) + + df_long[metric] = pd.to_numeric(df_long[metric]) + # agg_df = df_long.groupby(['replicate', 'model_name'])[metric].agg(['mean', 'std']).reset_index() + + os.makedirs('testing/output/plots', exist_ok=True) + + # # x-axis: replicate + # plt.figure(figsize=(10, 6)) + # #sns.lineplot(data=agg_df, x='replicate', y='mean', hue='model_name') + # sns.lineplot(data=df_long, x='replicate', y=metric, hue='model_name', errorbar='sd') + # # Customize the plot + # plt.title(f'{target} -- Mean and Std of {metric_title} Across Trials for Each Replicate') + # plt.xlabel('Replicate') + # plt.ylabel(metric_title) + # plt.legend(title='Model Name') + # # Show horizontal gridlines + # plt.grid(axis='y') + # # Show all x-ticks + # plt.xticks(df_long['replicate'].unique()) + # plt.tight_layout() + # #plt.show() + # # Save with 300 dpi + # plt.savefig(f'testing/output/plots/{target}_{metric_fname}_replicates_num_of_trials_{num_of_trials}.png', dpi=300) + # plt.close() + + # x-axis: trial + plt.figure(figsize=(10, 6)) + # sns.lineplot(data=agg_df, x='replicate', y='mean', hue='model_name') + sns.lineplot(data=df_long, x='trial', y=metric, hue='model_name', errorbar='sd') + # Customize the plot + plt.title(f'{target} -- Mean and Std of {metric_title} Across Replicates for Each Trial') + plt.xlabel('Trial') + plt.ylabel(metric_title) + plt.legend(title='Model Name') + # Show horizontal gridlines + plt.grid(axis='y') + # Make legend smaller + plt.legend(title='Model Name', fontsize='small') + # Show all x-ticks + # plt.xticks(df_long['trial'].unique()) + plt.tight_layout() + # plt.show() + # Save with 300 dpi + plt.savefig(f'testing/output/plots/{target}_{metric_fname}_trials_num_of_trials_{num_of_trials}.png', dpi=300) + print(f"Saved plots to 'plots' directory") + plt.close() + + +def main(): + # Read tables from the database as dataframes + tables_dict = read_db(target) + + #for metric in ['balanced_accuracy (test)', 'balanced_accuracy (validate)']: + for metric in ['balanced_accuracy (test)']: + # Prepare the dataframe for plotting + df_plotting = get_df_for_plotting(tables_dict, metric) + # Plot the metric across trials for each model + plotting(df_plotting, metric) + +if __name__ == '__main__': + main() + + + + From f681fad8465ad1c6338d17cc64fa0db751466cfb Mon Sep 17 00:00:00 2001 From: valosekj Date: Thu, 20 Feb 2025 11:01:10 -0500 Subject: [PATCH 05/26] Add `y_true_collector` and `y_pred_proba_collector` metrics To collect true binary labels and predicted probabilities for ROC curve generation --- study/__init__.py | 4 +++- study/metrics.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/study/__init__.py b/study/__init__.py index 96ee0ba..a12178a 100644 --- a/study/__init__.py +++ b/study/__init__.py @@ -18,5 +18,7 @@ "sk_f1_perclass": sk_f1_perclass, "importance_by_permutation": importance_by_permutation, "correct_samples": correct_samples, - "incorrect_samples": incorrect_samples + "incorrect_samples": incorrect_samples, + "y_true_collector": y_true_collector, + "y_pred_proba_collector": y_pred_proba_collector } diff --git a/study/metrics.py b/study/metrics.py index 5ba6b7e..47473ea 100644 --- a/study/metrics.py +++ b/study/metrics.py @@ -143,3 +143,15 @@ def incorrect_samples(manager: OptunaModelManager, x: BaseDataManager, y: BaseDa bad_samples = clean_val_for_db(bad_samples) return bad_samples + +""" ROC Curve """ +def y_true_collector(_: OptunaModelManager, __: BaseDataManager, y: BaseDataManager): + """ Collects the true binary labels for ROC curve generation. """ + return clean_val_for_db(list(y.as_array().flatten())) + +def y_pred_proba_collector(manager: OptunaModelManager, x: BaseDataManager, _: BaseDataManager): + """ Collects predicted probabilities for the positive class. """ + py = manager.predict_proba(x.as_array()) + if py.shape[1] != 2: + raise ValueError(f"Expected binary classification with two probability columns; found {py.shape[1]}.") + return clean_val_for_db(list(py[:, 1])) # Probabilities for the positive class \ No newline at end of file From 707fe6bff34c6bcdf1d0daa25bab44e8443e32b9 Mon Sep 17 00:00:00 2001 From: valosekj Date: Thu, 20 Feb 2025 15:49:35 -0500 Subject: [PATCH 06/26] Move getting the best replicate into a function --- inspect_output_db.py | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/inspect_output_db.py b/inspect_output_db.py index 0a2f9e8..8c41d89 100644 --- a/inspect_output_db.py +++ b/inspect_output_db.py @@ -128,27 +128,21 @@ def compute_weighted_feature_importance(best_models, metric): return weighted_importance_df -def get_df_for_plotting(tables_dict, metric): +def get_best_replicate(tables_dict, metric) -> None: """ - Iterate over the dataframes in tables_dict and merge them into a single dataframe for plotting + Get the best replicate (i.e., best performing model) for each trail based on the specified metric. + Also, compute the weighted average of `importance_by_permutation (test)` features, with the weight being + the model's performance (e.g., `balanced_accuracy (test)`). + Save the best models and weighted feature importance to CSV files. :param tables_dict: dictionary with the tables (dataframes) from the database - :param metric: metric to plot; e.g., 'balanced_accuracy (test)' - :return: dataframe for plotting + :param metric: metric to use for selecting the best models; e.g., 'balanced_accuracy (test)' """ os.makedirs('testing/output/csv', exist_ok=True) fname_out = f'testing/output/csv/{target}_{metric}_best_models' - df_plotting = pd.DataFrame(columns=['replicate', 'trial']) - # Loop over individual models for model_name, df in tables_dict.items(): - df_temp = df[['replicate', 'trial', metric]] - # Rename balanced_accuracy to model_name - df_temp = df_temp.rename(columns={metric: model_name}) - # Add df_temp to df_plotting based on 'replicate' and 'trial'; do not replicate the 'replicate' and 'trial' columns - df_plotting = pd.merge(df_plotting, df_temp, on=['replicate', 'trial'], how='outer') - # Get the best model (trial) for each replicate best_models = df.sort_values(metric, ascending=True).groupby('replicate').tail(1) # Sort by best_models by replicate @@ -172,6 +166,25 @@ def get_df_for_plotting(tables_dict, metric): print(f"Saved best models to {fname_out}.csv") print(f"Saved weighted feature importance to {fname_out}_weighted_feature_importance.csv") + +def get_df_for_plotting(tables_dict, metric) -> pd.DataFrame: + """ + Iterate over the dataframes in tables_dict and merge them into a single dataframe for plotting + :param tables_dict: dictionary with the tables (dataframes) from the database + :param metric: metric to plot; e.g., 'balanced_accuracy (test)' + :return: dataframe for plotting + """ + + df_plotting = pd.DataFrame(columns=['replicate', 'trial']) + + # Loop over individual models + for model_name, df in tables_dict.items(): + df_temp = df[['replicate', 'trial', metric]] + # Rename balanced_accuracy to model_name + df_temp = df_temp.rename(columns={metric: model_name}) + # Add df_temp to df_plotting based on 'replicate' and 'trial'; do not replicate the 'replicate' and 'trial' columns + df_plotting = pd.merge(df_plotting, df_temp, on=['replicate', 'trial'], how='outer') + # Some additional cleaning for plotting # Sort by 'replicate' and 'trial' df_plotting = df_plotting.sort_values(['replicate', 'trial']) @@ -252,6 +265,8 @@ def main(): #for metric in ['balanced_accuracy (test)', 'balanced_accuracy (validate)']: for metric in ['balanced_accuracy (test)']: + # Get the best replicate (i.e., best performing model) for each trial (train/test split) + get_best_replicate(tables_dict, metric) # Prepare the dataframe for plotting df_plotting = get_df_for_plotting(tables_dict, metric) # Plot the metric across trials for each model From c5c61a38ecbf906a6d03c3bb0ab7edb65aebc7ef Mon Sep 17 00:00:00 2001 From: valosekj Date: Thu, 20 Feb 2025 15:49:55 -0500 Subject: [PATCH 07/26] Remove empty lines --- inspect_output_db.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/inspect_output_db.py b/inspect_output_db.py index 8c41d89..a73809b 100644 --- a/inspect_output_db.py +++ b/inspect_output_db.py @@ -274,7 +274,3 @@ def main(): if __name__ == '__main__': main() - - - - From a2f59344005d8ca4a75400ad61703cc6dd61bb30 Mon Sep 17 00:00:00 2001 From: valosekj Date: Fri, 21 Feb 2025 13:21:40 -0500 Subject: [PATCH 08/26] Return `best_models_dict` by `get_best_replicate` --- inspect_output_db.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/inspect_output_db.py b/inspect_output_db.py index a73809b..fcbe181 100644 --- a/inspect_output_db.py +++ b/inspect_output_db.py @@ -128,7 +128,7 @@ def compute_weighted_feature_importance(best_models, metric): return weighted_importance_df -def get_best_replicate(tables_dict, metric) -> None: +def get_best_replicate(tables_dict, metric) -> dict: """ Get the best replicate (i.e., best performing model) for each trail based on the specified metric. Also, compute the weighted average of `importance_by_permutation (test)` features, with the weight being @@ -136,11 +136,14 @@ def get_best_replicate(tables_dict, metric) -> None: Save the best models and weighted feature importance to CSV files. :param tables_dict: dictionary with the tables (dataframes) from the database :param metric: metric to use for selecting the best models; e.g., 'balanced_accuracy (test)' + :return: dictionary of dataframes with the best models for each model """ os.makedirs('testing/output/csv', exist_ok=True) fname_out = f'testing/output/csv/{target}_{metric}_best_models' + best_models_dict = {} + # Loop over individual models for model_name, df in tables_dict.items(): # Get the best model (trial) for each replicate @@ -155,6 +158,8 @@ def get_best_replicate(tables_dict, metric) -> None: best_models[['model_name', 'replicate', 'trial', metric, 'importance_by_permutation (test)']].to_csv( f'{fname_out}.csv', mode='a', index=False, header=True) + best_models_dict[model_name] = best_models + # Compute weighted average of `importance_by_permutation (test)` features, with the weight being the model's # performance (e.g., `balanced_accuracy (test)`) weighted_importance_df = compute_weighted_feature_importance(best_models, metric) @@ -166,6 +171,8 @@ def get_best_replicate(tables_dict, metric) -> None: print(f"Saved best models to {fname_out}.csv") print(f"Saved weighted feature importance to {fname_out}_weighted_feature_importance.csv") + return best_models_dict + def get_df_for_plotting(tables_dict, metric) -> pd.DataFrame: """ @@ -266,7 +273,7 @@ def main(): #for metric in ['balanced_accuracy (test)', 'balanced_accuracy (validate)']: for metric in ['balanced_accuracy (test)']: # Get the best replicate (i.e., best performing model) for each trial (train/test split) - get_best_replicate(tables_dict, metric) + best_models_dict = get_best_replicate(tables_dict, metric) # Prepare the dataframe for plotting df_plotting = get_df_for_plotting(tables_dict, metric) # Plot the metric across trials for each model From d69a385d8bf3a6a3235a4541515b29b49443734c Mon Sep 17 00:00:00 2001 From: valosekj Date: Fri, 21 Feb 2025 13:22:42 -0500 Subject: [PATCH 09/26] Plot ROC --- inspect_output_db.py | 71 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/inspect_output_db.py b/inspect_output_db.py index fcbe181..14fa0a7 100644 --- a/inspect_output_db.py +++ b/inspect_output_db.py @@ -21,7 +21,7 @@ import seaborn as sns from sqlite3 import connect -from sqlalchemy.dialects.mssql.information_schema import columns +from sklearn.metrics import roc_curve, auc #target='AIS_change_bin' target='UEMS_change_bin' @@ -265,6 +265,69 @@ def plotting(df_plotting, metric): print(f"Saved plots to 'plots' directory") plt.close() +def extract_roc_data(best_models_dict): + """ + Extracts y_true and y_pred_proba from a dataframe. + """ + + models_roc_data = {} + + for model_name, df in best_models_dict.items(): + if "y_true_collector (test)" in df.columns and "y_pred_proba_collector (test)" in df.columns: + y_true_all = [] + y_pred_proba_all = [] + for y_true_str, y_pred_proba_str in zip(df["y_true_collector (test)"], df["y_pred_proba_collector (test)"]): + try: + y_true = np.array(eval(y_true_str)) + y_pred_proba = np.array(eval(y_pred_proba_str)) + y_true_all.extend(y_true) + y_pred_proba_all.extend(y_pred_proba) + except Exception as e: + print(f"Error parsing y_true/y_pred_proba: {e}") + continue + + models_roc_data[model_name] = (np.array(y_true_all), np.array(y_pred_proba_all)) + + return models_roc_data + +def plot_roc_curve(models_roc_data): + """ + Generates and saves a ROC curve with multiple models. + + :param models_roc_data: Dictionary where keys are model names, and values are (y_true, y_pred_proba). + """ + + plt.figure(figsize=(8, 6)) + + for model_name, (y_true, y_pred_proba) in models_roc_data.items(): + model_name = model_name.replace(f'{target}__LogisticRegression__', '') + if len(y_true) == 0 or len(y_pred_proba) == 0: + print(f"Skipping ROC for {model_name} due to missing data.") + continue + + # Compute ROC curve + fpr, tpr, _ = roc_curve(y_true, y_pred_proba) + roc_auc = auc(fpr, tpr) + + # Plot each model’s ROC curve + plt.plot(fpr, tpr, lw=2, label=f'{model_name} (AUC = {roc_auc:.2f})') + + # Plot random classifier baseline + plt.plot([0, 1], [0, 1], color='gray', linestyle='--') + + plt.xlim([0.0, 1.0]) + plt.ylim([0.0, 1.05]) + plt.xlabel('False Positive Rate') + plt.ylabel('True Positive Rate') + plt.title(f'ROC (Receiver Operating Characteristic) for {target_to_title_disct[target]}') + plt.legend(loc='lower right') + + # Save the plot + os.makedirs('testing/output/plots', exist_ok=True) + plt.savefig(f'testing/output/plots/{target}_roc_curve.png', dpi=300) + print(f"Saved ROC curve to 'testing/output/plots/{target}_roc_curve.png'") + plt.close() + def main(): # Read tables from the database as dataframes @@ -274,6 +337,12 @@ def main(): for metric in ['balanced_accuracy (test)']: # Get the best replicate (i.e., best performing model) for each trial (train/test split) best_models_dict = get_best_replicate(tables_dict, metric) + models_roc_data = extract_roc_data(best_models_dict) + if models_roc_data: + plot_roc_curve(models_roc_data) + else: + print("Skipping ROC curve generation: No valid data found.") + # Prepare the dataframe for plotting df_plotting = get_df_for_plotting(tables_dict, metric) # Plot the metric across trials for each model From adab105f6b02e090828a9a4d10852fae35a86587 Mon Sep 17 00:00:00 2001 From: valosekj Date: Fri, 21 Feb 2025 13:23:05 -0500 Subject: [PATCH 10/26] Keep only the models we are interested in --- inspect_output_db.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/inspect_output_db.py b/inspect_output_db.py index 14fa0a7..aaf13a1 100644 --- a/inspect_output_db.py +++ b/inspect_output_db.py @@ -30,6 +30,8 @@ #target='SNL_Class_initial_bin' num_of_trials = 100 +models_to_keep = ['mri_metrics_hemorrhage_included', 'all_metrics_AIS_Initial_ladder_encoded', + 'clinical_metrics_AIS_Initial_ladder_encoded'] def read_db(target): """ @@ -56,6 +58,11 @@ def read_db(target): continue con.close() + # Keep only the models we are interested in + for model_name in list(tables_dict.keys()): + if model_name.replace(f'{target}__LogisticRegression__', '') not in models_to_keep: + tables_dict.pop(model_name) + return tables_dict def weighted_std(values: np.ndarray, weights: np.ndarray) -> float: From e58a68ffcd78f5e1f3688eb5e7bf0cf74ed6bf02 Mon Sep 17 00:00:00 2001 From: valosekj Date: Fri, 21 Feb 2025 13:23:45 -0500 Subject: [PATCH 11/26] Improve figure titles --- inspect_output_db.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/inspect_output_db.py b/inspect_output_db.py index aaf13a1..12b15b9 100644 --- a/inspect_output_db.py +++ b/inspect_output_db.py @@ -33,6 +33,14 @@ models_to_keep = ['mri_metrics_hemorrhage_included', 'all_metrics_AIS_Initial_ladder_encoded', 'clinical_metrics_AIS_Initial_ladder_encoded'] +target_to_title_dict = { + 'AIS_change_bin': 'AIS Change', + 'UEMS_change_bin': 'UEMS Change', + 'LEMS_change_bin': 'LEMS Change', + 'AIS_change_bin_gt0': 'AIS Change' +} + + def read_db(target): """ Read tables from the database as dataframes @@ -255,7 +263,7 @@ def plotting(df_plotting, metric): # sns.lineplot(data=agg_df, x='replicate', y='mean', hue='model_name') sns.lineplot(data=df_long, x='trial', y=metric, hue='model_name', errorbar='sd') # Customize the plot - plt.title(f'{target} -- Mean and Std of {metric_title} Across Replicates for Each Trial') + plt.title(f'{target_to_title_dict[target]}: Mean and Std of {metric_title} Across Replicates for Each Trial') plt.xlabel('Trial') plt.ylabel(metric_title) plt.legend(title='Model Name') @@ -326,7 +334,7 @@ def plot_roc_curve(models_roc_data): plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') - plt.title(f'ROC (Receiver Operating Characteristic) for {target_to_title_disct[target]}') + plt.title(f'ROC (Receiver Operating Characteristic) for {target_to_title_dict[target]}') plt.legend(loc='lower right') # Save the plot From c14c7aa190d88c57326cf86798761766e1657e65 Mon Sep 17 00:00:00 2001 From: valosekj Date: Fri, 21 Feb 2025 13:57:38 -0500 Subject: [PATCH 12/26] =?UTF-8?q?Plot=20the=20mean=20=C2=B1=20std=20ROC=20?= =?UTF-8?q?curve=20for=20each=20model=20across=20replicates.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inspect_output_db.py | 50 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/inspect_output_db.py b/inspect_output_db.py index 12b15b9..38eda74 100644 --- a/inspect_output_db.py +++ b/inspect_output_db.py @@ -344,6 +344,53 @@ def plot_roc_curve(models_roc_data): plt.close() +def plot_mean_std_roc_curve(best_models_dict): + """ + Plot the mean ± std ROC curve for each model across replicates. + :param best_models_dict: dictionary with the best models for each model + """ + + plt.figure(figsize=(8, 6)) + mean_fpr = np.linspace(0, 1, 100) + + for model_name, df in best_models_dict.items(): + model_name_cleaned = model_name.split('__')[-1] + tprs = [] + + # Extract ROC data for each replicate + for _, row in df.iterrows(): + y_true = np.array(eval(row["y_true_collector (test)"])) + y_pred_proba = np.array(eval(row["y_pred_proba_collector (test)"])) + + fpr, tpr, _ = roc_curve(y_true, y_pred_proba) + tprs.append(np.interp(mean_fpr, fpr, tpr)) + + # Calculate mean and std of TPRs + mean_tpr = np.mean(tprs, axis=0) + std_tpr = np.std(tprs, axis=0) + roc_auc = auc(mean_fpr, mean_tpr) + + # Plot mean ROC curve with std deviation + plt.plot(mean_fpr, mean_tpr, label=f'{model_name_cleaned} (AUC = {roc_auc:.2f} ± {std_tpr.mean():.2f})') + plt.fill_between(mean_fpr, mean_tpr - std_tpr, mean_tpr + std_tpr, alpha=0.2) + + # Plot baseline + plt.plot([0, 1], [0, 1], color='gray', linestyle='--') + plt.xlabel('False Positive Rate') + plt.ylabel('True Positive Rate') + plt.title(f'Mean ± Std ROC Curve for {target_to_title_dict[target]}') + plt.legend(loc='lower right') + plt.grid(True) + plt.tight_layout() + #plt.show() + + # Save the plot + os.makedirs('testing/output/plots', exist_ok=True) + plt.savefig(f'testing/output/plots/{target}_roc_curve_mean.png', dpi=300) + print(f"Saved ROC curve to 'testing/output/plots/{target}_roc_curve_mean.png'") + plt.close() + + def main(): # Read tables from the database as dataframes tables_dict = read_db(target) @@ -352,6 +399,9 @@ def main(): for metric in ['balanced_accuracy (test)']: # Get the best replicate (i.e., best performing model) for each trial (train/test split) best_models_dict = get_best_replicate(tables_dict, metric) + + plot_mean_std_roc_curve(best_models_dict) + models_roc_data = extract_roc_data(best_models_dict) if models_roc_data: plot_roc_curve(models_roc_data) From c55c2f4d48e6069ee328c6b181fd86899f432400 Mon Sep 17 00:00:00 2001 From: valosekj Date: Fri, 21 Feb 2025 16:26:56 -0500 Subject: [PATCH 13/26] Unify titles across figures --- inspect_output_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inspect_output_db.py b/inspect_output_db.py index 38eda74..3f53cf9 100644 --- a/inspect_output_db.py +++ b/inspect_output_db.py @@ -334,7 +334,7 @@ def plot_roc_curve(models_roc_data): plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') - plt.title(f'ROC (Receiver Operating Characteristic) for {target_to_title_dict[target]}') + plt.title(f'{target_to_title_dict[target]}: ROC (Receiver Operating Characteristic)') plt.legend(loc='lower right') # Save the plot @@ -378,7 +378,7 @@ def plot_mean_std_roc_curve(best_models_dict): plt.plot([0, 1], [0, 1], color='gray', linestyle='--') plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') - plt.title(f'Mean ± Std ROC Curve for {target_to_title_dict[target]}') + plt.title(f'{target_to_title_dict[target]}: Mean ± Std ROC Curve') plt.legend(loc='lower right') plt.grid(True) plt.tight_layout() From 0b3ad3d5ee00117865269f4d7ad3b448d2af7e1f Mon Sep 17 00:00:00 2001 From: valosekj Date: Mon, 24 Feb 2025 10:48:57 -0500 Subject: [PATCH 14/26] include metric (e.g., 'balanced_accuracy (test)' or 'balanced_accuracy (validate)') to ROC figure fnames --- inspect_output_db.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/inspect_output_db.py b/inspect_output_db.py index 3f53cf9..f0103c6 100644 --- a/inspect_output_db.py +++ b/inspect_output_db.py @@ -305,13 +305,14 @@ def extract_roc_data(best_models_dict): return models_roc_data -def plot_roc_curve(models_roc_data): +def plot_roc_curve(models_roc_data, metric): """ Generates and saves a ROC curve with multiple models. - :param models_roc_data: Dictionary where keys are model names, and values are (y_true, y_pred_proba). """ + metric_fname = metric.replace(' ', '_').replace('(', '').replace(')', '') # e.g., balanced_accuracy_test + plt.figure(figsize=(8, 6)) for model_name, (y_true, y_pred_proba) in models_roc_data.items(): @@ -338,18 +339,22 @@ def plot_roc_curve(models_roc_data): plt.legend(loc='lower right') # Save the plot + fname_figure = f'testing/output/plots/{target}_{metric_fname}_roc_curve.png' os.makedirs('testing/output/plots', exist_ok=True) - plt.savefig(f'testing/output/plots/{target}_roc_curve.png', dpi=300) - print(f"Saved ROC curve to 'testing/output/plots/{target}_roc_curve.png'") + plt.savefig(fname_figure, dpi=300) + print(f"Saved ROC curve to {fname_figure}") plt.close() -def plot_mean_std_roc_curve(best_models_dict): +def plot_mean_std_roc_curve(best_models_dict, metric): """ Plot the mean ± std ROC curve for each model across replicates. :param best_models_dict: dictionary with the best models for each model + :param metric: metric to use for selecting the best models; e.g., 'balanced_accuracy (test)' """ + metric_fname = metric.replace(' ', '_').replace('(', '').replace(')', '') # e.g., balanced_accuracy_test + plt.figure(figsize=(8, 6)) mean_fpr = np.linspace(0, 1, 100) @@ -386,8 +391,9 @@ def plot_mean_std_roc_curve(best_models_dict): # Save the plot os.makedirs('testing/output/plots', exist_ok=True) - plt.savefig(f'testing/output/plots/{target}_roc_curve_mean.png', dpi=300) - print(f"Saved ROC curve to 'testing/output/plots/{target}_roc_curve_mean.png'") + fname_figure = f'testing/output/plots/{target}_{metric_fname}_roc_curve_mean.png' + plt.savefig(fname_figure, dpi=300) + print(f"Saved ROC curve to {fname_figure}") plt.close() @@ -400,11 +406,11 @@ def main(): # Get the best replicate (i.e., best performing model) for each trial (train/test split) best_models_dict = get_best_replicate(tables_dict, metric) - plot_mean_std_roc_curve(best_models_dict) + plot_mean_std_roc_curve(best_models_dict, metric) models_roc_data = extract_roc_data(best_models_dict) if models_roc_data: - plot_roc_curve(models_roc_data) + plot_roc_curve(models_roc_data, metric) else: print("Skipping ROC curve generation: No valid data found.") From 917f4ab14c7089a385c313c1805b0d3e11d3c2e6 Mon Sep 17 00:00:00 2001 From: valosekj Date: Wed, 2 Apr 2025 16:10:47 -0400 Subject: [PATCH 15/26] Add max_iter parameter to log_reg configuration --- testing/model_configs/log_reg.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/testing/model_configs/log_reg.json b/testing/model_configs/log_reg.json index e603d22..f27af1a 100644 --- a/testing/model_configs/log_reg.json +++ b/testing/model_configs/log_reg.json @@ -8,6 +8,12 @@ "choices": ["l1", "l2", "elasticnet", null] }, "solver": "saga", + "max_iter": { + "label": "max_iter", + "type": "int", + "low": 100, + "high": 2000 + }, "l1_c": { "label": "l1", "type": "float", From 928df4ad40ce071caf3dfda57984a118b7d3fdc2 Mon Sep 17 00:00:00 2001 From: valosekj Date: Thu, 10 Apr 2025 11:45:05 -0400 Subject: [PATCH 16/26] Add proper argparse --- inspect_output_db.py | 49 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/inspect_output_db.py b/inspect_output_db.py index f0103c6..ec26a98 100644 --- a/inspect_output_db.py +++ b/inspect_output_db.py @@ -14,6 +14,7 @@ import os import re +import argparse import numpy as np import pandas as pd @@ -40,14 +41,45 @@ 'AIS_change_bin_gt0': 'AIS Change' } +def get_parser(): + """ + parser function + """ + + parser = argparse.ArgumentParser( + description='Script for inspecting and analyzing the output database.', + prog=os.path.basename(__file__).strip('.py') + ) + parser.add_argument( + '-i', + metavar="FILE_NAME", + required=True, + type=str, + help='Absolute path to the SQLite database. ' + 'Example: /target_AIS_change_bin_1.db' + ) + parser.add_argument( + '-o', + metavar='DIR_NAME', + required=True, + type=str, + help='Absolute path to the output directory where the results (figures and CSV files) will be saved. ' + ) + + return parser -def read_db(target): + +def read_db(fname_path: str) -> dict: """ Read tables from the database as dataframes - :param target: target variable + :param fname_path: path to the SQLite database :return: dictionary with the tables (dataframes) from the database """ - con = connect(f'testing/output/output_{target}_{num_of_trials}_trials.db') + assert os.path.exists(fname_path), f"Database file not found: {fname_path}" + con = connect(fname_path) + + print('Reading tables...') + tables = pd.read_sql( "SELECT * FROM sqlite_master", con=con @@ -398,8 +430,17 @@ def plot_mean_std_roc_curve(best_models_dict, metric): def main(): + + # Parse the command line arguments + parser = get_parser() + args = parser.parse_args() + + fname_path = os.path.abspath(args.i) + output_path = os.path.abspath(args.o) + os.makedirs(output_path, exist_ok=True) + # Read tables from the database as dataframes - tables_dict = read_db(target) + tables_dict = read_db(fname_path) #for metric in ['balanced_accuracy (test)', 'balanced_accuracy (validate)']: for metric in ['balanced_accuracy (test)']: From bedb126a1b35f5503e18e65e659ce0ea7a1e6709 Mon Sep 17 00:00:00 2001 From: valosekj Date: Thu, 10 Apr 2025 11:47:22 -0400 Subject: [PATCH 17/26] Get unique targets dynamically and iterate over them --- inspect_output_db.py | 49 ++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/inspect_output_db.py b/inspect_output_db.py index ec26a98..f276ede 100644 --- a/inspect_output_db.py +++ b/inspect_output_db.py @@ -233,11 +233,13 @@ def get_df_for_plotting(tables_dict, metric) -> pd.DataFrame: # Loop over individual models for model_name, df in tables_dict.items(): - df_temp = df[['replicate', 'trial', metric]] - # Rename balanced_accuracy to model_name - df_temp = df_temp.rename(columns={metric: model_name}) - # Add df_temp to df_plotting based on 'replicate' and 'trial'; do not replicate the 'replicate' and 'trial' columns - df_plotting = pd.merge(df_plotting, df_temp, on=['replicate', 'trial'], how='outer') + if target in model_name: + # Get only the columns we need for plotting + df_temp = df[['replicate', 'trial', metric]] + # Rename balanced_accuracy to model_name + df_temp = df_temp.rename(columns={metric: model_name}) + # Add df_temp to df_plotting based on 'replicate' and 'trial'; do not replicate the 'replicate' and 'trial' columns + df_plotting = pd.merge(df_plotting, df_temp, on=['replicate', 'trial'], how='outer') # Some additional cleaning for plotting # Sort by 'replicate' and 'trial' @@ -442,23 +444,26 @@ def main(): # Read tables from the database as dataframes tables_dict = read_db(fname_path) - #for metric in ['balanced_accuracy (test)', 'balanced_accuracy (validate)']: - for metric in ['balanced_accuracy (test)']: - # Get the best replicate (i.e., best performing model) for each trial (train/test split) - best_models_dict = get_best_replicate(tables_dict, metric) - - plot_mean_std_roc_curve(best_models_dict, metric) - - models_roc_data = extract_roc_data(best_models_dict) - if models_roc_data: - plot_roc_curve(models_roc_data, metric) - else: - print("Skipping ROC curve generation: No valid data found.") - - # Prepare the dataframe for plotting - df_plotting = get_df_for_plotting(tables_dict, metric) - # Plot the metric across trials for each model - plotting(df_plotting, metric) + # Get unique targets + targets = set([item.split('__')[0] for item in tables_dict.keys()]) + + for target in targets: + #for metric in ['balanced_accuracy (test)', 'balanced_accuracy (validate)']: + for metric in ['balanced_accuracy (validate)']: + # Get the best replicate (i.e., best performing model) for each trial (train/test split) + best_models_dict = get_best_replicate(tables_dict, metric) + plot_mean_std_roc_curve(best_models_dict, metric) + + models_roc_data = extract_roc_data(best_models_dict) + if models_roc_data: + plot_roc_curve(models_roc_data, metric) + else: + print("Skipping ROC curve generation: No valid data found.") + + # Prepare the dataframe for plotting + df_plotting = get_df_for_plotting(tables_dict, metric, target) + # Plot the metric across trials for each model + plotting(df_plotting, metric, target, output_path) if __name__ == '__main__': main() From 8027b667b03e4d6e886ee4cbc62bd18097d2c300 Mon Sep 17 00:00:00 2001 From: Kalum Ost Date: Wed, 17 Sep 2025 18:31:49 -0400 Subject: [PATCH 18/26] Initial commit; added SHAP value calculation to the list of available metrics. --- environment.yml | 1 + study/__init__.py | 1 + study/metrics.py | 40 +++++++++++++++++++++++++++++++ testing/testing_study_config.json | 3 ++- 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index d17cc45..af6704e 100644 --- a/environment.yml +++ b/environment.yml @@ -9,3 +9,4 @@ dependencies: - scikit-learn - pandas - pytest + - shap diff --git a/study/__init__.py b/study/__init__.py index 96ee0ba..aacbcfe 100644 --- a/study/__init__.py +++ b/study/__init__.py @@ -17,6 +17,7 @@ "sk_f1_weighted_avg": sk_f1_weighted_avg, "sk_f1_perclass": sk_f1_perclass, "importance_by_permutation": importance_by_permutation, + "shap_additive": shap_additive, "correct_samples": correct_samples, "incorrect_samples": incorrect_samples } diff --git a/study/metrics.py b/study/metrics.py index 6f6defb..e4c539b 100644 --- a/study/metrics.py +++ b/study/metrics.py @@ -2,6 +2,7 @@ Metric-reporting closures for use in this framework. """ import numpy as np +import shap from sklearn.inspection import permutation_importance from sklearn.metrics import balanced_accuracy_score, log_loss, roc_auc_score, precision_score, recall_score, f1_score @@ -101,6 +102,45 @@ def importance_by_permutation(manager: OptunaModelManager, x: BaseDataManager, y importance_vals = clean_val_for_db(importance_vals) return importance_vals +def shap_additive(manager: OptunaModelManager, x: BaseDataManager, _: BaseDataManager): + """ + To restore the (raw) values for a given run, run the following snippet: + + ``` + from io import StringIO + + shap_entry = ... # Load the value you want from the DB; it will be a string + with StringIO(val) as sp: + shap_vals = np.loadtxt(sp) + ``` + + `shap_vals` will then be a Numpy array, of size (n,c), where + * n is the number of samples in the testing dataset, and + * c is the number of features the model was trying to predict + + For categorical targets with more than 2 classes, each class is treated as + unique feature by SHAP for the purpose of calculating SHAP values. + + TODO: Save the `shap_values` directly via pickle into a SQLite blob + """ + # Initialize the explainer, using the x data as both the mask and feature list + x_arr = x.as_array() + explainer = shap.Explainer( + manager.get_model(), masker=x_arr, feature_names=x.features() + ) + + # Calculate the Shapley values from this dataset + shap_values = explainer(x_arr) + + # Keep only the "primary" SHAP values, convert them to a string + val_str = str(shap_values.values) + + # Remove the brackets; despite Numpy adding them, it cannot parse them after... + val_str = val_str.replace("[", "").replace("]", "") + + # Return the result to be saved + return val_str + """ Sample Reporting """ def correct_samples(manager: OptunaModelManager, x: BaseDataManager, y: BaseDataManager): diff --git a/testing/testing_study_config.json b/testing/testing_study_config.json index 8527b11..836e5b8 100644 --- a/testing/testing_study_config.json +++ b/testing/testing_study_config.json @@ -21,7 +21,8 @@ "sk_f1_perclass", "correct_samples", "incorrect_samples", - "importance_by_permutation" + "importance_by_permutation", + "shap_additive" ] }, "track_params": true, From 6c6d3e92d97d61681964d20ab37c9b175c789b54 Mon Sep 17 00:00:00 2001 From: Kalum Ost Date: Tue, 7 Oct 2025 02:28:40 -0400 Subject: [PATCH 19/26] Swapped SHAP values from list to dict (bound by feature name) --- study/metrics.py | 58 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 12 deletions(-) diff --git a/study/metrics.py b/study/metrics.py index 755bb74..fd9ef72 100644 --- a/study/metrics.py +++ b/study/metrics.py @@ -1,6 +1,8 @@ """ Metric-reporting closures for use in this framework. """ +import sys + import numpy as np import shap from sklearn.inspection import permutation_importance @@ -110,14 +112,32 @@ def shap_additive(manager: OptunaModelManager, x: BaseDataManager, _: BaseDataMa ``` from io import StringIO - shap_entry = ... # Load the value you want from the DB; it will be a string - with StringIO(val) as sp: - shap_vals = np.loadtxt(sp) + # You can omit the Numpy import if you manually + # parse the inner string in the list comp + import numpy as np + + # This is the SHAP value within the database you want to parse + val = ... + + # Strip the brackets first + val = val.strip("{").strip("}") + # Split by commas + entry_strs = val.split(", ") + # Split the dataset by feature name + shap_map = dict() + for entry_str in entry_strs: + # Split the text along the colon to get the feature label back + feature_label, shap_value_str = entry_str.split(": ") + # Parse the shap value string back into numeric form + shap_vals = [list(np.fromstring(x, sep=" ")) for x in shap_value_str.split("\n")] + # Add it to the map + shap_map[feature_label] = shap_vals ``` - `shap_vals` will then be a Numpy array, of size (n,c), where - * n is the number of samples in the testing dataset, and - * c is the number of features the model was trying to predict + Each entry in `shap_map` will be Numpy array of with the following dimensions: + * n is the number of samples in the input dataset (train, validate, or test), and + * c is the number of categorical classes used during training; + If this is binary classification, or a continuous target, c=1. For categorical targets with more than 2 classes, each class is treated as unique feature by SHAP for the purpose of calculating SHAP values. @@ -133,14 +153,28 @@ def shap_additive(manager: OptunaModelManager, x: BaseDataManager, _: BaseDataMa # Calculate the Shapley values from this dataset shap_values = explainer(x_arr) - # Keep only the "primary" SHAP values, convert them to a string - val_str = str(shap_values.values) - - # Remove the brackets; despite Numpy adding them, it cannot parse them after... - val_str = val_str.replace("[", "").replace("]", "") + shap_list = list() + for i, v in enumerate(shap_values.feature_names): + # SHAP auto-reduces the shape of its features if it is targeting + # a binary classification OR a continuous metric + if len(shap_values.values.shape) < 3: + val_str = np.array2string(shap_values.values[:, i], max_line_width=sys.maxsize, threshold=sys.maxsize) + else: + val_str = np.array2string(shap_values.values[:, i, :], max_line_width=sys.maxsize, threshold=sys.maxsize) + # Remove the brackets; despite Numpy adding them, it cannot parse them after... + val_str = val_str.replace("[", "").replace("]", "") + val_str = f"{v}: {val_str}" + shap_list.append(val_str) + + # This nonsense is required because Python maps + # "/n" to "//n" if you string convert a dict; + # why the hell does it do that?!?!? + full_str = "{" + full_str += ", ".join(shap_list) + full_str += "}" # Return the result to be saved - return val_str + return full_str """ Sample Reporting """ From 54cbe33454fc8c2641beaa71a82d7e74c09fabd4 Mon Sep 17 00:00:00 2001 From: Kalum Ost Date: Sun, 23 Nov 2025 03:22:44 -0500 Subject: [PATCH 20/26] Fixed error when a model cannot natively be parsed by SHAP. --- study/metrics.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/study/metrics.py b/study/metrics.py index fd9ef72..09d1b23 100644 --- a/study/metrics.py +++ b/study/metrics.py @@ -146,12 +146,24 @@ def shap_additive(manager: OptunaModelManager, x: BaseDataManager, _: BaseDataMa """ # Initialize the explainer, using the x data as both the mask and feature list x_arr = x.as_array() - explainer = shap.Explainer( - manager.get_model(), masker=x_arr, feature_names=x.features() - ) - - # Calculate the Shapley values from this dataset - shap_values = explainer(x_arr) + model = manager.get_model() + try: + # Default to the "generic" explainer + explainer = shap.Explainer( + model, x_arr, feature_names=x.features() + ) + # Calculate the Shapley values from this dataset + shap_values = explainer(x_arr) + except TypeError as err: + # If that failed, try to use the model's "predict" function instead + if hasattr(model, "predict"): + explainer = shap.Explainer( + model.predict, x_arr, feature_names=x.features() + ) + # Calculate the Shapley values from this dataset + shap_values = explainer(x_arr) + else: + raise err shap_list = list() for i, v in enumerate(shap_values.feature_names): From 9201bd4174ce9e1c2a2222bf054e9faa1dfd9f63 Mon Sep 17 00:00:00 2001 From: Kalum Ost Date: Wed, 26 Nov 2025 16:15:40 -0500 Subject: [PATCH 21/26] Added new "VarianceDrop" data hook, allowing low-variance features to be dropped as part of a trial's run. --- data/hooks/__init__.py | 2 +- data/hooks/feature_selection.py | 77 ++++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/data/hooks/__init__.py b/data/hooks/__init__.py index d21af55..ac19216 100644 --- a/data/hooks/__init__.py +++ b/data/hooks/__init__.py @@ -30,7 +30,7 @@ def _decorator(cls: Type[DataHook]): # TODO: Find a more elegant way to do this from data.hooks.feature_selection import ( SampleNullityDrop, FeatureNullityDrop, ExplicitDrop, ExplicitKeep, PrincipalComponentAnalysis, - RecursiveFeatureElimination + RecursiveFeatureElimination, VarianceDrop ) from data.hooks.imputation import SimpleImputation from data.hooks.encoding import OneHotEncoding diff --git a/data/hooks/feature_selection.py b/data/hooks/feature_selection.py index 9d1091a..4632f87 100644 --- a/data/hooks/feature_selection.py +++ b/data/hooks/feature_selection.py @@ -6,7 +6,7 @@ import pandas as pd from optuna import Trial from sklearn.decomposition import PCA -from sklearn.feature_selection import RFE +from sklearn.feature_selection import RFE, VarianceThreshold from sklearn.linear_model import LogisticRegression from config.utils import default_as, is_float, is_list, parse_data_config_entry @@ -135,6 +135,81 @@ def run(self, x: BaseDataManager, y: Optional[BaseDataManager] = None) -> BaseDa return x.drop_features(drop_idx) +## Feature selection by homogeneity +@registered_data_hook("drop_low_variance") +class VarianceDrop(FittedDataHook): + """ + Thin wrapper for SciKit-Learn's VarianceThreshold class, for use as a data hook within MOOP. + + Runs additional checks on top of the default implementation provided by SciKit-Learn: + * Ensures the resulting dataset always contains at least 1 feature. + + Example usage: + { + "type": "drop_low_variance", + "threshold": 0.1 + } + """ + def __init__(self, config, **kwargs): + # TODO: make this tunable + super().__init__(config, **kwargs) + + # Get the variance threshold + threshold = parse_data_config_entry( + "threshold", config, + default_as(0.0, self.logger), is_float(self.logger) + ) + + # Build the wrapped VarianceThreshold object + self.threshold = threshold + self.selected_features: list[str] | None = None + + @classmethod + def from_config(cls, config: dict, logger: Logger = Logger.root) -> Self: + return cls(config=config, logger=logger) + + def run(self, x: BaseDataManager, y: Optional[BaseDataManager] = None) -> BaseDataManager: + # If x contains only one feature already, just return that feature, as RFE has a stroke otherwise + if x.n_features() == 1: + self.logger.warning("Only one feature in the dataset was found; " + "dropping any further would result in a null dataset." + "Original (unmodified) dataset returned instead.") + return x + + # Fit the model to the dataset + vt = VarianceThreshold(threshold=self.threshold) + vt.fit(x.as_array(), np.ravel(y.as_array())) # Ravel prevents some warning spam + + # Select only the features with variance less than the threshold + self.selected_features = vt.get_feature_names_out(x.features()) + + # Ensure that at least one feature was kept + if self.selected_features.shape[0] < 1: + # Find the + highest_var = np.max(vt.variances_) + highest_var_feature = list(x.features())[np.argmax(vt.variances_)] + self.selected_features = [highest_var_feature] + self.logger.warning( + f"Low-variance filter almost dropped all features; kept highest variance " + f"feature ({highest_var_feature}, variance {highest_var}) alone to prevent crash!" + ) + + # Return the copy of x containing only these features + x_out = x.get_features(self.selected_features) + return x_out + + def run_fitted(self, x_train: BaseDataManager, x_test: Optional[BaseDataManager], + y_train: Optional[BaseDataManager] = None, y_test: Optional[BaseDataManager] = None) -> \ + tuple[BaseDataManager, BaseDataManager]: + # Run the fitted analysis first + train_out = self.run(x_train, y_train) + + # Use the same set of features to filter the x_test set + test_out = x_test.get_features(self.selected_features) + + return train_out, test_out + + ### Principal Component Analysis ### @registered_data_hook("principal_component_analysis") class PrincipalComponentAnalysis(Tunable, FittedDataHook): From 704c50afedc4a919cbea21f0db7d9209cb83c49c Mon Sep 17 00:00:00 2001 From: Kalum Ost Date: Wed, 26 Nov 2025 16:16:05 -0500 Subject: [PATCH 22/26] Updated iris testing dataset + config with new encoders. --- testing/iris_data/iris_config.json | 20 +- testing/iris_data/iris_testing.tsv | 302 ++++++++++++++--------------- 2 files changed, 170 insertions(+), 152 deletions(-) diff --git a/testing/iris_data/iris_config.json b/testing/iris_data/iris_config.json index 117ac40..5552d72 100644 --- a/testing/iris_data/iris_config.json +++ b/testing/iris_data/iris_config.json @@ -15,9 +15,14 @@ } ], "post_split_hooks": [ + { + "type": "imputation_simple", + "strategy": "most_frequent", + "features": ["color", "flower_category", "is_flower", "size"] + }, { "type": "one_hot_encode", - "features": ["color", "flower_category"] + "features": ["color", "flower_category", "is_flower"] }, { "type": "ladder_encode", @@ -33,6 +38,19 @@ "type": "standard_scaling", "run_per_cross": true }, + { + "type": "drop_low_variance", + "threshold": 0.0 + }, + { + "type": "principal_component_analysis", + "proportion": { + "label": "pca_feature_proportion", + "type": "float", + "low": 0.1, + "high": 0.9 + } + }, { "type": "recursive_feature_elimination", "proportion": { diff --git a/testing/iris_data/iris_testing.tsv b/testing/iris_data/iris_testing.tsv index 9c4335e..68c574f 100644 --- a/testing/iris_data/iris_testing.tsv +++ b/testing/iris_data/iris_testing.tsv @@ -1,151 +1,151 @@ -id sepal length (cm) sepal width (cm) petal length (cm) petal width (cm) color target size flower_category -1a 5.1 3.5 1.4 0.2 white 0 medium small-flower -2 4.9 3 1.4 0.2 white 0 medium small-flower -4 4.7 3.2 1.3 white 0 large large-flower -5 4.6 3.1 1.5 white 0 large large-flower -7b 5 3.6 0.2 white 0 medium small-flower -8 5.4 3.9 1.7 0.4 white 0 medium small-flower -10 4.6 1.4 0.3 white 0 large large-flower -11 5 3.4 1.5 0.2 white 0 small small-flower -12 4.4 2.9 1.4 0.2 white 0 small small-flower -13 4.9 3.1 1.5 0.1 white 0 medium small-flower -14 5.4 3.7 1.5 0.2 white 0 medium small-flower -15 4.8 3.4 1.6 0.2 pink 0 large large-flower -17 4.8 3 1.4 0.1 white 0 medium small-flower -18 1.1 0.1 white 0 medium small-flower -19 5.8 4 1.2 0.2 white 0 small small-flower -20 5.7 4.4 1.5 0.4 pink 0 medium small-flower -21 5.4 3.9 1.3 0.4 white 0 large large-flower -23 5.1 3.5 1.4 0.3 white 0 small small-flower -25 5.7 3.8 1.7 0.3 pink 0 large large-flower -26 5.1 3.8 1.5 0.3 white 0 large large-flower -27 5.4 3.4 1.7 0.2 white 0 small small-flower -29 5.1 3.7 1.5 0.4 white 0 large large-flower -31 4.6 3.6 1 0.2 white 0 small small-flower -33 5.1 3.3 1.7 0.5 pink 0 large large-flower -35 4.8 3.4 1.9 0.2 white 0 medium small-flower -37 5 3 1.6 0.2 white 0 large large-flower -38 5 3.4 1.6 0.4 white 0 small small-flower -40 5.2 3.5 1.5 0.2 white 0 large large-flower -41 5.2 3.4 1.4 0.2 white 0 medium small-flower -42 4.7 3.2 1.6 0.2 pink 0 large large-flower -43 4.8 1.6 0.2 white 0 small small-flower -45 5.4 3.4 1.5 0.4 white 0 small small-flower -47 5.2 4.1 1.5 0.1 white 0 small small-flower -48 5.5 4.2 1.4 0.2 white 0 small small-flower -50 4.9 1.5 0.2 pink 0 small small-flower -51 5 3.2 0.2 pink 0 small small-flower -53 5.5 3.5 1.3 0.2 white 0 small small-flower -54 4.9 3.6 1.4 0.1 white 0 large large-flower -55 4.4 3 1.3 0.2 white 0 medium small-flower -56 5.1 3.4 1.5 0.2 white 0 small small-flower -57 5 3.5 1.3 0.3 0 large large-flower -59 4.5 2.3 1.3 0.3 white 0 medium small-flower -61 4.4 3.2 1.3 0.2 0 medium small-flower -62 5 3.5 1.6 0.6 white 0 small small-flower -63 5.1 3.8 1.9 0.4 pink 0 medium small-flower -65 4.8 3 1.4 white 0 large large-flower -66 5.1 3.8 1.6 0.2 white 0 large large-flower -68 4.6 3.2 1.4 0.2 white 0 large large-flower -70 5.3 3.7 1.5 0.2 white 0 small small-flower -71 5 3.3 1.4 0.2 white 0 large large-flower -72 7 3.2 4.7 1.4 pink 1 medium small-flower -74 6.4 3.2 4.5 1.5 pink 1 small small-flower -76 6.9 3.1 4.9 1.5 pink 1 medium small-flower -78 5.5 2.3 4 1.3 pink 1 large large-flower -80 6.5 2.8 4.6 1.5 pink 1 large large-flower -81 5.7 2.8 4.5 1.3 purple 1 large large-flower -82 3.3 4.7 1.6 pink 1 small small-flower -84 4.9 2.4 3.3 1 pink 1 small small-flower -85 2.9 4.6 1.3 pink 1 medium small-flower -86 5.2 2.7 3.9 1.4 purple 1 large large-flower -88 5 2 3.5 1 purple 1 large large-flower -90 5.9 3 4.2 pink 1 medium small-flower -91 6 2.2 4 1 pink 1 medium small-flower -93 6.1 2.9 4.7 1.4 pink 1 small small-flower -95 5.6 2.9 3.6 pink 1 large large-flower -97 6.7 3.1 4.4 1.4 1 large large-flower -99 5.6 3 4.5 1.5 pink 1 medium small-flower -101 5.8 2.7 1 pink 1 large large-flower -102 6.2 2.2 4.5 1.5 pink 1 large large-flower -104 5.6 2.5 3.9 1.1 purple 1 large large-flower -105 5.9 3.2 4.8 1.8 pink 1 small small-flower -106 6.1 2.8 4 1.3 pink 1 medium small-flower -108 6.3 2.5 1.5 pink 1 large large-flower -110 6.1 2.8 4.7 1.2 pink 1 small small-flower -112 6.4 2.9 4.3 1.3 pink 1 medium small-flower -113 6.6 3 4.4 1.4 purple 1 small small-flower -115 6.8 2.8 4.8 1.4 purple 1 medium small-flower -116 6.7 3 5 1.7 purple 1 small small-flower -118 6 2.9 4.5 1.5 pink 1 medium small-flower -120 2.6 1 pink 1 large large-flower -122 5.5 2.4 3.8 1.1 pink 1 large large-flower -124 5.5 2.4 3.7 1 pink 1 large large-flower -125 5.8 2.7 3.9 1.2 1 small small-flower -127 2.7 5.1 1.6 purple 1 small small-flower -128 5.4 3 4.5 1.5 pink 1 medium small-flower -129 6 3.4 4.5 1.6 pink 1 large large-flower -130 6.7 3.1 4.7 1.5 purple 1 medium small-flower -131 6.3 4.4 1.3 pink 1 small small-flower -133 5.6 3 4.1 1.3 pink 1 medium small-flower -135 5.5 2.5 4 1.3 pink 1 large large-flower -136 5.5 2.6 4.4 1.2 purple 1 small small-flower -138 3 4.6 1.4 pink 1 medium small-flower -140 5.8 2.6 4 1.2 pink 1 large large-flower -142 5 2.3 3.3 1 pink 1 medium small-flower -143 5.6 2.7 4.2 1.3 pink 1 medium small-flower -144 5.7 3 4.2 1.2 purple 1 medium small-flower -146 5.7 2.9 4.2 1.3 pink 1 medium small-flower -148 6.2 2.9 4.3 1.3 pink 1 large large-flower -150 2.5 3 1.1 pink 1 large large-flower -151 5.7 2.8 4.1 1.3 pink 1 small small-flower -153 6.3 3.3 6 2.5 2 medium small-flower -154 5.8 1.9 purple 2 small small-flower -156 7.1 3 5.9 2.1 pink 2 small small-flower -158 6.3 2.9 5.6 1.8 purple 2 small small-flower -160 6.5 3 2.2 purple 2 large large-flower -161 7.6 3 6.6 2.1 pink 2 small small-flower -162 4.9 2.5 4.5 2 small small-flower -163 7.3 2.9 6.3 1.8 pink 2 small small-flower -165 6.7 2.5 5.8 1.8 white 2 large large-flower -166 7.2 3.6 6.1 2.5 pink 2 small small-flower -168 6.5 3.2 5.1 2 purple 2 small small-flower -170 6.4 2.7 5.3 1.9 purple 2 medium small-flower -171 6.8 3 5.5 2.1 purple 2 large large-flower -172 5.7 2.5 5 2 purple 2 large large-flower -174 5.8 2.8 5.1 2.4 purple 2 small small-flower -176 6.4 3.2 5.3 2.3 purple 2 small small-flower -178 6.5 3 5.5 1.8 purple 2 medium small-flower -179 7.7 3.8 6.7 2.2 purple 2 large large-flower -181 7.7 2.6 2.3 white 2 large large-flower -182 6 2.2 5 1.5 purple 2 large large-flower -183 6.9 3.2 5.7 2.3 2 large large-flower -185 5.6 2.8 4.9 2 white 2 large large-flower -186 7.7 2.8 6.7 2 pink 2 medium small-flower -188 6.3 2.7 4.9 1.8 purple 2 large large-flower -190 6.7 3.3 5.7 2.1 purple 2 small small-flower -191 3.2 6 1.8 purple 2 small small-flower -193 6.2 2.8 4.8 1.8 purple 2 small small-flower -194 6.1 3 4.9 1.8 pink 2 large large-flower -196 6.4 2.8 5.6 2.1 purple 2 small small-flower -198 7.2 3 5.8 1.6 pink 2 large large-flower -199 7.4 2.8 6.1 1.9 purple 2 small small-flower -200 7.9 3.8 6.4 2 pink 2 large large-flower -202 6.4 5.6 2.2 purple 2 small small-flower -203 6.3 2.8 5.1 1.5 purple 2 large large-flower -204 6.1 2.6 5.6 1.4 2 medium small-flower -206 7.7 3 6.1 2.3 purple 2 medium small-flower -207 6.3 3.4 5.6 2.4 pink 2 small small-flower -209 6.4 3.1 5.5 1.8 white 2 small small-flower -211 6 3 4.8 1.8 pink 2 small small-flower -212 6.9 3.1 5.4 white 2 small small-flower -214 6.7 5.6 2.4 pink 2 large large-flower -216 6.9 3.1 5.1 2.3 purple 2 small small-flower -217 5.8 2.7 5.1 1.9 pink 2 small small-flower -218 6.8 3.2 5.9 2.3 purple 2 small small-flower -219 6.7 3.3 5.7 2.5 white 2 large large-flower -221 6.7 3 5.2 2.3 pink 2 small small-flower -222 6.3 2.5 5 1.9 pink 2 medium small-flower -223 6.5 3 5.2 2 pink 2 small small-flower -224 6.2 3.4 5.4 white 2 small small-flower -226 5.9 3 5.1 1.8 purple 2 small small-flower +id sepal length (cm) sepal width (cm) petal length (cm) petal width (cm) color target size flower_category is_flower +1a 5.1 3.5 1.4 0.2 white 0 medium small-flower yes +2 4.9 3 1.4 0.2 white 0 medium small-flower yes +4 4.7 3.2 1.3 white 0 large large-flower yes +5 4.6 3.1 1.5 white 0 large large-flower yes +7b 5 3.6 0.2 white 0 medium small-flower yes +8 5.4 3.9 1.7 0.4 white 0 medium small-flower yes +10 4.6 1.4 0.3 white 0 large large-flower yes +11 5 3.4 1.5 0.2 white 0 small small-flower yes +12 4.4 2.9 1.4 0.2 white 0 small small-flower yes +13 4.9 3.1 1.5 0.1 white 0 medium small-flower yes +14 5.4 3.7 1.5 0.2 white 0 medium small-flower yes +15 4.8 3.4 1.6 0.2 pink 0 large large-flower yes +17 4.8 3 1.4 0.1 white 0 medium small-flower yes +18 1.1 0.1 white 0 medium small-flower yes +19 5.8 4 1.2 0.2 white 0 small small-flower yes +20 5.7 4.4 1.5 0.4 pink 0 medium small-flower yes +21 5.4 3.9 1.3 0.4 white 0 large large-flower yes +23 5.1 3.5 1.4 0.3 white 0 small small-flower yes +25 5.7 3.8 1.7 0.3 pink 0 large large-flower yes +26 5.1 3.8 1.5 0.3 white 0 large large-flower +27 5.4 3.4 1.7 0.2 white 0 small small-flower yes +29 5.1 3.7 1.5 0.4 white 0 large large-flower yes +31 4.6 3.6 1 0.2 white 0 small small-flower yes +33 5.1 3.3 1.7 0.5 pink 0 large large-flower yes +35 4.8 3.4 1.9 0.2 white 0 medium small-flower yes +37 5 3 1.6 0.2 white 0 large large-flower yes +38 5 3.4 1.6 0.4 white 0 small small-flower yes +40 5.2 3.5 1.5 0.2 white 0 large large-flower yes +41 5.2 3.4 1.4 0.2 white 0 medium small-flower yes +42 4.7 3.2 1.6 0.2 pink 0 large large-flower yes +43 4.8 1.6 0.2 white 0 small small-flower yes +45 5.4 3.4 1.5 0.4 white 0 small small-flower yes +47 5.2 4.1 1.5 0.1 white 0 small small-flower yes +48 5.5 4.2 1.4 0.2 white 0 small small-flower yes +50 4.9 1.5 0.2 pink 0 small small-flower yes +51 5 3.2 0.2 pink 0 small small-flower +53 5.5 3.5 1.3 0.2 white 0 small small-flower yes +54 4.9 3.6 1.4 0.1 white 0 large large-flower yes +55 4.4 3 1.3 0.2 white 0 medium small-flower yes +56 5.1 3.4 1.5 0.2 white 0 small small-flower yes +57 5 3.5 1.3 0.3 0 large large-flower yes +59 4.5 2.3 1.3 0.3 white 0 medium small-flower yes +61 4.4 3.2 1.3 0.2 0 medium small-flower yes +62 5 3.5 1.6 0.6 white 0 small small-flower yes +63 5.1 3.8 1.9 0.4 pink 0 medium small-flower yes +65 4.8 3 1.4 white 0 large large-flower yes +66 5.1 3.8 1.6 0.2 white 0 large large-flower yes +68 4.6 3.2 1.4 0.2 white 0 large large-flower yes +70 5.3 3.7 1.5 0.2 white 0 small small-flower yes +71 5 3.3 1.4 0.2 white 0 large large-flower yes +72 7 3.2 4.7 1.4 pink 1 medium small-flower yes +74 6.4 3.2 4.5 1.5 pink 1 small small-flower yes +76 6.9 3.1 4.9 1.5 pink 1 medium small-flower yes +78 5.5 2.3 4 1.3 pink 1 large large-flower yes +80 6.5 2.8 4.6 1.5 pink 1 large large-flower yes +81 5.7 2.8 4.5 1.3 purple 1 large large-flower yes +82 3.3 4.7 1.6 pink 1 small small-flower yes +84 4.9 2.4 3.3 1 pink 1 small small-flower yes +85 2.9 4.6 1.3 pink 1 medium small-flower yes +86 5.2 2.7 3.9 1.4 purple 1 large large-flower yes +88 5 2 3.5 1 purple 1 large large-flower +90 5.9 3 4.2 pink 1 medium small-flower yes +91 6 2.2 4 1 pink 1 medium small-flower yes +93 6.1 2.9 4.7 1.4 pink 1 small small-flower yes +95 5.6 2.9 3.6 pink 1 large large-flower yes +97 6.7 3.1 4.4 1.4 1 large large-flower yes +99 5.6 3 4.5 1.5 pink 1 medium small-flower yes +101 5.8 2.7 1 pink 1 large large-flower yes +102 6.2 2.2 4.5 1.5 pink 1 large large-flower yes +104 5.6 2.5 3.9 1.1 purple 1 large large-flower yes +105 5.9 3.2 4.8 1.8 pink 1 small small-flower yes +106 6.1 2.8 4 1.3 pink 1 medium small-flower yes +108 6.3 2.5 1.5 pink 1 large large-flower yes +110 6.1 2.8 4.7 1.2 pink 1 small small-flower yes +112 6.4 2.9 4.3 1.3 pink 1 medium small-flower yes +113 6.6 3 4.4 1.4 purple 1 small small-flower yes +115 6.8 2.8 4.8 1.4 purple 1 medium small-flower yes +116 6.7 3 5 1.7 purple 1 small small-flower yes +118 6 2.9 4.5 1.5 pink 1 medium small-flower yes +120 2.6 1 pink 1 large large-flower yes +122 5.5 2.4 3.8 1.1 pink 1 large large-flower yes +124 5.5 2.4 3.7 1 pink 1 large large-flower yes +125 5.8 2.7 3.9 1.2 1 small small-flower yes +127 2.7 5.1 1.6 purple 1 small small-flower +128 5.4 3 4.5 1.5 pink 1 medium small-flower yes +129 6 3.4 4.5 1.6 pink 1 large large-flower yes +130 6.7 3.1 4.7 1.5 purple 1 medium small-flower yes +131 6.3 4.4 1.3 pink 1 small small-flower yes +133 5.6 3 4.1 1.3 pink 1 medium small-flower yes +135 5.5 2.5 4 1.3 pink 1 large large-flower yes +136 5.5 2.6 4.4 1.2 purple 1 small small-flower yes +138 3 4.6 1.4 pink 1 medium small-flower yes +140 5.8 2.6 4 1.2 pink 1 large large-flower +142 5 2.3 3.3 1 pink 1 medium small-flower yes +143 5.6 2.7 4.2 1.3 pink 1 medium small-flower yes +144 5.7 3 4.2 1.2 purple 1 medium small-flower yes +146 5.7 2.9 4.2 1.3 pink 1 medium small-flower yes +148 6.2 2.9 4.3 1.3 pink 1 large large-flower yes +150 2.5 3 1.1 pink 1 large large-flower yes +151 5.7 2.8 4.1 1.3 pink 1 small small-flower yes +153 6.3 3.3 6 2.5 2 medium small-flower yes +154 5.8 1.9 purple 2 small small-flower yes +156 7.1 3 5.9 2.1 pink 2 small small-flower yes +158 6.3 2.9 5.6 1.8 purple 2 small small-flower yes +160 6.5 3 2.2 purple 2 large large-flower yes +161 7.6 3 6.6 2.1 pink 2 small small-flower yes +162 4.9 2.5 4.5 2 small small-flower yes +163 7.3 2.9 6.3 1.8 pink 2 small small-flower yes +165 6.7 2.5 5.8 1.8 white 2 large large-flower yes +166 7.2 3.6 6.1 2.5 pink 2 small small-flower yes +168 6.5 3.2 5.1 2 purple 2 small small-flower yes +170 6.4 2.7 5.3 1.9 purple 2 medium small-flower yes +171 6.8 3 5.5 2.1 purple 2 large large-flower yes +172 5.7 2.5 5 2 purple 2 large large-flower yes +174 5.8 2.8 5.1 2.4 purple 2 small small-flower yes +176 6.4 3.2 5.3 2.3 purple 2 small small-flower yes +178 6.5 3 5.5 1.8 purple 2 medium small-flower yes +179 7.7 3.8 6.7 2.2 purple 2 large large-flower yes +181 7.7 2.6 2.3 white 2 large large-flower yes +182 6 2.2 5 1.5 purple 2 large large-flower yes +183 6.9 3.2 5.7 2.3 2 large large-flower yes +185 5.6 2.8 4.9 2 white 2 large large-flower yes +186 7.7 2.8 6.7 2 pink 2 medium small-flower yes +188 6.3 2.7 4.9 1.8 purple 2 large large-flower yes +190 6.7 3.3 5.7 2.1 purple 2 small small-flower yes +191 3.2 6 1.8 purple 2 small small-flower yes +193 6.2 2.8 4.8 1.8 purple 2 small small-flower yes +194 6.1 3 4.9 1.8 pink 2 large large-flower yes +196 6.4 2.8 5.6 2.1 purple 2 small small-flower yes +198 7.2 3 5.8 1.6 pink 2 large large-flower yes +199 7.4 2.8 6.1 1.9 purple 2 small small-flower yes +200 7.9 3.8 6.4 2 pink 2 large large-flower yes +202 6.4 5.6 2.2 purple 2 small small-flower yes +203 6.3 2.8 5.1 1.5 purple 2 large large-flower yes +204 6.1 2.6 5.6 1.4 2 medium small-flower yes +206 7.7 3 6.1 2.3 purple 2 medium small-flower yes +207 6.3 3.4 5.6 2.4 pink 2 small small-flower yes +209 6.4 3.1 5.5 1.8 white 2 small small-flower yes +211 6 3 4.8 1.8 pink 2 small small-flower yes +212 6.9 3.1 5.4 white 2 small small-flower yes +214 6.7 5.6 2.4 pink 2 large large-flower yes +216 6.9 3.1 5.1 2.3 purple 2 small small-flower +217 5.8 2.7 5.1 1.9 pink 2 small small-flower yes +218 6.8 3.2 5.9 2.3 purple 2 small small-flower yes +219 6.7 3.3 5.7 2.5 white 2 large large-flower yes +221 6.7 3 5.2 2.3 pink 2 small small-flower yes +222 6.3 2.5 5 1.9 pink 2 medium small-flower yes +223 6.5 3 5.2 2 pink 2 small small-flower yes +224 6.2 3.4 5.4 white 2 small small-flower yes +226 5.9 3 5.1 1.8 purple 2 small small-flower yes From b336dddca4d4b05da9687e5f2f95b78d0db43050 Mon Sep 17 00:00:00 2001 From: Kalum Ost Date: Wed, 26 Nov 2025 16:16:39 -0500 Subject: [PATCH 23/26] Added Jupyter Notebooks to the git ignore, as we occasionally use them for visual validation of tests. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ad2dc06..d5c9061 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ *__pycache__/ .vscode/ .DS_Store +/.ipynb_checkpoints/ # Environments .env From a2bf6c878e835b52f5d01f81ae20f3e2ffbd085a Mon Sep 17 00:00:00 2001 From: Kalum Ost Date: Wed, 3 Dec 2025 23:22:33 -0500 Subject: [PATCH 24/26] Added catch for homogeneity when running SHAP tests. --- study/metrics.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/study/metrics.py b/study/metrics.py index 09d1b23..56de84e 100644 --- a/study/metrics.py +++ b/study/metrics.py @@ -147,6 +147,12 @@ def shap_additive(manager: OptunaModelManager, x: BaseDataManager, _: BaseDataMa # Initialize the explainer, using the x data as both the mask and feature list x_arr = x.as_array() model = manager.get_model() + + if np.unique(x_arr).shape[0] < 2: + # SHAP cannot run on a dataset which is entirely homogenous; + # return early to avoid an error + return "NULL" + try: # Default to the "generic" explainer explainer = shap.Explainer( From 01a4bc1a149b5274e377d79c573bc4c0dd17fdb2 Mon Sep 17 00:00:00 2001 From: Kalum Ost Date: Thu, 26 Feb 2026 16:14:01 -0500 Subject: [PATCH 25/26] Removed "inspect_output_db", as it is too specific to Jan's analysis. --- inspect_output_db.py | 469 ------------------------------------------- 1 file changed, 469 deletions(-) delete mode 100644 inspect_output_db.py diff --git a/inspect_output_db.py b/inspect_output_db.py deleted file mode 100644 index f276ede..0000000 --- a/inspect_output_db.py +++ /dev/null @@ -1,469 +0,0 @@ -""" -Script for inspecting and analyzing the output database. - -This script: -- Reads tables from an SQLite database containing ML trial results. -- Extracts feature importance values and model performance metrics. -- Computes weighted statistics (mean and standard deviation) using model performance as weights. -- Identifies the best models based on a specified metric. -- Saves feature importance and model performance data as CSV files. -- Generates plots to visualize model performance across replicates and trials. - -Author: Jan Valosek, Kalum Ost -""" - -import os -import re -import argparse - -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt -import seaborn as sns -from sqlite3 import connect - -from sklearn.metrics import roc_curve, auc - -#target='AIS_change_bin' -target='UEMS_change_bin' -#target='LEMS_change_bin' -#target='AIS_change_bin_gt0' -#target='SNL_Class_initial_bin' -num_of_trials = 100 - -models_to_keep = ['mri_metrics_hemorrhage_included', 'all_metrics_AIS_Initial_ladder_encoded', - 'clinical_metrics_AIS_Initial_ladder_encoded'] - -target_to_title_dict = { - 'AIS_change_bin': 'AIS Change', - 'UEMS_change_bin': 'UEMS Change', - 'LEMS_change_bin': 'LEMS Change', - 'AIS_change_bin_gt0': 'AIS Change' -} - -def get_parser(): - """ - parser function - """ - - parser = argparse.ArgumentParser( - description='Script for inspecting and analyzing the output database.', - prog=os.path.basename(__file__).strip('.py') - ) - parser.add_argument( - '-i', - metavar="FILE_NAME", - required=True, - type=str, - help='Absolute path to the SQLite database. ' - 'Example: /target_AIS_change_bin_1.db' - ) - parser.add_argument( - '-o', - metavar='DIR_NAME', - required=True, - type=str, - help='Absolute path to the output directory where the results (figures and CSV files) will be saved. ' - ) - - return parser - - -def read_db(fname_path: str) -> dict: - """ - Read tables from the database as dataframes - :param fname_path: path to the SQLite database - :return: dictionary with the tables (dataframes) from the database - """ - assert os.path.exists(fname_path), f"Database file not found: {fname_path}" - con = connect(fname_path) - - print('Reading tables...') - - tables = pd.read_sql( - "SELECT * FROM sqlite_master", - con=con - ).loc[:, 'name'] - tables_dict = {} - for t in tables: - # Pull the dataframe from the database - try: - df = pd.read_sql( - f"SELECT * FROM {t}", - con=con - ) - tables_dict[t] = df - except: - print(f"Failed to read table {t}, ignoring it") - continue - con.close() - - # Keep only the models we are interested in - for model_name in list(tables_dict.keys()): - if model_name.replace(f'{target}__LogisticRegression__', '') not in models_to_keep: - tables_dict.pop(model_name) - - return tables_dict - -def weighted_std(values: np.ndarray, weights: np.ndarray) -> float: - """ - Compute the weighted standard deviation. - :param values: Array of feature importance values. - :param weights: Array of weights (e.g., model performance scores). - :return: Weighted standard deviation. - """ - # Compute the weighted mean - weighted_mean = np.average(values, weights=weights) - # Compute the weighted variance - weighted_variance = np.average((values - weighted_mean) ** 2, weights=weights) - - # Take the square root to obtain the weighted standard deviation - return np.sqrt(weighted_variance) - -def compute_weighted_feature_importance(best_models, metric): - """ - Compute the weighted average feature importance using `importance_by_permutation (test)` - with `balanced_accuracy (test)` as the weight. - - :param best_models: DataFrame with best models selected for each replicate - :param metric: Performance metric used as weight (e.g., 'balanced_accuracy (test)') - :return: DataFrame with weighted feature importance - """ - - # Extract importance and performance metric - best_models = best_models[['model_name', 'replicate', 'trial', metric, 'importance_by_permutation (test)']] - - # Convert 'importance_by_permutation (test)' from str to dict using re - pattern = r'([\w\s\(\)<\-]+): ([\d\.]+)' # Works with names containing spaces, (), <-, and _ - best_models['importance_by_permutation (test)'] = best_models['importance_by_permutation (test)'].apply( - lambda x: {match[0].strip(): float(match[1]) for match in re.findall(pattern, x)}) - - # Convert the dictionaries contained with the feature_col dicts into dataframes which can be stacked - raw_dfs = [] - weighted_dfs = [] - for r in best_models.iterrows(): - rvals = r[1] - tmp_df = pd.DataFrame.from_dict({k: [v] for k, v in rvals['importance_by_permutation (test)'].items()}) - raw_dfs.append(tmp_df) - - # Stack the dataframes - raw_feature_imps = pd.concat(raw_dfs).fillna(0) - - # Query the weights list - weights = best_models[metric].astype('float64') - - # For each feature, calculate our desired statistics - return_cols = ['Mean', 'STD', 'Weighted Mean', 'Weighted STD'] - return_df_dict = {} - for c in raw_feature_imps.columns: - # Single query of the dataframe, as pandas can be slow w/ repeated queries - samples = raw_feature_imps[c] - # Raw Mean - c_mean = np.mean(samples) - # Raw STD - c_std = np.std(samples) - # Weighted mean - c_mean_weighted = np.average(samples, weights=weights) - # Weighted STD - c_std_weighted = weighted_std(samples, weights) - # Stack them into a list and store it in the dictionary - return_df_dict[c] = [c_mean, c_std, c_mean_weighted, c_std_weighted] - - weighted_importance_df = pd.DataFrame.from_dict(return_df_dict, columns=return_cols, orient='index') - # Sort by 'Weighted Mean' - weighted_importance_df = weighted_importance_df.sort_values('Weighted Mean', ascending=False) - - return weighted_importance_df - -def get_best_replicate(tables_dict, metric) -> dict: - """ - Get the best replicate (i.e., best performing model) for each trail based on the specified metric. - Also, compute the weighted average of `importance_by_permutation (test)` features, with the weight being - the model's performance (e.g., `balanced_accuracy (test)`). - Save the best models and weighted feature importance to CSV files. - :param tables_dict: dictionary with the tables (dataframes) from the database - :param metric: metric to use for selecting the best models; e.g., 'balanced_accuracy (test)' - :return: dictionary of dataframes with the best models for each model - """ - - os.makedirs('testing/output/csv', exist_ok=True) - fname_out = f'testing/output/csv/{target}_{metric}_best_models' - - best_models_dict = {} - - # Loop over individual models - for model_name, df in tables_dict.items(): - # Get the best model (trial) for each replicate - best_models = df.sort_values(metric, ascending=True).groupby('replicate').tail(1) - # Sort by best_models by replicate - best_models = best_models.sort_values('replicate') - - # Save metric and 'importance_by_permutation (test)' into a XLSX file; append models to the same file - # include model name as the first column - best_models.insert(0, 'model_name', model_name) - # Save the best models to a CSV file - best_models[['model_name', 'replicate', 'trial', metric, 'importance_by_permutation (test)']].to_csv( - f'{fname_out}.csv', mode='a', index=False, header=True) - - best_models_dict[model_name] = best_models - - # Compute weighted average of `importance_by_permutation (test)` features, with the weight being the model's - # performance (e.g., `balanced_accuracy (test)`) - weighted_importance_df = compute_weighted_feature_importance(best_models, metric) - weighted_importance_df.insert(0, 'model_name', model_name) - # Save the weighted feature importance to a CSV file - weighted_importance_df.to_csv(f'{fname_out}_weighted_feature_importance.csv', - mode='a', index=True, header=True) - - print(f"Saved best models to {fname_out}.csv") - print(f"Saved weighted feature importance to {fname_out}_weighted_feature_importance.csv") - - return best_models_dict - - -def get_df_for_plotting(tables_dict, metric) -> pd.DataFrame: - """ - Iterate over the dataframes in tables_dict and merge them into a single dataframe for plotting - :param tables_dict: dictionary with the tables (dataframes) from the database - :param metric: metric to plot; e.g., 'balanced_accuracy (test)' - :return: dataframe for plotting - """ - - df_plotting = pd.DataFrame(columns=['replicate', 'trial']) - - # Loop over individual models - for model_name, df in tables_dict.items(): - if target in model_name: - # Get only the columns we need for plotting - df_temp = df[['replicate', 'trial', metric]] - # Rename balanced_accuracy to model_name - df_temp = df_temp.rename(columns={metric: model_name}) - # Add df_temp to df_plotting based on 'replicate' and 'trial'; do not replicate the 'replicate' and 'trial' columns - df_plotting = pd.merge(df_plotting, df_temp, on=['replicate', 'trial'], how='outer') - - # Some additional cleaning for plotting - # Sort by 'replicate' and 'trial' - df_plotting = df_plotting.sort_values(['replicate', 'trial']) - # Shorten column names (first two columns are 'replicate' and 'trial') - for column in df_plotting.columns[2:]: - df_plotting.rename(columns={column: column.replace(f'{target}__LogisticRegression__', '')}, inplace=True) - - return df_plotting - -def plotting(df_plotting, metric): - """ - Plot the balanced accuracy across replicates and trials for each model - :param df_plotting: dataframe for plotting - :param metric: metric to plot; e.g., 'balanced_accuracy (test)' - """ - - metric_title = metric.replace('_', ' ').title() # e.g., Balanced Accuracy (Test) - metric_fname = metric.replace(' ', '_').replace('(', '').replace(')', '') # e.g., balanced_accuracy_test - - # Melt the dataframe to a long format for easier plotting - df_long = df_plotting.melt( - id_vars=['replicate', 'trial'], - value_vars=df_plotting.columns[2:], # Skip 'replicate' and 'trial' - var_name='model_name', - value_name=metric - ) - - df_long[metric] = pd.to_numeric(df_long[metric]) - # agg_df = df_long.groupby(['replicate', 'model_name'])[metric].agg(['mean', 'std']).reset_index() - - os.makedirs('testing/output/plots', exist_ok=True) - - # # x-axis: replicate - # plt.figure(figsize=(10, 6)) - # #sns.lineplot(data=agg_df, x='replicate', y='mean', hue='model_name') - # sns.lineplot(data=df_long, x='replicate', y=metric, hue='model_name', errorbar='sd') - # # Customize the plot - # plt.title(f'{target} -- Mean and Std of {metric_title} Across Trials for Each Replicate') - # plt.xlabel('Replicate') - # plt.ylabel(metric_title) - # plt.legend(title='Model Name') - # # Show horizontal gridlines - # plt.grid(axis='y') - # # Show all x-ticks - # plt.xticks(df_long['replicate'].unique()) - # plt.tight_layout() - # #plt.show() - # # Save with 300 dpi - # plt.savefig(f'testing/output/plots/{target}_{metric_fname}_replicates_num_of_trials_{num_of_trials}.png', dpi=300) - # plt.close() - - # x-axis: trial - plt.figure(figsize=(10, 6)) - # sns.lineplot(data=agg_df, x='replicate', y='mean', hue='model_name') - sns.lineplot(data=df_long, x='trial', y=metric, hue='model_name', errorbar='sd') - # Customize the plot - plt.title(f'{target_to_title_dict[target]}: Mean and Std of {metric_title} Across Replicates for Each Trial') - plt.xlabel('Trial') - plt.ylabel(metric_title) - plt.legend(title='Model Name') - # Show horizontal gridlines - plt.grid(axis='y') - # Make legend smaller - plt.legend(title='Model Name', fontsize='small') - # Show all x-ticks - # plt.xticks(df_long['trial'].unique()) - plt.tight_layout() - # plt.show() - # Save with 300 dpi - plt.savefig(f'testing/output/plots/{target}_{metric_fname}_trials_num_of_trials_{num_of_trials}.png', dpi=300) - print(f"Saved plots to 'plots' directory") - plt.close() - -def extract_roc_data(best_models_dict): - """ - Extracts y_true and y_pred_proba from a dataframe. - """ - - models_roc_data = {} - - for model_name, df in best_models_dict.items(): - if "y_true_collector (test)" in df.columns and "y_pred_proba_collector (test)" in df.columns: - y_true_all = [] - y_pred_proba_all = [] - for y_true_str, y_pred_proba_str in zip(df["y_true_collector (test)"], df["y_pred_proba_collector (test)"]): - try: - y_true = np.array(eval(y_true_str)) - y_pred_proba = np.array(eval(y_pred_proba_str)) - y_true_all.extend(y_true) - y_pred_proba_all.extend(y_pred_proba) - except Exception as e: - print(f"Error parsing y_true/y_pred_proba: {e}") - continue - - models_roc_data[model_name] = (np.array(y_true_all), np.array(y_pred_proba_all)) - - return models_roc_data - -def plot_roc_curve(models_roc_data, metric): - """ - Generates and saves a ROC curve with multiple models. - :param models_roc_data: Dictionary where keys are model names, and values are (y_true, y_pred_proba). - """ - - metric_fname = metric.replace(' ', '_').replace('(', '').replace(')', '') # e.g., balanced_accuracy_test - - plt.figure(figsize=(8, 6)) - - for model_name, (y_true, y_pred_proba) in models_roc_data.items(): - model_name = model_name.replace(f'{target}__LogisticRegression__', '') - if len(y_true) == 0 or len(y_pred_proba) == 0: - print(f"Skipping ROC for {model_name} due to missing data.") - continue - - # Compute ROC curve - fpr, tpr, _ = roc_curve(y_true, y_pred_proba) - roc_auc = auc(fpr, tpr) - - # Plot each model’s ROC curve - plt.plot(fpr, tpr, lw=2, label=f'{model_name} (AUC = {roc_auc:.2f})') - - # Plot random classifier baseline - plt.plot([0, 1], [0, 1], color='gray', linestyle='--') - - plt.xlim([0.0, 1.0]) - plt.ylim([0.0, 1.05]) - plt.xlabel('False Positive Rate') - plt.ylabel('True Positive Rate') - plt.title(f'{target_to_title_dict[target]}: ROC (Receiver Operating Characteristic)') - plt.legend(loc='lower right') - - # Save the plot - fname_figure = f'testing/output/plots/{target}_{metric_fname}_roc_curve.png' - os.makedirs('testing/output/plots', exist_ok=True) - plt.savefig(fname_figure, dpi=300) - print(f"Saved ROC curve to {fname_figure}") - plt.close() - - -def plot_mean_std_roc_curve(best_models_dict, metric): - """ - Plot the mean ± std ROC curve for each model across replicates. - :param best_models_dict: dictionary with the best models for each model - :param metric: metric to use for selecting the best models; e.g., 'balanced_accuracy (test)' - """ - - metric_fname = metric.replace(' ', '_').replace('(', '').replace(')', '') # e.g., balanced_accuracy_test - - plt.figure(figsize=(8, 6)) - mean_fpr = np.linspace(0, 1, 100) - - for model_name, df in best_models_dict.items(): - model_name_cleaned = model_name.split('__')[-1] - tprs = [] - - # Extract ROC data for each replicate - for _, row in df.iterrows(): - y_true = np.array(eval(row["y_true_collector (test)"])) - y_pred_proba = np.array(eval(row["y_pred_proba_collector (test)"])) - - fpr, tpr, _ = roc_curve(y_true, y_pred_proba) - tprs.append(np.interp(mean_fpr, fpr, tpr)) - - # Calculate mean and std of TPRs - mean_tpr = np.mean(tprs, axis=0) - std_tpr = np.std(tprs, axis=0) - roc_auc = auc(mean_fpr, mean_tpr) - - # Plot mean ROC curve with std deviation - plt.plot(mean_fpr, mean_tpr, label=f'{model_name_cleaned} (AUC = {roc_auc:.2f} ± {std_tpr.mean():.2f})') - plt.fill_between(mean_fpr, mean_tpr - std_tpr, mean_tpr + std_tpr, alpha=0.2) - - # Plot baseline - plt.plot([0, 1], [0, 1], color='gray', linestyle='--') - plt.xlabel('False Positive Rate') - plt.ylabel('True Positive Rate') - plt.title(f'{target_to_title_dict[target]}: Mean ± Std ROC Curve') - plt.legend(loc='lower right') - plt.grid(True) - plt.tight_layout() - #plt.show() - - # Save the plot - os.makedirs('testing/output/plots', exist_ok=True) - fname_figure = f'testing/output/plots/{target}_{metric_fname}_roc_curve_mean.png' - plt.savefig(fname_figure, dpi=300) - print(f"Saved ROC curve to {fname_figure}") - plt.close() - - -def main(): - - # Parse the command line arguments - parser = get_parser() - args = parser.parse_args() - - fname_path = os.path.abspath(args.i) - output_path = os.path.abspath(args.o) - os.makedirs(output_path, exist_ok=True) - - # Read tables from the database as dataframes - tables_dict = read_db(fname_path) - - # Get unique targets - targets = set([item.split('__')[0] for item in tables_dict.keys()]) - - for target in targets: - #for metric in ['balanced_accuracy (test)', 'balanced_accuracy (validate)']: - for metric in ['balanced_accuracy (validate)']: - # Get the best replicate (i.e., best performing model) for each trial (train/test split) - best_models_dict = get_best_replicate(tables_dict, metric) - plot_mean_std_roc_curve(best_models_dict, metric) - - models_roc_data = extract_roc_data(best_models_dict) - if models_roc_data: - plot_roc_curve(models_roc_data, metric) - else: - print("Skipping ROC curve generation: No valid data found.") - - # Prepare the dataframe for plotting - df_plotting = get_df_for_plotting(tables_dict, metric, target) - # Plot the metric across trials for each model - plotting(df_plotting, metric, target, output_path) - -if __name__ == '__main__': - main() From c323da8ce39cde67fe7c9435a8c9f150502ccb1b Mon Sep 17 00:00:00 2001 From: Kalum Ost Date: Thu, 26 Feb 2026 17:18:57 -0500 Subject: [PATCH 26/26] Pinned to pre-3.0 version of Pandas until `dtype` issues can be addressed. --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index af6704e..fc42b65 100644 --- a/environment.yml +++ b/environment.yml @@ -7,6 +7,6 @@ dependencies: - ca-certificates - openssl - scikit-learn - - pandas + - pandas<3 - pytest - shap