diff --git a/.gitignore b/.gitignore index ad2dc06..d5c9061 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ *__pycache__/ .vscode/ .DS_Store +/.ipynb_checkpoints/ # Environments .env diff --git a/data/hooks/__init__.py b/data/hooks/__init__.py index d21af55..ac19216 100644 --- a/data/hooks/__init__.py +++ b/data/hooks/__init__.py @@ -30,7 +30,7 @@ def _decorator(cls: Type[DataHook]): # TODO: Find a more elegant way to do this from data.hooks.feature_selection import ( SampleNullityDrop, FeatureNullityDrop, ExplicitDrop, ExplicitKeep, PrincipalComponentAnalysis, - RecursiveFeatureElimination + RecursiveFeatureElimination, VarianceDrop ) from data.hooks.imputation import SimpleImputation from data.hooks.encoding import OneHotEncoding diff --git a/data/hooks/feature_selection.py b/data/hooks/feature_selection.py index 93eb743..4632f87 100644 --- a/data/hooks/feature_selection.py +++ b/data/hooks/feature_selection.py @@ -6,7 +6,7 @@ import pandas as pd from optuna import Trial from sklearn.decomposition import PCA -from sklearn.feature_selection import RFE +from sklearn.feature_selection import RFE, VarianceThreshold from sklearn.linear_model import LogisticRegression from config.utils import default_as, is_float, is_list, parse_data_config_entry @@ -135,6 +135,81 @@ def run(self, x: BaseDataManager, y: Optional[BaseDataManager] = None) -> BaseDa return x.drop_features(drop_idx) +## Feature selection by homogeneity +@registered_data_hook("drop_low_variance") +class VarianceDrop(FittedDataHook): + """ + Thin wrapper for SciKit-Learn's VarianceThreshold class, for use as a data hook within MOOP. + + Runs additional checks on top of the default implementation provided by SciKit-Learn: + * Ensures the resulting dataset always contains at least 1 feature. + + Example usage: + { + "type": "drop_low_variance", + "threshold": 0.1 + } + """ + def __init__(self, config, **kwargs): + # TODO: make this tunable + super().__init__(config, **kwargs) + + # Get the variance threshold + threshold = parse_data_config_entry( + "threshold", config, + default_as(0.0, self.logger), is_float(self.logger) + ) + + # Build the wrapped VarianceThreshold object + self.threshold = threshold + self.selected_features: list[str] | None = None + + @classmethod + def from_config(cls, config: dict, logger: Logger = Logger.root) -> Self: + return cls(config=config, logger=logger) + + def run(self, x: BaseDataManager, y: Optional[BaseDataManager] = None) -> BaseDataManager: + # If x contains only one feature already, just return that feature, as RFE has a stroke otherwise + if x.n_features() == 1: + self.logger.warning("Only one feature in the dataset was found; " + "dropping any further would result in a null dataset." + "Original (unmodified) dataset returned instead.") + return x + + # Fit the model to the dataset + vt = VarianceThreshold(threshold=self.threshold) + vt.fit(x.as_array(), np.ravel(y.as_array())) # Ravel prevents some warning spam + + # Select only the features with variance less than the threshold + self.selected_features = vt.get_feature_names_out(x.features()) + + # Ensure that at least one feature was kept + if self.selected_features.shape[0] < 1: + # Find the + highest_var = np.max(vt.variances_) + highest_var_feature = list(x.features())[np.argmax(vt.variances_)] + self.selected_features = [highest_var_feature] + self.logger.warning( + f"Low-variance filter almost dropped all features; kept highest variance " + f"feature ({highest_var_feature}, variance {highest_var}) alone to prevent crash!" + ) + + # Return the copy of x containing only these features + x_out = x.get_features(self.selected_features) + return x_out + + def run_fitted(self, x_train: BaseDataManager, x_test: Optional[BaseDataManager], + y_train: Optional[BaseDataManager] = None, y_test: Optional[BaseDataManager] = None) -> \ + tuple[BaseDataManager, BaseDataManager]: + # Run the fitted analysis first + train_out = self.run(x_train, y_train) + + # Use the same set of features to filter the x_test set + test_out = x_test.get_features(self.selected_features) + + return train_out, test_out + + ### Principal Component Analysis ### @registered_data_hook("principal_component_analysis") class PrincipalComponentAnalysis(Tunable, FittedDataHook): @@ -263,8 +338,7 @@ def from_config(cls, config: dict, logger: Logger = Logger.root) -> Self: def tune(self, trial: Trial): self.prop_tuner.tune(trial) # Generate the new backing model based on this setup - # TODO: Generalize this to work with continuous targets as well - new_lor = LogisticRegression() + new_lor = LogisticRegression(solver='saga') self.backing_rfe = RFE(estimator=new_lor, n_features_to_select=self.prop_tuner.value) def tunable_params(self) -> list[TunableParam]: diff --git a/environment.yml b/environment.yml index d17cc45..fc42b65 100644 --- a/environment.yml +++ b/environment.yml @@ -7,5 +7,6 @@ dependencies: - ca-certificates - openssl - scikit-learn - - pandas + - pandas<3 - pytest + - shap diff --git a/study/__init__.py b/study/__init__.py index 96ee0ba..1d330cc 100644 --- a/study/__init__.py +++ b/study/__init__.py @@ -17,6 +17,9 @@ "sk_f1_weighted_avg": sk_f1_weighted_avg, "sk_f1_perclass": sk_f1_perclass, "importance_by_permutation": importance_by_permutation, + "shap_additive": shap_additive, "correct_samples": correct_samples, - "incorrect_samples": incorrect_samples + "incorrect_samples": incorrect_samples, + "y_true_collector": y_true_collector, + "y_pred_proba_collector": y_pred_proba_collector } diff --git a/study/metrics.py b/study/metrics.py index 6f6defb..56de84e 100644 --- a/study/metrics.py +++ b/study/metrics.py @@ -1,7 +1,10 @@ """ Metric-reporting closures for use in this framework. """ +import sys + import numpy as np +import shap from sklearn.inspection import permutation_importance from sklearn.metrics import balanced_accuracy_score, log_loss, roc_auc_score, precision_score, recall_score, f1_score @@ -18,7 +21,8 @@ def clean_val_for_db(val): def sk_log_loss(manager: OptunaModelManager, x: BaseDataManager, y: BaseDataManager): # Log Loss py = manager.predict_proba(x.as_array()) - return log_loss(y.as_array(), py) + y_labels = [i for i in range(py.shape[1])] + return log_loss(y.as_array(), py, labels=y_labels) def sk_balanced_accuracy(manager: OptunaModelManager, x: BaseDataManager, y: BaseDataManager): # Balanced Accuracy @@ -101,6 +105,95 @@ def importance_by_permutation(manager: OptunaModelManager, x: BaseDataManager, y importance_vals = clean_val_for_db(importance_vals) return importance_vals +def shap_additive(manager: OptunaModelManager, x: BaseDataManager, _: BaseDataManager): + """ + To restore the (raw) values for a given run, run the following snippet: + + ``` + from io import StringIO + + # You can omit the Numpy import if you manually + # parse the inner string in the list comp + import numpy as np + + # This is the SHAP value within the database you want to parse + val = ... + + # Strip the brackets first + val = val.strip("{").strip("}") + # Split by commas + entry_strs = val.split(", ") + # Split the dataset by feature name + shap_map = dict() + for entry_str in entry_strs: + # Split the text along the colon to get the feature label back + feature_label, shap_value_str = entry_str.split(": ") + # Parse the shap value string back into numeric form + shap_vals = [list(np.fromstring(x, sep=" ")) for x in shap_value_str.split("\n")] + # Add it to the map + shap_map[feature_label] = shap_vals + ``` + + Each entry in `shap_map` will be Numpy array of with the following dimensions: + * n is the number of samples in the input dataset (train, validate, or test), and + * c is the number of categorical classes used during training; + If this is binary classification, or a continuous target, c=1. + + For categorical targets with more than 2 classes, each class is treated as + unique feature by SHAP for the purpose of calculating SHAP values. + + TODO: Save the `shap_values` directly via pickle into a SQLite blob + """ + # Initialize the explainer, using the x data as both the mask and feature list + x_arr = x.as_array() + model = manager.get_model() + + if np.unique(x_arr).shape[0] < 2: + # SHAP cannot run on a dataset which is entirely homogenous; + # return early to avoid an error + return "NULL" + + try: + # Default to the "generic" explainer + explainer = shap.Explainer( + model, x_arr, feature_names=x.features() + ) + # Calculate the Shapley values from this dataset + shap_values = explainer(x_arr) + except TypeError as err: + # If that failed, try to use the model's "predict" function instead + if hasattr(model, "predict"): + explainer = shap.Explainer( + model.predict, x_arr, feature_names=x.features() + ) + # Calculate the Shapley values from this dataset + shap_values = explainer(x_arr) + else: + raise err + + shap_list = list() + for i, v in enumerate(shap_values.feature_names): + # SHAP auto-reduces the shape of its features if it is targeting + # a binary classification OR a continuous metric + if len(shap_values.values.shape) < 3: + val_str = np.array2string(shap_values.values[:, i], max_line_width=sys.maxsize, threshold=sys.maxsize) + else: + val_str = np.array2string(shap_values.values[:, i, :], max_line_width=sys.maxsize, threshold=sys.maxsize) + # Remove the brackets; despite Numpy adding them, it cannot parse them after... + val_str = val_str.replace("[", "").replace("]", "") + val_str = f"{v}: {val_str}" + shap_list.append(val_str) + + # This nonsense is required because Python maps + # "/n" to "//n" if you string convert a dict; + # why the hell does it do that?!?!? + full_str = "{" + full_str += ", ".join(shap_list) + full_str += "}" + + # Return the result to be saved + return full_str + """ Sample Reporting """ def correct_samples(manager: OptunaModelManager, x: BaseDataManager, y: BaseDataManager): @@ -133,4 +226,16 @@ def incorrect_samples(manager: OptunaModelManager, x: BaseDataManager, y: BaseDa # Strip quotation marks from the result so the DB backend doesn't explode bad_samples = clean_val_for_db(bad_samples) - return bad_samples \ No newline at end of file + return bad_samples + +""" ROC Curve """ +def y_true_collector(_: OptunaModelManager, __: BaseDataManager, y: BaseDataManager): + """ Collects the true binary labels for ROC curve generation. """ + return clean_val_for_db(list(y.as_array().flatten())) + +def y_pred_proba_collector(manager: OptunaModelManager, x: BaseDataManager, _: BaseDataManager): + """ Collects predicted probabilities for the positive class. """ + py = manager.predict_proba(x.as_array()) + if py.shape[1] != 2: + raise ValueError(f"Expected binary classification with two probability columns; found {py.shape[1]}.") + return clean_val_for_db(list(py[:, 1])) # Probabilities for the positive class \ No newline at end of file diff --git a/testing/iris_data/iris_config.json b/testing/iris_data/iris_config.json index 117ac40..5552d72 100644 --- a/testing/iris_data/iris_config.json +++ b/testing/iris_data/iris_config.json @@ -15,9 +15,14 @@ } ], "post_split_hooks": [ + { + "type": "imputation_simple", + "strategy": "most_frequent", + "features": ["color", "flower_category", "is_flower", "size"] + }, { "type": "one_hot_encode", - "features": ["color", "flower_category"] + "features": ["color", "flower_category", "is_flower"] }, { "type": "ladder_encode", @@ -33,6 +38,19 @@ "type": "standard_scaling", "run_per_cross": true }, + { + "type": "drop_low_variance", + "threshold": 0.0 + }, + { + "type": "principal_component_analysis", + "proportion": { + "label": "pca_feature_proportion", + "type": "float", + "low": 0.1, + "high": 0.9 + } + }, { "type": "recursive_feature_elimination", "proportion": { diff --git a/testing/iris_data/iris_testing.tsv b/testing/iris_data/iris_testing.tsv index 9c4335e..68c574f 100644 --- a/testing/iris_data/iris_testing.tsv +++ b/testing/iris_data/iris_testing.tsv @@ -1,151 +1,151 @@ -id sepal length (cm) sepal width (cm) petal length (cm) petal width (cm) color target size flower_category -1a 5.1 3.5 1.4 0.2 white 0 medium small-flower -2 4.9 3 1.4 0.2 white 0 medium small-flower -4 4.7 3.2 1.3 white 0 large large-flower -5 4.6 3.1 1.5 white 0 large large-flower -7b 5 3.6 0.2 white 0 medium small-flower -8 5.4 3.9 1.7 0.4 white 0 medium small-flower -10 4.6 1.4 0.3 white 0 large large-flower -11 5 3.4 1.5 0.2 white 0 small small-flower -12 4.4 2.9 1.4 0.2 white 0 small small-flower -13 4.9 3.1 1.5 0.1 white 0 medium small-flower -14 5.4 3.7 1.5 0.2 white 0 medium small-flower -15 4.8 3.4 1.6 0.2 pink 0 large large-flower -17 4.8 3 1.4 0.1 white 0 medium small-flower -18 1.1 0.1 white 0 medium small-flower -19 5.8 4 1.2 0.2 white 0 small small-flower -20 5.7 4.4 1.5 0.4 pink 0 medium small-flower -21 5.4 3.9 1.3 0.4 white 0 large large-flower -23 5.1 3.5 1.4 0.3 white 0 small small-flower -25 5.7 3.8 1.7 0.3 pink 0 large large-flower -26 5.1 3.8 1.5 0.3 white 0 large large-flower -27 5.4 3.4 1.7 0.2 white 0 small small-flower -29 5.1 3.7 1.5 0.4 white 0 large large-flower -31 4.6 3.6 1 0.2 white 0 small small-flower -33 5.1 3.3 1.7 0.5 pink 0 large large-flower -35 4.8 3.4 1.9 0.2 white 0 medium small-flower -37 5 3 1.6 0.2 white 0 large large-flower -38 5 3.4 1.6 0.4 white 0 small small-flower -40 5.2 3.5 1.5 0.2 white 0 large large-flower -41 5.2 3.4 1.4 0.2 white 0 medium small-flower -42 4.7 3.2 1.6 0.2 pink 0 large large-flower -43 4.8 1.6 0.2 white 0 small small-flower -45 5.4 3.4 1.5 0.4 white 0 small small-flower -47 5.2 4.1 1.5 0.1 white 0 small small-flower -48 5.5 4.2 1.4 0.2 white 0 small small-flower -50 4.9 1.5 0.2 pink 0 small small-flower -51 5 3.2 0.2 pink 0 small small-flower -53 5.5 3.5 1.3 0.2 white 0 small small-flower -54 4.9 3.6 1.4 0.1 white 0 large large-flower -55 4.4 3 1.3 0.2 white 0 medium small-flower -56 5.1 3.4 1.5 0.2 white 0 small small-flower -57 5 3.5 1.3 0.3 0 large large-flower -59 4.5 2.3 1.3 0.3 white 0 medium small-flower -61 4.4 3.2 1.3 0.2 0 medium small-flower -62 5 3.5 1.6 0.6 white 0 small small-flower -63 5.1 3.8 1.9 0.4 pink 0 medium small-flower -65 4.8 3 1.4 white 0 large large-flower -66 5.1 3.8 1.6 0.2 white 0 large large-flower -68 4.6 3.2 1.4 0.2 white 0 large large-flower -70 5.3 3.7 1.5 0.2 white 0 small small-flower -71 5 3.3 1.4 0.2 white 0 large large-flower -72 7 3.2 4.7 1.4 pink 1 medium small-flower -74 6.4 3.2 4.5 1.5 pink 1 small small-flower -76 6.9 3.1 4.9 1.5 pink 1 medium small-flower -78 5.5 2.3 4 1.3 pink 1 large large-flower -80 6.5 2.8 4.6 1.5 pink 1 large large-flower -81 5.7 2.8 4.5 1.3 purple 1 large large-flower -82 3.3 4.7 1.6 pink 1 small small-flower -84 4.9 2.4 3.3 1 pink 1 small small-flower -85 2.9 4.6 1.3 pink 1 medium small-flower -86 5.2 2.7 3.9 1.4 purple 1 large large-flower -88 5 2 3.5 1 purple 1 large large-flower -90 5.9 3 4.2 pink 1 medium small-flower -91 6 2.2 4 1 pink 1 medium small-flower -93 6.1 2.9 4.7 1.4 pink 1 small small-flower -95 5.6 2.9 3.6 pink 1 large large-flower -97 6.7 3.1 4.4 1.4 1 large large-flower -99 5.6 3 4.5 1.5 pink 1 medium small-flower -101 5.8 2.7 1 pink 1 large large-flower -102 6.2 2.2 4.5 1.5 pink 1 large large-flower -104 5.6 2.5 3.9 1.1 purple 1 large large-flower -105 5.9 3.2 4.8 1.8 pink 1 small small-flower -106 6.1 2.8 4 1.3 pink 1 medium small-flower -108 6.3 2.5 1.5 pink 1 large large-flower -110 6.1 2.8 4.7 1.2 pink 1 small small-flower -112 6.4 2.9 4.3 1.3 pink 1 medium small-flower -113 6.6 3 4.4 1.4 purple 1 small small-flower -115 6.8 2.8 4.8 1.4 purple 1 medium small-flower -116 6.7 3 5 1.7 purple 1 small small-flower -118 6 2.9 4.5 1.5 pink 1 medium small-flower -120 2.6 1 pink 1 large large-flower -122 5.5 2.4 3.8 1.1 pink 1 large large-flower -124 5.5 2.4 3.7 1 pink 1 large large-flower -125 5.8 2.7 3.9 1.2 1 small small-flower -127 2.7 5.1 1.6 purple 1 small small-flower -128 5.4 3 4.5 1.5 pink 1 medium small-flower -129 6 3.4 4.5 1.6 pink 1 large large-flower -130 6.7 3.1 4.7 1.5 purple 1 medium small-flower -131 6.3 4.4 1.3 pink 1 small small-flower -133 5.6 3 4.1 1.3 pink 1 medium small-flower -135 5.5 2.5 4 1.3 pink 1 large large-flower -136 5.5 2.6 4.4 1.2 purple 1 small small-flower -138 3 4.6 1.4 pink 1 medium small-flower -140 5.8 2.6 4 1.2 pink 1 large large-flower -142 5 2.3 3.3 1 pink 1 medium small-flower -143 5.6 2.7 4.2 1.3 pink 1 medium small-flower -144 5.7 3 4.2 1.2 purple 1 medium small-flower -146 5.7 2.9 4.2 1.3 pink 1 medium small-flower -148 6.2 2.9 4.3 1.3 pink 1 large large-flower -150 2.5 3 1.1 pink 1 large large-flower -151 5.7 2.8 4.1 1.3 pink 1 small small-flower -153 6.3 3.3 6 2.5 2 medium small-flower -154 5.8 1.9 purple 2 small small-flower -156 7.1 3 5.9 2.1 pink 2 small small-flower -158 6.3 2.9 5.6 1.8 purple 2 small small-flower -160 6.5 3 2.2 purple 2 large large-flower -161 7.6 3 6.6 2.1 pink 2 small small-flower -162 4.9 2.5 4.5 2 small small-flower -163 7.3 2.9 6.3 1.8 pink 2 small small-flower -165 6.7 2.5 5.8 1.8 white 2 large large-flower -166 7.2 3.6 6.1 2.5 pink 2 small small-flower -168 6.5 3.2 5.1 2 purple 2 small small-flower -170 6.4 2.7 5.3 1.9 purple 2 medium small-flower -171 6.8 3 5.5 2.1 purple 2 large large-flower -172 5.7 2.5 5 2 purple 2 large large-flower -174 5.8 2.8 5.1 2.4 purple 2 small small-flower -176 6.4 3.2 5.3 2.3 purple 2 small small-flower -178 6.5 3 5.5 1.8 purple 2 medium small-flower -179 7.7 3.8 6.7 2.2 purple 2 large large-flower -181 7.7 2.6 2.3 white 2 large large-flower -182 6 2.2 5 1.5 purple 2 large large-flower -183 6.9 3.2 5.7 2.3 2 large large-flower -185 5.6 2.8 4.9 2 white 2 large large-flower -186 7.7 2.8 6.7 2 pink 2 medium small-flower -188 6.3 2.7 4.9 1.8 purple 2 large large-flower -190 6.7 3.3 5.7 2.1 purple 2 small small-flower -191 3.2 6 1.8 purple 2 small small-flower -193 6.2 2.8 4.8 1.8 purple 2 small small-flower -194 6.1 3 4.9 1.8 pink 2 large large-flower -196 6.4 2.8 5.6 2.1 purple 2 small small-flower -198 7.2 3 5.8 1.6 pink 2 large large-flower -199 7.4 2.8 6.1 1.9 purple 2 small small-flower -200 7.9 3.8 6.4 2 pink 2 large large-flower -202 6.4 5.6 2.2 purple 2 small small-flower -203 6.3 2.8 5.1 1.5 purple 2 large large-flower -204 6.1 2.6 5.6 1.4 2 medium small-flower -206 7.7 3 6.1 2.3 purple 2 medium small-flower -207 6.3 3.4 5.6 2.4 pink 2 small small-flower -209 6.4 3.1 5.5 1.8 white 2 small small-flower -211 6 3 4.8 1.8 pink 2 small small-flower -212 6.9 3.1 5.4 white 2 small small-flower -214 6.7 5.6 2.4 pink 2 large large-flower -216 6.9 3.1 5.1 2.3 purple 2 small small-flower -217 5.8 2.7 5.1 1.9 pink 2 small small-flower -218 6.8 3.2 5.9 2.3 purple 2 small small-flower -219 6.7 3.3 5.7 2.5 white 2 large large-flower -221 6.7 3 5.2 2.3 pink 2 small small-flower -222 6.3 2.5 5 1.9 pink 2 medium small-flower -223 6.5 3 5.2 2 pink 2 small small-flower -224 6.2 3.4 5.4 white 2 small small-flower -226 5.9 3 5.1 1.8 purple 2 small small-flower +id sepal length (cm) sepal width (cm) petal length (cm) petal width (cm) color target size flower_category is_flower +1a 5.1 3.5 1.4 0.2 white 0 medium small-flower yes +2 4.9 3 1.4 0.2 white 0 medium small-flower yes +4 4.7 3.2 1.3 white 0 large large-flower yes +5 4.6 3.1 1.5 white 0 large large-flower yes +7b 5 3.6 0.2 white 0 medium small-flower yes +8 5.4 3.9 1.7 0.4 white 0 medium small-flower yes +10 4.6 1.4 0.3 white 0 large large-flower yes +11 5 3.4 1.5 0.2 white 0 small small-flower yes +12 4.4 2.9 1.4 0.2 white 0 small small-flower yes +13 4.9 3.1 1.5 0.1 white 0 medium small-flower yes +14 5.4 3.7 1.5 0.2 white 0 medium small-flower yes +15 4.8 3.4 1.6 0.2 pink 0 large large-flower yes +17 4.8 3 1.4 0.1 white 0 medium small-flower yes +18 1.1 0.1 white 0 medium small-flower yes +19 5.8 4 1.2 0.2 white 0 small small-flower yes +20 5.7 4.4 1.5 0.4 pink 0 medium small-flower yes +21 5.4 3.9 1.3 0.4 white 0 large large-flower yes +23 5.1 3.5 1.4 0.3 white 0 small small-flower yes +25 5.7 3.8 1.7 0.3 pink 0 large large-flower yes +26 5.1 3.8 1.5 0.3 white 0 large large-flower +27 5.4 3.4 1.7 0.2 white 0 small small-flower yes +29 5.1 3.7 1.5 0.4 white 0 large large-flower yes +31 4.6 3.6 1 0.2 white 0 small small-flower yes +33 5.1 3.3 1.7 0.5 pink 0 large large-flower yes +35 4.8 3.4 1.9 0.2 white 0 medium small-flower yes +37 5 3 1.6 0.2 white 0 large large-flower yes +38 5 3.4 1.6 0.4 white 0 small small-flower yes +40 5.2 3.5 1.5 0.2 white 0 large large-flower yes +41 5.2 3.4 1.4 0.2 white 0 medium small-flower yes +42 4.7 3.2 1.6 0.2 pink 0 large large-flower yes +43 4.8 1.6 0.2 white 0 small small-flower yes +45 5.4 3.4 1.5 0.4 white 0 small small-flower yes +47 5.2 4.1 1.5 0.1 white 0 small small-flower yes +48 5.5 4.2 1.4 0.2 white 0 small small-flower yes +50 4.9 1.5 0.2 pink 0 small small-flower yes +51 5 3.2 0.2 pink 0 small small-flower +53 5.5 3.5 1.3 0.2 white 0 small small-flower yes +54 4.9 3.6 1.4 0.1 white 0 large large-flower yes +55 4.4 3 1.3 0.2 white 0 medium small-flower yes +56 5.1 3.4 1.5 0.2 white 0 small small-flower yes +57 5 3.5 1.3 0.3 0 large large-flower yes +59 4.5 2.3 1.3 0.3 white 0 medium small-flower yes +61 4.4 3.2 1.3 0.2 0 medium small-flower yes +62 5 3.5 1.6 0.6 white 0 small small-flower yes +63 5.1 3.8 1.9 0.4 pink 0 medium small-flower yes +65 4.8 3 1.4 white 0 large large-flower yes +66 5.1 3.8 1.6 0.2 white 0 large large-flower yes +68 4.6 3.2 1.4 0.2 white 0 large large-flower yes +70 5.3 3.7 1.5 0.2 white 0 small small-flower yes +71 5 3.3 1.4 0.2 white 0 large large-flower yes +72 7 3.2 4.7 1.4 pink 1 medium small-flower yes +74 6.4 3.2 4.5 1.5 pink 1 small small-flower yes +76 6.9 3.1 4.9 1.5 pink 1 medium small-flower yes +78 5.5 2.3 4 1.3 pink 1 large large-flower yes +80 6.5 2.8 4.6 1.5 pink 1 large large-flower yes +81 5.7 2.8 4.5 1.3 purple 1 large large-flower yes +82 3.3 4.7 1.6 pink 1 small small-flower yes +84 4.9 2.4 3.3 1 pink 1 small small-flower yes +85 2.9 4.6 1.3 pink 1 medium small-flower yes +86 5.2 2.7 3.9 1.4 purple 1 large large-flower yes +88 5 2 3.5 1 purple 1 large large-flower +90 5.9 3 4.2 pink 1 medium small-flower yes +91 6 2.2 4 1 pink 1 medium small-flower yes +93 6.1 2.9 4.7 1.4 pink 1 small small-flower yes +95 5.6 2.9 3.6 pink 1 large large-flower yes +97 6.7 3.1 4.4 1.4 1 large large-flower yes +99 5.6 3 4.5 1.5 pink 1 medium small-flower yes +101 5.8 2.7 1 pink 1 large large-flower yes +102 6.2 2.2 4.5 1.5 pink 1 large large-flower yes +104 5.6 2.5 3.9 1.1 purple 1 large large-flower yes +105 5.9 3.2 4.8 1.8 pink 1 small small-flower yes +106 6.1 2.8 4 1.3 pink 1 medium small-flower yes +108 6.3 2.5 1.5 pink 1 large large-flower yes +110 6.1 2.8 4.7 1.2 pink 1 small small-flower yes +112 6.4 2.9 4.3 1.3 pink 1 medium small-flower yes +113 6.6 3 4.4 1.4 purple 1 small small-flower yes +115 6.8 2.8 4.8 1.4 purple 1 medium small-flower yes +116 6.7 3 5 1.7 purple 1 small small-flower yes +118 6 2.9 4.5 1.5 pink 1 medium small-flower yes +120 2.6 1 pink 1 large large-flower yes +122 5.5 2.4 3.8 1.1 pink 1 large large-flower yes +124 5.5 2.4 3.7 1 pink 1 large large-flower yes +125 5.8 2.7 3.9 1.2 1 small small-flower yes +127 2.7 5.1 1.6 purple 1 small small-flower +128 5.4 3 4.5 1.5 pink 1 medium small-flower yes +129 6 3.4 4.5 1.6 pink 1 large large-flower yes +130 6.7 3.1 4.7 1.5 purple 1 medium small-flower yes +131 6.3 4.4 1.3 pink 1 small small-flower yes +133 5.6 3 4.1 1.3 pink 1 medium small-flower yes +135 5.5 2.5 4 1.3 pink 1 large large-flower yes +136 5.5 2.6 4.4 1.2 purple 1 small small-flower yes +138 3 4.6 1.4 pink 1 medium small-flower yes +140 5.8 2.6 4 1.2 pink 1 large large-flower +142 5 2.3 3.3 1 pink 1 medium small-flower yes +143 5.6 2.7 4.2 1.3 pink 1 medium small-flower yes +144 5.7 3 4.2 1.2 purple 1 medium small-flower yes +146 5.7 2.9 4.2 1.3 pink 1 medium small-flower yes +148 6.2 2.9 4.3 1.3 pink 1 large large-flower yes +150 2.5 3 1.1 pink 1 large large-flower yes +151 5.7 2.8 4.1 1.3 pink 1 small small-flower yes +153 6.3 3.3 6 2.5 2 medium small-flower yes +154 5.8 1.9 purple 2 small small-flower yes +156 7.1 3 5.9 2.1 pink 2 small small-flower yes +158 6.3 2.9 5.6 1.8 purple 2 small small-flower yes +160 6.5 3 2.2 purple 2 large large-flower yes +161 7.6 3 6.6 2.1 pink 2 small small-flower yes +162 4.9 2.5 4.5 2 small small-flower yes +163 7.3 2.9 6.3 1.8 pink 2 small small-flower yes +165 6.7 2.5 5.8 1.8 white 2 large large-flower yes +166 7.2 3.6 6.1 2.5 pink 2 small small-flower yes +168 6.5 3.2 5.1 2 purple 2 small small-flower yes +170 6.4 2.7 5.3 1.9 purple 2 medium small-flower yes +171 6.8 3 5.5 2.1 purple 2 large large-flower yes +172 5.7 2.5 5 2 purple 2 large large-flower yes +174 5.8 2.8 5.1 2.4 purple 2 small small-flower yes +176 6.4 3.2 5.3 2.3 purple 2 small small-flower yes +178 6.5 3 5.5 1.8 purple 2 medium small-flower yes +179 7.7 3.8 6.7 2.2 purple 2 large large-flower yes +181 7.7 2.6 2.3 white 2 large large-flower yes +182 6 2.2 5 1.5 purple 2 large large-flower yes +183 6.9 3.2 5.7 2.3 2 large large-flower yes +185 5.6 2.8 4.9 2 white 2 large large-flower yes +186 7.7 2.8 6.7 2 pink 2 medium small-flower yes +188 6.3 2.7 4.9 1.8 purple 2 large large-flower yes +190 6.7 3.3 5.7 2.1 purple 2 small small-flower yes +191 3.2 6 1.8 purple 2 small small-flower yes +193 6.2 2.8 4.8 1.8 purple 2 small small-flower yes +194 6.1 3 4.9 1.8 pink 2 large large-flower yes +196 6.4 2.8 5.6 2.1 purple 2 small small-flower yes +198 7.2 3 5.8 1.6 pink 2 large large-flower yes +199 7.4 2.8 6.1 1.9 purple 2 small small-flower yes +200 7.9 3.8 6.4 2 pink 2 large large-flower yes +202 6.4 5.6 2.2 purple 2 small small-flower yes +203 6.3 2.8 5.1 1.5 purple 2 large large-flower yes +204 6.1 2.6 5.6 1.4 2 medium small-flower yes +206 7.7 3 6.1 2.3 purple 2 medium small-flower yes +207 6.3 3.4 5.6 2.4 pink 2 small small-flower yes +209 6.4 3.1 5.5 1.8 white 2 small small-flower yes +211 6 3 4.8 1.8 pink 2 small small-flower yes +212 6.9 3.1 5.4 white 2 small small-flower yes +214 6.7 5.6 2.4 pink 2 large large-flower yes +216 6.9 3.1 5.1 2.3 purple 2 small small-flower +217 5.8 2.7 5.1 1.9 pink 2 small small-flower yes +218 6.8 3.2 5.9 2.3 purple 2 small small-flower yes +219 6.7 3.3 5.7 2.5 white 2 large large-flower yes +221 6.7 3 5.2 2.3 pink 2 small small-flower yes +222 6.3 2.5 5 1.9 pink 2 medium small-flower yes +223 6.5 3 5.2 2 pink 2 small small-flower yes +224 6.2 3.4 5.4 white 2 small small-flower yes +226 5.9 3 5.1 1.8 purple 2 small small-flower yes diff --git a/testing/model_configs/log_reg.json b/testing/model_configs/log_reg.json index e603d22..f27af1a 100644 --- a/testing/model_configs/log_reg.json +++ b/testing/model_configs/log_reg.json @@ -8,6 +8,12 @@ "choices": ["l1", "l2", "elasticnet", null] }, "solver": "saga", + "max_iter": { + "label": "max_iter", + "type": "int", + "low": 100, + "high": 2000 + }, "l1_c": { "label": "l1", "type": "float", diff --git a/testing/testing_study_config.json b/testing/testing_study_config.json index 8527b11..836e5b8 100644 --- a/testing/testing_study_config.json +++ b/testing/testing_study_config.json @@ -21,7 +21,8 @@ "sk_f1_perclass", "correct_samples", "incorrect_samples", - "importance_by_permutation" + "importance_by_permutation", + "shap_additive" ] }, "track_params": true,