diff --git a/cobra/preprocessing/preprocessor.py b/cobra/preprocessing/preprocessor.py index fa7ddf1..dba6ff4 100644 --- a/cobra/preprocessing/preprocessor.py +++ b/cobra/preprocessing/preprocessor.py @@ -140,11 +140,11 @@ def from_params( parameter, the bigger the contribution of the overall mean. When set to zero, there is no smoothing (e.g. the pure target incidence is used). imputation_strategy : str, optional + Valid imputation strategies = mean, median, min or max In case there is a particular column which contains new categories, the encoding will lead to NULL values which should be imputed. - Valid strategies are to replace with the global mean of the train - set or the min (resp. max) incidence of the categories of that - particular variable. + For more information about how the imputation works go see the documentation of the + :class:`cobra.preprocessing.TargetEncoder` Returns ------- diff --git a/cobra/preprocessing/target_encoder.py b/cobra/preprocessing/target_encoder.py index 3eda39d..9342a5b 100644 --- a/cobra/preprocessing/target_encoder.py +++ b/cobra/preprocessing/target_encoder.py @@ -18,7 +18,7 @@ class TargetEncoder(BaseEstimator): Note that, when applying this target encoding, values of the categorical feature that have not been seen during fit will be imputed according to the - configured imputation strategy (replacement with the mean, minimum or + configured imputation strategy (replacement with the mean, median, minimum or maximum value of the categorical variable). The main problem with Target encoding is overfitting; the fact that we are @@ -48,8 +48,13 @@ class TargetEncoder(BaseEstimator): In case there is a particular column which contains new categories, the encoding will lead to NULL values which should be imputed. Valid strategies then are to replace the NULL values with the global - mean of the train set or the min (resp. max) incidence of the - categories of that particular variable. + mean, median or the min (resp. max) incidence of the variable. + + Ex: By taking the mean strategy the mean of the known encoded variables + is computed and the missing encoded values would be imputed with this value. + + + weight : float Smoothing parameter (non-negative). The higher the value of the parameter, the bigger the contribution of the overall mean of targets @@ -60,7 +65,7 @@ class TargetEncoder(BaseEstimator): current categorical value is used). """ - valid_imputation_strategies = ("mean", "min", "max") + valid_imputation_strategies = ("mean", "min", "max", "median") def __init__(self, weight: float=0.0, imputation_strategy: str="mean"): @@ -223,13 +228,15 @@ def transform(self, data: pd.DataFrame, Exception when TargetEncoder was not fitted before calling this method. """ + _data = data.copy() + if (len(self._mapping) == 0) or (self._global_mean is None): msg = ("This {} instance is not fitted yet. Call 'fit' with " "appropriate arguments before using this method.") raise NotFittedError(msg.format(self.__class__.__name__)) for column in tqdm(column_names, desc="Applying target encoding..."): - if column not in data.columns: + if column not in _data.columns: log.warning("Unknown column '{}' will be skipped." .format(column)) continue @@ -237,9 +244,9 @@ def transform(self, data: pd.DataFrame, log.warning("Column '{}' is not in fitted output " "and will be skipped.".format(column)) continue - data = self._transform_column(data, column) + _data = self._transform_column(_data, column) - return data + return _data def _transform_column(self, data: pd.DataFrame, column_name: str) -> pd.DataFrame: @@ -260,30 +267,34 @@ def _transform_column(self, data: pd.DataFrame, pd.DataFrame Resulting transformed data. """ + _data = data.copy() new_column = TargetEncoder._clean_column_name(column_name) # Convert dtype to float, because when the original dtype # is of type "category", the resulting dtype would otherwise also be of # type "category": - data[new_column] = (data[column_name].map(self._mapping[column_name]) + _data[new_column] = (_data[column_name].map(self._mapping[column_name]) .astype("float")) # In case of categorical data, it could be that new categories will # emerge which were not present in the train set, so this will result # in missing values, which should be replaced according to the # configured imputation strategy: - if data[new_column].isnull().sum() > 0: + if _data[new_column].isnull().sum() > 0: if self.imputation_strategy == "mean": - data[new_column].fillna(self._global_mean, + _data[new_column].fillna(self._global_mean, inplace=True) elif self.imputation_strategy == "min": - data[new_column].fillna(data[new_column].min(), + _data[new_column].fillna(_data[new_column].min(), inplace=True) elif self.imputation_strategy == "max": - data[new_column].fillna(data[new_column].max(), + _data[new_column].fillna(_data[new_column].max(), + inplace=True) + elif self.imputation_strategy == "median": + _data[new_column].fillna(_data[new_column].median(), inplace=True) - return data + return _data def fit_transform(self, data: pd.DataFrame, column_names: list, diff --git a/tests/preprocessing/test_target_encoder.py b/tests/preprocessing/test_target_encoder.py index 51ebd79..3301121 100644 --- a/tests/preprocessing/test_target_encoder.py +++ b/tests/preprocessing/test_target_encoder.py @@ -13,7 +13,7 @@ def test_target_encoder_constructor_weight_value_error(self): def test_target_encoder_constructor_imputation_value_error(self): with pytest.raises(ValueError): - TargetEncoder(imputation_strategy="median") + TargetEncoder(imputation_strategy="something") # Tests for attributes_attributes_to_dict and set_attributes_from_dict def test_target_encoder_attributes_to_dict(self): @@ -52,12 +52,11 @@ def test_target_encoder_set_attributes_from_dict_unfitted(self, attribute): if attribute == "weight": actual = encoder.weight expected = 1.0 - assert expected == actual + elif attribute == "mapping": actual = encoder._mapping expected = {} - assert expected == actual def test_target_encoder_set_attributes_from_dict(self): @@ -304,6 +303,59 @@ def test_target_encoder_transform_new_category_linear_regression(self): pd.testing.assert_frame_equal(actual, expected) + + def test_target_encoder_transform_new_category_linear_regression_median(self): + df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', + 'neutral', 'negative', 'positive', + 'negative', 'neutral', 'neutral', + 'neutral', 'positive'], + 'target': [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4]}) + + df_appended = df.append({"variable": "new", "target": 10}, + ignore_index=True) + + # inputs of TargetEncoder will be of dtype category + df["variable"] = df["variable"].astype("category") + df_appended["variable"] = df_appended["variable"].astype("category") + + expected = df_appended.copy() + expected["variable_enc"] = [4.500000, 4.500000, -4.666667, 0.250000, + -4.666667, 4.500000, -4.666667, 0.250000, + 0.250000, 0.250000, 4.500000, + 0.250000] # median imputation for new value + + encoder = TargetEncoder(imputation_strategy="median") + encoder.fit(data=df, column_names=["variable"], target_column="target") + actual = encoder.transform(data=df_appended, column_names=["variable"]) + + pd.testing.assert_frame_equal(actual, expected) + + def test_target_encoder_transform_new_category_binary_classification_median(self): + df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', + 'neutral', 'negative', 'positive', + 'negative', 'neutral', 'neutral', + 'neutral'], + 'target': [1, 1, 0, 0, 1, 0, 0, 0, 1, 1]}) + + df_appended = df.append({"variable": "new", "target": 1}, + ignore_index=True) + + # inputs of TargetEncoder will be of dtype category + df["variable"] = df["variable"].astype("category") + df_appended["variable"] = df_appended["variable"].astype("category") + + expected = df_appended.copy() + expected["variable_enc"] = [0.666667, 0.666667, 0.333333, 0.50000, + 0.333333, 0.666667, 0.333333, 0.50000, + 0.50000, 0.50000, 0.50000] + + encoder = TargetEncoder(imputation_strategy="median") + encoder.fit(data=df, column_names=["variable"], target_column="target") + actual = encoder.transform(data=df_appended, column_names=["variable"]) + + pd.testing.assert_frame_equal(actual, expected) + + # Tests for _clean_column_name: def test_target_encoder_clean_column_name_binned_column(self): column_name = "test_column_bin"