From 61ef47c8e47d1893e2d983535b2673e634234cc6 Mon Sep 17 00:00:00 2001 From: Patrick Leonardy Date: Tue, 4 Apr 2023 15:17:27 +0200 Subject: [PATCH 1/4] remove FutureWarnings due to pandas --- cobra/preprocessing/kbins_discretizer.py | 2 +- tests/preprocessing/test_target_encoder.py | 423 +++++++++++++++------ 2 files changed, 314 insertions(+), 111 deletions(-) diff --git a/cobra/preprocessing/kbins_discretizer.py b/cobra/preprocessing/kbins_discretizer.py index c30d7de..ba993bd 100644 --- a/cobra/preprocessing/kbins_discretizer.py +++ b/cobra/preprocessing/kbins_discretizer.py @@ -327,7 +327,7 @@ def _transform_column(self, data: pd.DataFrame, if data[column_name_bin].isnull().sum() > 0: # Add an additional bin for missing values - data[column_name_bin].cat.add_categories(["Missing"], inplace=True) + data[column_name_bin]=data[column_name_bin].cat.add_categories(["Missing"]) # Replace NULL with "Missing" # Otherwise these will be ignored in groupby diff --git a/tests/preprocessing/test_target_encoder.py b/tests/preprocessing/test_target_encoder.py index 51ebd79..bc2211e 100644 --- a/tests/preprocessing/test_target_encoder.py +++ b/tests/preprocessing/test_target_encoder.py @@ -1,12 +1,11 @@ - import pytest import pandas as pd from sklearn.exceptions import NotFittedError from cobra.preprocessing.target_encoder import TargetEncoder -class TestTargetEncoder: +class TestTargetEncoder: def test_target_encoder_constructor_weight_value_error(self): with pytest.raises(ValueError): TargetEncoder(weight=-1) @@ -19,8 +18,10 @@ def test_target_encoder_constructor_imputation_value_error(self): def test_target_encoder_attributes_to_dict(self): encoder = TargetEncoder() - mapping_data = pd.Series(data=[0.333333, 0.50000, 0.666667], - index=["negative", "neutral", "positive"]) + mapping_data = pd.Series( + data=[0.333333, 0.50000, 0.666667], + index=["negative", "neutral", "positive"], + ) mapping_data.index.name = "variable" encoder._mapping["variable"] = mapping_data @@ -29,20 +30,24 @@ def test_target_encoder_attributes_to_dict(self): actual = encoder.attributes_to_dict() - expected = {"weight": 0.0, - "imputation_strategy": "mean", - "_global_mean": 0.5, - "_mapping": {"variable": { - "negative": 0.333333, - "neutral": 0.50000, - "positive": 0.666667 - }}} + expected = { + "weight": 0.0, + "imputation_strategy": "mean", + "_global_mean": 0.5, + "_mapping": { + "variable": { + "negative": 0.333333, + "neutral": 0.50000, + "positive": 0.666667, + } + }, + } assert actual == expected - @pytest.mark.parametrize("attribute", - ["weight", "mapping"], - ids=["test_weight", "test_mapping"]) + @pytest.mark.parametrize( + "attribute", ["weight", "mapping"], ids=["test_weight", "test_mapping"] + ) def test_target_encoder_set_attributes_from_dict_unfitted(self, attribute): encoder = TargetEncoder() @@ -63,18 +68,24 @@ def test_target_encoder_set_attributes_from_dict_unfitted(self, attribute): def test_target_encoder_set_attributes_from_dict(self): encoder = TargetEncoder() - data = {"weight": 0.0, - "_global_mean": 0.5, - "_mapping": {"variable": { + data = { + "weight": 0.0, + "_global_mean": 0.5, + "_mapping": { + "variable": { "negative": 0.333333, "neutral": 0.50000, - "positive": 0.666667 - }}} + "positive": 0.666667, + } + }, + } encoder.set_attributes_from_dict(data) - expected = pd.Series(data=[0.333333, 0.50000, 0.666667], - index=["negative", "neutral", "positive"]) + expected = pd.Series( + data=[0.333333, 0.50000, 0.666667], + index=["negative", "neutral", "positive"], + ) expected.index.name = "variable" actual = encoder._mapping["variable"] @@ -83,63 +94,118 @@ def test_target_encoder_set_attributes_from_dict(self): # Tests for _fit_column: def test_target_encoder_fit_column_binary_classification(self): - df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', - 'neutral', 'negative', 'positive', - 'negative', 'neutral', 'neutral', - 'neutral'], - 'target': [1, 1, 0, 0, 1, 0, 0, 0, 1, 1]}) + df = pd.DataFrame( + { + "variable": [ + "positive", + "positive", + "negative", + "neutral", + "negative", + "positive", + "negative", + "neutral", + "neutral", + "neutral", + ], + "target": [1, 1, 0, 0, 1, 0, 0, 0, 1, 1], + } + ) encoder = TargetEncoder() encoder._global_mean = 0.5 actual = encoder._fit_column(X=df.variable, y=df.target) - expected = pd.Series(data=[0.333333, 0.50000, 0.666667], - index=["negative", "neutral", "positive"]) + expected = pd.Series( + data=[0.333333, 0.50000, 0.666667], + index=["negative", "neutral", "positive"], + ) expected.index.name = "variable" pd.testing.assert_series_equal(actual, expected) def test_target_encoder_fit_column_linear_regression(self): - df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', - 'neutral', 'negative', 'positive', - 'negative', 'neutral', 'neutral', - 'neutral', 'positive'], - 'target': [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4]}) + df = pd.DataFrame( + { + "variable": [ + "positive", + "positive", + "negative", + "neutral", + "negative", + "positive", + "negative", + "neutral", + "neutral", + "neutral", + "positive", + ], + "target": [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4], + } + ) encoder = TargetEncoder() encoder._global_mean = 0.454545 actual = encoder._fit_column(X=df.variable, y=df.target) - expected = pd.Series(data=[-4.666667, 0.250000, 4.500000], - index=["negative", "neutral", "positive"]) + expected = pd.Series( + data=[-4.666667, 0.250000, 4.500000], + index=["negative", "neutral", "positive"], + ) expected.index.name = "variable" pd.testing.assert_series_equal(actual, expected) def test_target_encoder_fit_column_global_mean_binary_classification(self): - df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', - 'neutral', 'negative', 'positive', - 'negative', 'neutral', 'neutral', - 'neutral'], - 'target': [1, 1, 0, 0, 1, 0, 0, 0, 1, 1]}) + df = pd.DataFrame( + { + "variable": [ + "positive", + "positive", + "negative", + "neutral", + "negative", + "positive", + "negative", + "neutral", + "neutral", + "neutral", + ], + "target": [1, 1, 0, 0, 1, 0, 0, 0, 1, 1], + } + ) encoder = TargetEncoder(weight=1) encoder._global_mean = df.target.sum() / df.target.count() # is 0.5 actual = encoder._fit_column(X=df.variable, y=df.target) - expected = pd.Series(data=[0.375, 0.500, 0.625], - index=["negative", "neutral", "positive"]) + expected = pd.Series( + data=[0.375, 0.500, 0.625], index=["negative", "neutral", "positive"] + ) expected.index.name = "variable" pd.testing.assert_series_equal(actual, expected) def test_target_encoder_fit_column_global_mean_linear_regression(self): - df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', - 'neutral', 'negative', 'positive', - 'negative', 'neutral', 'neutral', - 'neutral', 'positive'], - 'target': [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4]}) + df = pd.DataFrame( + { + "variable": [ + "positive", + "positive", + "negative", + "neutral", + "negative", + "positive", + "negative", + "neutral", + "neutral", + "neutral", + "positive", + ], + "target": [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4], + } + ) encoder = TargetEncoder(weight=1) encoder._global_mean = 0.454545 @@ -149,10 +215,14 @@ def test_target_encoder_fit_column_global_mean_linear_regression(self): # expected new value: # [count of the value * its mean encoding + weight (= 1) * global mean] # / [count of the value + weight (=1)]. - expected = pd.Series(data=[(3 * -4.666667 + 1 * 0.454545) / (3 + 1), - (4 * 0.250000 + 1 * 0.454545) / (4 + 1), - (4 * 4.500000 + 1 * 0.454545) / (4 + 1)], - index=["negative", "neutral", "positive"]) + expected = pd.Series( + data=[ + (3 * -4.666667 + 1 * 0.454545) / (3 + 1), + (4 * 0.250000 + 1 * 0.454545) / (4 + 1), + (4 * 4.500000 + 1 * 0.454545) / (4 + 1), + ], + index=["negative", "neutral", "positive"], + ) expected.index.name = "variable" pd.testing.assert_series_equal(actual, expected) @@ -161,17 +231,31 @@ def test_target_encoder_fit_column_global_mean_linear_regression(self): def test_target_encoder_fit_binary_classification(self): # test_target_encoder_fit_column_linear_regression() tested on one # column input as a numpy series; this test runs on a dataframe input. - df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', - 'neutral', 'negative', 'positive', - 'negative', 'neutral', 'neutral', - 'neutral'], - 'target': [1, 1, 0, 0, 1, 0, 0, 0, 1, 1]}) + df = pd.DataFrame( + { + "variable": [ + "positive", + "positive", + "negative", + "neutral", + "negative", + "positive", + "negative", + "neutral", + "neutral", + "neutral", + ], + "target": [1, 1, 0, 0, 1, 0, 0, 0, 1, 1], + } + ) encoder = TargetEncoder() encoder.fit(data=df, column_names=["variable"], target_column="target") - expected = pd.Series(data=[0.333333, 0.50000, 0.666667], - index=["negative", "neutral", "positive"]) + expected = pd.Series( + data=[0.333333, 0.50000, 0.666667], + index=["negative", "neutral", "positive"], + ) expected.index.name = "variable" actual = encoder._mapping["variable"] @@ -180,17 +264,32 @@ def test_target_encoder_fit_binary_classification(self): def test_target_encoder_fit_linear_regression(self): # test_target_encoder_fit_column_linear_regression() tested on one # column input as a numpy series; this test runs on a dataframe input. - df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', - 'neutral', 'negative', 'positive', - 'negative', 'neutral', 'neutral', - 'neutral', 'positive'], - 'target': [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4]}) + df = pd.DataFrame( + { + "variable": [ + "positive", + "positive", + "negative", + "neutral", + "negative", + "positive", + "negative", + "neutral", + "neutral", + "neutral", + "positive", + ], + "target": [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4], + } + ) encoder = TargetEncoder() encoder.fit(data=df, column_names=["variable"], target_column="target") - expected = pd.Series(data=[-4.666667, 0.250000, 4.500000], - index=["negative", "neutral", "positive"]) + expected = pd.Series( + data=[-4.666667, 0.250000, 4.500000], + index=["negative", "neutral", "positive"], + ) expected.index.name = "variable" actual = encoder._mapping["variable"] @@ -198,11 +297,23 @@ def test_target_encoder_fit_linear_regression(self): # Tests for transform method def test_target_encoder_transform_when_not_fitted(self): - df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', - 'neutral', 'negative', 'positive', - 'negative', 'neutral', 'neutral', - 'neutral'], - 'target': [1, 1, 0, 0, 1, 0, 0, 0, 1, 1]}) + df = pd.DataFrame( + { + "variable": [ + "positive", + "positive", + "negative", + "neutral", + "negative", + "positive", + "negative", + "neutral", + "neutral", + "neutral", + ], + "target": [1, 1, 0, 0, 1, 0, 0, 0, 1, 1], + } + ) # inputs of TargetEncoder will be of dtype category df["variable"] = df["variable"].astype("category") @@ -212,19 +323,40 @@ def test_target_encoder_transform_when_not_fitted(self): encoder.transform(data=df, column_names=["variable"]) def test_target_encoder_transform_binary_classification(self): - df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', - 'neutral', 'negative', 'positive', - 'negative', 'neutral', 'neutral', - 'neutral'], - 'target': [1, 1, 0, 0, 1, 0, 0, 0, 1, 1]}) + df = pd.DataFrame( + { + "variable": [ + "positive", + "positive", + "negative", + "neutral", + "negative", + "positive", + "negative", + "neutral", + "neutral", + "neutral", + ], + "target": [1, 1, 0, 0, 1, 0, 0, 0, 1, 1], + } + ) # inputs of TargetEncoder will be of dtype category df["variable"] = df["variable"].astype("category") expected = df.copy() - expected["variable_enc"] = [0.666667, 0.666667, 0.333333, 0.50000, - 0.333333, 0.666667, 0.333333, 0.50000, - 0.50000, 0.50000] + expected["variable_enc"] = [ + 0.666667, + 0.666667, + 0.333333, + 0.50000, + 0.333333, + 0.666667, + 0.333333, + 0.50000, + 0.50000, + 0.50000, + ] encoder = TargetEncoder() encoder.fit(data=df, column_names=["variable"], target_column="target") @@ -233,19 +365,42 @@ def test_target_encoder_transform_binary_classification(self): pd.testing.assert_frame_equal(actual, expected) def test_target_encoder_transform_linear_regression(self): - df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', - 'neutral', 'negative', 'positive', - 'negative', 'neutral', 'neutral', - 'neutral', 'positive'], - 'target': [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4]}) + df = pd.DataFrame( + { + "variable": [ + "positive", + "positive", + "negative", + "neutral", + "negative", + "positive", + "negative", + "neutral", + "neutral", + "neutral", + "positive", + ], + "target": [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4], + } + ) # inputs of TargetEncoder will be of dtype category df["variable"] = df["variable"].astype("category") expected = df.copy() - expected["variable_enc"] = [4.500000, 4.500000, -4.666667, 0.250000, - -4.666667, 4.500000, -4.666667, 0.250000, - 0.250000, 0.250000, 4.500000] + expected["variable_enc"] = [ + 4.500000, + 4.500000, + -4.666667, + 0.250000, + -4.666667, + 4.500000, + -4.666667, + 0.250000, + 0.250000, + 0.250000, + 4.500000, + ] encoder = TargetEncoder() encoder.fit(data=df, column_names=["variable"], target_column="target") @@ -254,23 +409,46 @@ def test_target_encoder_transform_linear_regression(self): pd.testing.assert_frame_equal(actual, expected) def test_target_encoder_transform_new_category_binary_classification(self): - df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', - 'neutral', 'negative', 'positive', - 'negative', 'neutral', 'neutral', - 'neutral'], - 'target': [1, 1, 0, 0, 1, 0, 0, 0, 1, 1]}) - - df_appended = df.append({"variable": "new", "target": 1}, - ignore_index=True) + df = pd.DataFrame( + { + "variable": [ + "positive", + "positive", + "negative", + "neutral", + "negative", + "positive", + "negative", + "neutral", + "neutral", + "neutral", + ], + "target": [1, 1, 0, 0, 1, 0, 0, 0, 1, 1], + } + ) + df_appended = pd.concat( + [df, pd.Series({"variable": "new", "target": 1}).to_frame().T], + axis=0, + ) # inputs of TargetEncoder will be of dtype category df["variable"] = df["variable"].astype("category") df_appended["variable"] = df_appended["variable"].astype("category") expected = df_appended.copy() - expected["variable_enc"] = [0.666667, 0.666667, 0.333333, 0.50000, - 0.333333, 0.666667, 0.333333, 0.50000, - 0.50000, 0.50000, 0.333333] + expected["variable_enc"] = [ + 0.666667, + 0.666667, + 0.333333, + 0.50000, + 0.333333, + 0.666667, + 0.333333, + 0.50000, + 0.50000, + 0.50000, + 0.333333, + ] encoder = TargetEncoder(imputation_strategy="min") encoder.fit(data=df, column_names=["variable"], target_column="target") @@ -279,24 +457,49 @@ def test_target_encoder_transform_new_category_binary_classification(self): pd.testing.assert_frame_equal(actual, expected) def test_target_encoder_transform_new_category_linear_regression(self): - df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', - 'neutral', 'negative', 'positive', - 'negative', 'neutral', 'neutral', - 'neutral', 'positive'], - 'target': [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4]}) - - df_appended = df.append({"variable": "new", "target": 10}, - ignore_index=True) + df = pd.DataFrame( + { + "variable": [ + "positive", + "positive", + "negative", + "neutral", + "negative", + "positive", + "negative", + "neutral", + "neutral", + "neutral", + "positive", + ], + "target": [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4], + } + ) + + df_appended = pd.concat( + [df, pd.Series({"variable": "new", "target": 10}).to_frame().T], + axis=0, + ) # inputs of TargetEncoder will be of dtype category df["variable"] = df["variable"].astype("category") df_appended["variable"] = df_appended["variable"].astype("category") expected = df_appended.copy() - expected["variable_enc"] = [4.500000, 4.500000, -4.666667, 0.250000, - -4.666667, 4.500000, -4.666667, 0.250000, - 0.250000, 0.250000, 4.500000, - -4.666667] # min imputation for new value + expected["variable_enc"] = [ + 4.500000, + 4.500000, + -4.666667, + 0.250000, + -4.666667, + 4.500000, + -4.666667, + 0.250000, + 0.250000, + 0.250000, + 4.500000, + -4.666667, + ] # min imputation for new value encoder = TargetEncoder(imputation_strategy="min") encoder.fit(data=df, column_names=["variable"], target_column="target") From 9fb464fb77eb2c5180db7ce357c3d55da2fd2c59 Mon Sep 17 00:00:00 2001 From: patrickleonardy Date: Wed, 5 Apr 2023 12:01:53 +0200 Subject: [PATCH 2/4] CHANGE d.loc[:, "colname"]= TO d["colname"]= --- cobra/preprocessing/categorical_data_processor.py | 8 ++++---- cobra/preprocessing/kbins_discretizer.py | 4 ++-- tests/preprocessing/test_preprocessor.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cobra/preprocessing/categorical_data_processor.py b/cobra/preprocessing/categorical_data_processor.py index 175bfb5..bc7f733 100644 --- a/cobra/preprocessing/categorical_data_processor.py +++ b/cobra/preprocessing/categorical_data_processor.py @@ -309,10 +309,10 @@ def _transform_column(self, data: pd.DataFrame, """ column_name_clean = column_name + "_processed" - data.loc[:, column_name_clean] = data[column_name].astype(object) + data[column_name_clean] = data[column_name].astype(object) # Fill missings first - data.loc[:, column_name_clean] = (CategoricalDataProcessor + data[column_name_clean] = (CategoricalDataProcessor ._replace_missings( data, column_name_clean @@ -329,14 +329,14 @@ def _transform_column(self, data: pd.DataFrame, "and will be skipped".format(column_name)) return data - data.loc[:, column_name_clean] = (CategoricalDataProcessor + data[column_name_clean] = (CategoricalDataProcessor ._replace_categories( data[column_name_clean], categories, self.regroup_name)) # change data to categorical - data.loc[:, column_name_clean] = (data[column_name_clean] + data[column_name_clean] = (data[column_name_clean] .astype("category")) return data diff --git a/cobra/preprocessing/kbins_discretizer.py b/cobra/preprocessing/kbins_discretizer.py index ba993bd..e30b834 100644 --- a/cobra/preprocessing/kbins_discretizer.py +++ b/cobra/preprocessing/kbins_discretizer.py @@ -315,13 +315,13 @@ def _transform_column(self, data: pd.DataFrame, column_name_bin = column_name + "_bin" # use pd.cut to compute bins - data.loc[:, column_name_bin] = pd.cut(x=data[column_name], + data[column_name_bin] = pd.cut(x=data[column_name], bins=interval_idx) # Rename bins so that the output has a proper format bin_labels = self._create_bin_labels(bins) - data.loc[:, column_name_bin] = (data[column_name_bin] + data[column_name_bin] = (data[column_name_bin] .cat.rename_categories(bin_labels)) if data[column_name_bin].isnull().sum() > 0: diff --git a/tests/preprocessing/test_preprocessor.py b/tests/preprocessing/test_preprocessor.py index 08f5b63..6d69bd9 100644 --- a/tests/preprocessing/test_preprocessor.py +++ b/tests/preprocessing/test_preprocessor.py @@ -35,7 +35,7 @@ def test_train_selection_validation_split( ): X = np.arange(100).reshape(10, 10) data = pd.DataFrame(X, columns=[f"c{i+1}" for i in range(10)]) - data.loc[:, "target"] = np.array([0] * 7 + [1] * 3) + data["target"] = np.array([0] * 7 + [1] * 3) actual = PreProcessor.train_selection_validation_split( data, train_prop, selection_prop, validation_prop From 73447476cc4e910e7a17c4afdd46bc3714d3019b Mon Sep 17 00:00:00 2001 From: patrickleonardy Date: Wed, 5 Apr 2023 12:23:56 +0200 Subject: [PATCH 3/4] Revert "remove FutureWarnings due to pandas" This reverts commit 61ef47c8e47d1893e2d983535b2673e634234cc6. --- cobra/preprocessing/kbins_discretizer.py | 2 +- tests/preprocessing/test_target_encoder.py | 423 ++++++--------------- 2 files changed, 111 insertions(+), 314 deletions(-) diff --git a/cobra/preprocessing/kbins_discretizer.py b/cobra/preprocessing/kbins_discretizer.py index e30b834..56fbe0c 100644 --- a/cobra/preprocessing/kbins_discretizer.py +++ b/cobra/preprocessing/kbins_discretizer.py @@ -327,7 +327,7 @@ def _transform_column(self, data: pd.DataFrame, if data[column_name_bin].isnull().sum() > 0: # Add an additional bin for missing values - data[column_name_bin]=data[column_name_bin].cat.add_categories(["Missing"]) + data[column_name_bin].cat.add_categories(["Missing"], inplace=True) # Replace NULL with "Missing" # Otherwise these will be ignored in groupby diff --git a/tests/preprocessing/test_target_encoder.py b/tests/preprocessing/test_target_encoder.py index bc2211e..51ebd79 100644 --- a/tests/preprocessing/test_target_encoder.py +++ b/tests/preprocessing/test_target_encoder.py @@ -1,11 +1,12 @@ + import pytest import pandas as pd from sklearn.exceptions import NotFittedError from cobra.preprocessing.target_encoder import TargetEncoder - class TestTargetEncoder: + def test_target_encoder_constructor_weight_value_error(self): with pytest.raises(ValueError): TargetEncoder(weight=-1) @@ -18,10 +19,8 @@ def test_target_encoder_constructor_imputation_value_error(self): def test_target_encoder_attributes_to_dict(self): encoder = TargetEncoder() - mapping_data = pd.Series( - data=[0.333333, 0.50000, 0.666667], - index=["negative", "neutral", "positive"], - ) + mapping_data = pd.Series(data=[0.333333, 0.50000, 0.666667], + index=["negative", "neutral", "positive"]) mapping_data.index.name = "variable" encoder._mapping["variable"] = mapping_data @@ -30,24 +29,20 @@ def test_target_encoder_attributes_to_dict(self): actual = encoder.attributes_to_dict() - expected = { - "weight": 0.0, - "imputation_strategy": "mean", - "_global_mean": 0.5, - "_mapping": { - "variable": { - "negative": 0.333333, - "neutral": 0.50000, - "positive": 0.666667, - } - }, - } + expected = {"weight": 0.0, + "imputation_strategy": "mean", + "_global_mean": 0.5, + "_mapping": {"variable": { + "negative": 0.333333, + "neutral": 0.50000, + "positive": 0.666667 + }}} assert actual == expected - @pytest.mark.parametrize( - "attribute", ["weight", "mapping"], ids=["test_weight", "test_mapping"] - ) + @pytest.mark.parametrize("attribute", + ["weight", "mapping"], + ids=["test_weight", "test_mapping"]) def test_target_encoder_set_attributes_from_dict_unfitted(self, attribute): encoder = TargetEncoder() @@ -68,24 +63,18 @@ def test_target_encoder_set_attributes_from_dict_unfitted(self, attribute): def test_target_encoder_set_attributes_from_dict(self): encoder = TargetEncoder() - data = { - "weight": 0.0, - "_global_mean": 0.5, - "_mapping": { - "variable": { + data = {"weight": 0.0, + "_global_mean": 0.5, + "_mapping": {"variable": { "negative": 0.333333, "neutral": 0.50000, - "positive": 0.666667, - } - }, - } + "positive": 0.666667 + }}} encoder.set_attributes_from_dict(data) - expected = pd.Series( - data=[0.333333, 0.50000, 0.666667], - index=["negative", "neutral", "positive"], - ) + expected = pd.Series(data=[0.333333, 0.50000, 0.666667], + index=["negative", "neutral", "positive"]) expected.index.name = "variable" actual = encoder._mapping["variable"] @@ -94,118 +83,63 @@ def test_target_encoder_set_attributes_from_dict(self): # Tests for _fit_column: def test_target_encoder_fit_column_binary_classification(self): - df = pd.DataFrame( - { - "variable": [ - "positive", - "positive", - "negative", - "neutral", - "negative", - "positive", - "negative", - "neutral", - "neutral", - "neutral", - ], - "target": [1, 1, 0, 0, 1, 0, 0, 0, 1, 1], - } - ) + df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', + 'neutral', 'negative', 'positive', + 'negative', 'neutral', 'neutral', + 'neutral'], + 'target': [1, 1, 0, 0, 1, 0, 0, 0, 1, 1]}) encoder = TargetEncoder() encoder._global_mean = 0.5 actual = encoder._fit_column(X=df.variable, y=df.target) - expected = pd.Series( - data=[0.333333, 0.50000, 0.666667], - index=["negative", "neutral", "positive"], - ) + expected = pd.Series(data=[0.333333, 0.50000, 0.666667], + index=["negative", "neutral", "positive"]) expected.index.name = "variable" pd.testing.assert_series_equal(actual, expected) def test_target_encoder_fit_column_linear_regression(self): - df = pd.DataFrame( - { - "variable": [ - "positive", - "positive", - "negative", - "neutral", - "negative", - "positive", - "negative", - "neutral", - "neutral", - "neutral", - "positive", - ], - "target": [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4], - } - ) + df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', + 'neutral', 'negative', 'positive', + 'negative', 'neutral', 'neutral', + 'neutral', 'positive'], + 'target': [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4]}) encoder = TargetEncoder() encoder._global_mean = 0.454545 actual = encoder._fit_column(X=df.variable, y=df.target) - expected = pd.Series( - data=[-4.666667, 0.250000, 4.500000], - index=["negative", "neutral", "positive"], - ) + expected = pd.Series(data=[-4.666667, 0.250000, 4.500000], + index=["negative", "neutral", "positive"]) expected.index.name = "variable" pd.testing.assert_series_equal(actual, expected) def test_target_encoder_fit_column_global_mean_binary_classification(self): - df = pd.DataFrame( - { - "variable": [ - "positive", - "positive", - "negative", - "neutral", - "negative", - "positive", - "negative", - "neutral", - "neutral", - "neutral", - ], - "target": [1, 1, 0, 0, 1, 0, 0, 0, 1, 1], - } - ) + df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', + 'neutral', 'negative', 'positive', + 'negative', 'neutral', 'neutral', + 'neutral'], + 'target': [1, 1, 0, 0, 1, 0, 0, 0, 1, 1]}) encoder = TargetEncoder(weight=1) encoder._global_mean = df.target.sum() / df.target.count() # is 0.5 actual = encoder._fit_column(X=df.variable, y=df.target) - expected = pd.Series( - data=[0.375, 0.500, 0.625], index=["negative", "neutral", "positive"] - ) + expected = pd.Series(data=[0.375, 0.500, 0.625], + index=["negative", "neutral", "positive"]) expected.index.name = "variable" pd.testing.assert_series_equal(actual, expected) def test_target_encoder_fit_column_global_mean_linear_regression(self): - df = pd.DataFrame( - { - "variable": [ - "positive", - "positive", - "negative", - "neutral", - "negative", - "positive", - "negative", - "neutral", - "neutral", - "neutral", - "positive", - ], - "target": [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4], - } - ) + df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', + 'neutral', 'negative', 'positive', + 'negative', 'neutral', 'neutral', + 'neutral', 'positive'], + 'target': [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4]}) encoder = TargetEncoder(weight=1) encoder._global_mean = 0.454545 @@ -215,14 +149,10 @@ def test_target_encoder_fit_column_global_mean_linear_regression(self): # expected new value: # [count of the value * its mean encoding + weight (= 1) * global mean] # / [count of the value + weight (=1)]. - expected = pd.Series( - data=[ - (3 * -4.666667 + 1 * 0.454545) / (3 + 1), - (4 * 0.250000 + 1 * 0.454545) / (4 + 1), - (4 * 4.500000 + 1 * 0.454545) / (4 + 1), - ], - index=["negative", "neutral", "positive"], - ) + expected = pd.Series(data=[(3 * -4.666667 + 1 * 0.454545) / (3 + 1), + (4 * 0.250000 + 1 * 0.454545) / (4 + 1), + (4 * 4.500000 + 1 * 0.454545) / (4 + 1)], + index=["negative", "neutral", "positive"]) expected.index.name = "variable" pd.testing.assert_series_equal(actual, expected) @@ -231,31 +161,17 @@ def test_target_encoder_fit_column_global_mean_linear_regression(self): def test_target_encoder_fit_binary_classification(self): # test_target_encoder_fit_column_linear_regression() tested on one # column input as a numpy series; this test runs on a dataframe input. - df = pd.DataFrame( - { - "variable": [ - "positive", - "positive", - "negative", - "neutral", - "negative", - "positive", - "negative", - "neutral", - "neutral", - "neutral", - ], - "target": [1, 1, 0, 0, 1, 0, 0, 0, 1, 1], - } - ) + df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', + 'neutral', 'negative', 'positive', + 'negative', 'neutral', 'neutral', + 'neutral'], + 'target': [1, 1, 0, 0, 1, 0, 0, 0, 1, 1]}) encoder = TargetEncoder() encoder.fit(data=df, column_names=["variable"], target_column="target") - expected = pd.Series( - data=[0.333333, 0.50000, 0.666667], - index=["negative", "neutral", "positive"], - ) + expected = pd.Series(data=[0.333333, 0.50000, 0.666667], + index=["negative", "neutral", "positive"]) expected.index.name = "variable" actual = encoder._mapping["variable"] @@ -264,32 +180,17 @@ def test_target_encoder_fit_binary_classification(self): def test_target_encoder_fit_linear_regression(self): # test_target_encoder_fit_column_linear_regression() tested on one # column input as a numpy series; this test runs on a dataframe input. - df = pd.DataFrame( - { - "variable": [ - "positive", - "positive", - "negative", - "neutral", - "negative", - "positive", - "negative", - "neutral", - "neutral", - "neutral", - "positive", - ], - "target": [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4], - } - ) + df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', + 'neutral', 'negative', 'positive', + 'negative', 'neutral', 'neutral', + 'neutral', 'positive'], + 'target': [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4]}) encoder = TargetEncoder() encoder.fit(data=df, column_names=["variable"], target_column="target") - expected = pd.Series( - data=[-4.666667, 0.250000, 4.500000], - index=["negative", "neutral", "positive"], - ) + expected = pd.Series(data=[-4.666667, 0.250000, 4.500000], + index=["negative", "neutral", "positive"]) expected.index.name = "variable" actual = encoder._mapping["variable"] @@ -297,23 +198,11 @@ def test_target_encoder_fit_linear_regression(self): # Tests for transform method def test_target_encoder_transform_when_not_fitted(self): - df = pd.DataFrame( - { - "variable": [ - "positive", - "positive", - "negative", - "neutral", - "negative", - "positive", - "negative", - "neutral", - "neutral", - "neutral", - ], - "target": [1, 1, 0, 0, 1, 0, 0, 0, 1, 1], - } - ) + df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', + 'neutral', 'negative', 'positive', + 'negative', 'neutral', 'neutral', + 'neutral'], + 'target': [1, 1, 0, 0, 1, 0, 0, 0, 1, 1]}) # inputs of TargetEncoder will be of dtype category df["variable"] = df["variable"].astype("category") @@ -323,40 +212,19 @@ def test_target_encoder_transform_when_not_fitted(self): encoder.transform(data=df, column_names=["variable"]) def test_target_encoder_transform_binary_classification(self): - df = pd.DataFrame( - { - "variable": [ - "positive", - "positive", - "negative", - "neutral", - "negative", - "positive", - "negative", - "neutral", - "neutral", - "neutral", - ], - "target": [1, 1, 0, 0, 1, 0, 0, 0, 1, 1], - } - ) + df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', + 'neutral', 'negative', 'positive', + 'negative', 'neutral', 'neutral', + 'neutral'], + 'target': [1, 1, 0, 0, 1, 0, 0, 0, 1, 1]}) # inputs of TargetEncoder will be of dtype category df["variable"] = df["variable"].astype("category") expected = df.copy() - expected["variable_enc"] = [ - 0.666667, - 0.666667, - 0.333333, - 0.50000, - 0.333333, - 0.666667, - 0.333333, - 0.50000, - 0.50000, - 0.50000, - ] + expected["variable_enc"] = [0.666667, 0.666667, 0.333333, 0.50000, + 0.333333, 0.666667, 0.333333, 0.50000, + 0.50000, 0.50000] encoder = TargetEncoder() encoder.fit(data=df, column_names=["variable"], target_column="target") @@ -365,42 +233,19 @@ def test_target_encoder_transform_binary_classification(self): pd.testing.assert_frame_equal(actual, expected) def test_target_encoder_transform_linear_regression(self): - df = pd.DataFrame( - { - "variable": [ - "positive", - "positive", - "negative", - "neutral", - "negative", - "positive", - "negative", - "neutral", - "neutral", - "neutral", - "positive", - ], - "target": [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4], - } - ) + df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', + 'neutral', 'negative', 'positive', + 'negative', 'neutral', 'neutral', + 'neutral', 'positive'], + 'target': [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4]}) # inputs of TargetEncoder will be of dtype category df["variable"] = df["variable"].astype("category") expected = df.copy() - expected["variable_enc"] = [ - 4.500000, - 4.500000, - -4.666667, - 0.250000, - -4.666667, - 4.500000, - -4.666667, - 0.250000, - 0.250000, - 0.250000, - 4.500000, - ] + expected["variable_enc"] = [4.500000, 4.500000, -4.666667, 0.250000, + -4.666667, 4.500000, -4.666667, 0.250000, + 0.250000, 0.250000, 4.500000] encoder = TargetEncoder() encoder.fit(data=df, column_names=["variable"], target_column="target") @@ -409,46 +254,23 @@ def test_target_encoder_transform_linear_regression(self): pd.testing.assert_frame_equal(actual, expected) def test_target_encoder_transform_new_category_binary_classification(self): - df = pd.DataFrame( - { - "variable": [ - "positive", - "positive", - "negative", - "neutral", - "negative", - "positive", - "negative", - "neutral", - "neutral", - "neutral", - ], - "target": [1, 1, 0, 0, 1, 0, 0, 0, 1, 1], - } - ) - df_appended = pd.concat( - [df, pd.Series({"variable": "new", "target": 1}).to_frame().T], - axis=0, - ) + df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', + 'neutral', 'negative', 'positive', + 'negative', 'neutral', 'neutral', + 'neutral'], + 'target': [1, 1, 0, 0, 1, 0, 0, 0, 1, 1]}) + + df_appended = df.append({"variable": "new", "target": 1}, + ignore_index=True) # inputs of TargetEncoder will be of dtype category df["variable"] = df["variable"].astype("category") df_appended["variable"] = df_appended["variable"].astype("category") expected = df_appended.copy() - expected["variable_enc"] = [ - 0.666667, - 0.666667, - 0.333333, - 0.50000, - 0.333333, - 0.666667, - 0.333333, - 0.50000, - 0.50000, - 0.50000, - 0.333333, - ] + expected["variable_enc"] = [0.666667, 0.666667, 0.333333, 0.50000, + 0.333333, 0.666667, 0.333333, 0.50000, + 0.50000, 0.50000, 0.333333] encoder = TargetEncoder(imputation_strategy="min") encoder.fit(data=df, column_names=["variable"], target_column="target") @@ -457,49 +279,24 @@ def test_target_encoder_transform_new_category_binary_classification(self): pd.testing.assert_frame_equal(actual, expected) def test_target_encoder_transform_new_category_linear_regression(self): - df = pd.DataFrame( - { - "variable": [ - "positive", - "positive", - "negative", - "neutral", - "negative", - "positive", - "negative", - "neutral", - "neutral", - "neutral", - "positive", - ], - "target": [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4], - } - ) - - df_appended = pd.concat( - [df, pd.Series({"variable": "new", "target": 10}).to_frame().T], - axis=0, - ) + df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', + 'neutral', 'negative', 'positive', + 'negative', 'neutral', 'neutral', + 'neutral', 'positive'], + 'target': [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4]}) + + df_appended = df.append({"variable": "new", "target": 10}, + ignore_index=True) # inputs of TargetEncoder will be of dtype category df["variable"] = df["variable"].astype("category") df_appended["variable"] = df_appended["variable"].astype("category") expected = df_appended.copy() - expected["variable_enc"] = [ - 4.500000, - 4.500000, - -4.666667, - 0.250000, - -4.666667, - 4.500000, - -4.666667, - 0.250000, - 0.250000, - 0.250000, - 4.500000, - -4.666667, - ] # min imputation for new value + expected["variable_enc"] = [4.500000, 4.500000, -4.666667, 0.250000, + -4.666667, 4.500000, -4.666667, 0.250000, + 0.250000, 0.250000, 4.500000, + -4.666667] # min imputation for new value encoder = TargetEncoder(imputation_strategy="min") encoder.fit(data=df, column_names=["variable"], target_column="target") From 10338e4c4f5dfddab1c2180f5d5594dcb58523e9 Mon Sep 17 00:00:00 2001 From: patrickleonardy Date: Wed, 5 Apr 2023 12:31:57 +0200 Subject: [PATCH 4/4] format test back and fix further FutureWarnings --- cobra/preprocessing/kbins_discretizer.py | 2 +- tests/preprocessing/test_target_encoder.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cobra/preprocessing/kbins_discretizer.py b/cobra/preprocessing/kbins_discretizer.py index 56fbe0c..e30b834 100644 --- a/cobra/preprocessing/kbins_discretizer.py +++ b/cobra/preprocessing/kbins_discretizer.py @@ -327,7 +327,7 @@ def _transform_column(self, data: pd.DataFrame, if data[column_name_bin].isnull().sum() > 0: # Add an additional bin for missing values - data[column_name_bin].cat.add_categories(["Missing"], inplace=True) + data[column_name_bin]=data[column_name_bin].cat.add_categories(["Missing"]) # Replace NULL with "Missing" # Otherwise these will be ignored in groupby diff --git a/tests/preprocessing/test_target_encoder.py b/tests/preprocessing/test_target_encoder.py index 51ebd79..f477bad 100644 --- a/tests/preprocessing/test_target_encoder.py +++ b/tests/preprocessing/test_target_encoder.py @@ -260,8 +260,7 @@ def test_target_encoder_transform_new_category_binary_classification(self): 'neutral'], 'target': [1, 1, 0, 0, 1, 0, 0, 0, 1, 1]}) - df_appended = df.append({"variable": "new", "target": 1}, - ignore_index=True) + df_appended = pd.concat([df, pd.DataFrame({"variable": "new", "target": 1}, index=[len(df)])], ignore_index=True) # inputs of TargetEncoder will be of dtype category df["variable"] = df["variable"].astype("category") @@ -285,8 +284,7 @@ def test_target_encoder_transform_new_category_linear_regression(self): 'neutral', 'positive'], 'target': [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4]}) - df_appended = df.append({"variable": "new", "target": 10}, - ignore_index=True) + df_appended = pd.concat([df, pd.DataFrame({"variable": "new", "target": 10}, index=[len(df)])], ignore_index=True) # inputs of TargetEncoder will be of dtype category df["variable"] = df["variable"].astype("category")