From 3ac84d2119ed4216221f09556f8150381d475231 Mon Sep 17 00:00:00 2001 From: patrickleonardy Date: Mon, 24 Apr 2023 13:50:35 +0200 Subject: [PATCH 1/7] Add median imputation and adapt tests --- cobra/preprocessing/preprocessor.py | 6 +-- cobra/preprocessing/target_encoder.py | 5 +- tests/preprocessing/test_target_encoder.py | 58 ++++++++++++++++++++-- 3 files changed, 62 insertions(+), 7 deletions(-) diff --git a/cobra/preprocessing/preprocessor.py b/cobra/preprocessing/preprocessor.py index fa7ddf1..0b9e171 100644 --- a/cobra/preprocessing/preprocessor.py +++ b/cobra/preprocessing/preprocessor.py @@ -140,11 +140,11 @@ def from_params( parameter, the bigger the contribution of the overall mean. When set to zero, there is no smoothing (e.g. the pure target incidence is used). imputation_strategy : str, optional + Valid imputation strategies = mean, median, min or max In case there is a particular column which contains new categories, the encoding will lead to NULL values which should be imputed. - Valid strategies are to replace with the global mean of the train - set or the min (resp. max) incidence of the categories of that - particular variable. + The imputation replaces the so found null values with the encoded value + according to the metric taken over all the encoded values for this variable. Returns ------- diff --git a/cobra/preprocessing/target_encoder.py b/cobra/preprocessing/target_encoder.py index 3eda39d..9caedf8 100644 --- a/cobra/preprocessing/target_encoder.py +++ b/cobra/preprocessing/target_encoder.py @@ -60,7 +60,7 @@ class TargetEncoder(BaseEstimator): current categorical value is used). """ - valid_imputation_strategies = ("mean", "min", "max") + valid_imputation_strategies = ("mean", "min", "max", "median") def __init__(self, weight: float=0.0, imputation_strategy: str="mean"): @@ -282,6 +282,9 @@ def _transform_column(self, data: pd.DataFrame, elif self.imputation_strategy == "max": data[new_column].fillna(data[new_column].max(), inplace=True) + elif self.imputation_strategy == "median": + data[new_column].fillna(data[new_column].median(), + inplace=True) return data diff --git a/tests/preprocessing/test_target_encoder.py b/tests/preprocessing/test_target_encoder.py index 51ebd79..3301121 100644 --- a/tests/preprocessing/test_target_encoder.py +++ b/tests/preprocessing/test_target_encoder.py @@ -13,7 +13,7 @@ def test_target_encoder_constructor_weight_value_error(self): def test_target_encoder_constructor_imputation_value_error(self): with pytest.raises(ValueError): - TargetEncoder(imputation_strategy="median") + TargetEncoder(imputation_strategy="something") # Tests for attributes_attributes_to_dict and set_attributes_from_dict def test_target_encoder_attributes_to_dict(self): @@ -52,12 +52,11 @@ def test_target_encoder_set_attributes_from_dict_unfitted(self, attribute): if attribute == "weight": actual = encoder.weight expected = 1.0 - assert expected == actual + elif attribute == "mapping": actual = encoder._mapping expected = {} - assert expected == actual def test_target_encoder_set_attributes_from_dict(self): @@ -304,6 +303,59 @@ def test_target_encoder_transform_new_category_linear_regression(self): pd.testing.assert_frame_equal(actual, expected) + + def test_target_encoder_transform_new_category_linear_regression_median(self): + df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', + 'neutral', 'negative', 'positive', + 'negative', 'neutral', 'neutral', + 'neutral', 'positive'], + 'target': [5, 4, -5, 0, -4, 5, -5, 0, 1, 0, 4]}) + + df_appended = df.append({"variable": "new", "target": 10}, + ignore_index=True) + + # inputs of TargetEncoder will be of dtype category + df["variable"] = df["variable"].astype("category") + df_appended["variable"] = df_appended["variable"].astype("category") + + expected = df_appended.copy() + expected["variable_enc"] = [4.500000, 4.500000, -4.666667, 0.250000, + -4.666667, 4.500000, -4.666667, 0.250000, + 0.250000, 0.250000, 4.500000, + 0.250000] # median imputation for new value + + encoder = TargetEncoder(imputation_strategy="median") + encoder.fit(data=df, column_names=["variable"], target_column="target") + actual = encoder.transform(data=df_appended, column_names=["variable"]) + + pd.testing.assert_frame_equal(actual, expected) + + def test_target_encoder_transform_new_category_binary_classification_median(self): + df = pd.DataFrame({'variable': ['positive', 'positive', 'negative', + 'neutral', 'negative', 'positive', + 'negative', 'neutral', 'neutral', + 'neutral'], + 'target': [1, 1, 0, 0, 1, 0, 0, 0, 1, 1]}) + + df_appended = df.append({"variable": "new", "target": 1}, + ignore_index=True) + + # inputs of TargetEncoder will be of dtype category + df["variable"] = df["variable"].astype("category") + df_appended["variable"] = df_appended["variable"].astype("category") + + expected = df_appended.copy() + expected["variable_enc"] = [0.666667, 0.666667, 0.333333, 0.50000, + 0.333333, 0.666667, 0.333333, 0.50000, + 0.50000, 0.50000, 0.50000] + + encoder = TargetEncoder(imputation_strategy="median") + encoder.fit(data=df, column_names=["variable"], target_column="target") + actual = encoder.transform(data=df_appended, column_names=["variable"]) + + pd.testing.assert_frame_equal(actual, expected) + + # Tests for _clean_column_name: def test_target_encoder_clean_column_name_binned_column(self): column_name = "test_column_bin" From 922083f487e683ddbaab4bea99f4d6f4868823f7 Mon Sep 17 00:00:00 2001 From: patrickleonardy Date: Mon, 24 Apr 2023 13:55:49 +0200 Subject: [PATCH 2/7] change TargetEncoder documentation --- cobra/preprocessing/target_encoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cobra/preprocessing/target_encoder.py b/cobra/preprocessing/target_encoder.py index 9caedf8..b6f9322 100644 --- a/cobra/preprocessing/target_encoder.py +++ b/cobra/preprocessing/target_encoder.py @@ -48,7 +48,7 @@ class TargetEncoder(BaseEstimator): In case there is a particular column which contains new categories, the encoding will lead to NULL values which should be imputed. Valid strategies then are to replace the NULL values with the global - mean of the train set or the min (resp. max) incidence of the + mean or median of the train set or the min (resp. max) incidence of the categories of that particular variable. weight : float Smoothing parameter (non-negative). The higher the value of the From 80f4de2c9b5fb0fc9ca06a6b9124b2fa8a424775 Mon Sep 17 00:00:00 2001 From: patrickleonardy Date: Mon, 24 Apr 2023 15:17:39 +0200 Subject: [PATCH 3/7] target_encoder does not change initial data --- cobra/preprocessing/target_encoder.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/cobra/preprocessing/target_encoder.py b/cobra/preprocessing/target_encoder.py index b6f9322..09debbe 100644 --- a/cobra/preprocessing/target_encoder.py +++ b/cobra/preprocessing/target_encoder.py @@ -18,7 +18,7 @@ class TargetEncoder(BaseEstimator): Note that, when applying this target encoding, values of the categorical feature that have not been seen during fit will be imputed according to the - configured imputation strategy (replacement with the mean, minimum or + configured imputation strategy (replacement with the mean, median, minimum or maximum value of the categorical variable). The main problem with Target encoding is overfitting; the fact that we are @@ -223,13 +223,15 @@ def transform(self, data: pd.DataFrame, Exception when TargetEncoder was not fitted before calling this method. """ + _data = data.copy() + if (len(self._mapping) == 0) or (self._global_mean is None): msg = ("This {} instance is not fitted yet. Call 'fit' with " "appropriate arguments before using this method.") raise NotFittedError(msg.format(self.__class__.__name__)) for column in tqdm(column_names, desc="Applying target encoding..."): - if column not in data.columns: + if column not in _data.columns: log.warning("Unknown column '{}' will be skipped." .format(column)) continue @@ -237,9 +239,9 @@ def transform(self, data: pd.DataFrame, log.warning("Column '{}' is not in fitted output " "and will be skipped.".format(column)) continue - data = self._transform_column(data, column) + _data = self._transform_column(_data, column) - return data + return _data def _transform_column(self, data: pd.DataFrame, column_name: str) -> pd.DataFrame: @@ -260,33 +262,34 @@ def _transform_column(self, data: pd.DataFrame, pd.DataFrame Resulting transformed data. """ + _data = data.copy() new_column = TargetEncoder._clean_column_name(column_name) # Convert dtype to float, because when the original dtype # is of type "category", the resulting dtype would otherwise also be of # type "category": - data[new_column] = (data[column_name].map(self._mapping[column_name]) + _data[new_column] = (_data[column_name].map(self._mapping[column_name]) .astype("float")) # In case of categorical data, it could be that new categories will # emerge which were not present in the train set, so this will result # in missing values, which should be replaced according to the # configured imputation strategy: - if data[new_column].isnull().sum() > 0: + if _data[new_column].isnull().sum() > 0: if self.imputation_strategy == "mean": - data[new_column].fillna(self._global_mean, + _data[new_column].fillna(self._global_mean, inplace=True) elif self.imputation_strategy == "min": - data[new_column].fillna(data[new_column].min(), + _data[new_column].fillna(_data[new_column].min(), inplace=True) elif self.imputation_strategy == "max": - data[new_column].fillna(data[new_column].max(), + _data[new_column].fillna(_data[new_column].max(), inplace=True) elif self.imputation_strategy == "median": - data[new_column].fillna(data[new_column].median(), + _data[new_column].fillna(_data[new_column].median(), inplace=True) - return data + return _data def fit_transform(self, data: pd.DataFrame, column_names: list, From da8bd6b918bf9374339669bb02d01d722f927cdf Mon Sep 17 00:00:00 2001 From: patrickleonardy Date: Mon, 24 Apr 2023 16:06:34 +0200 Subject: [PATCH 4/7] add some explanation notebook for target encoder --- tutorials/explanation_Target_encoder.ipynb | 867 +++++++++++++++++++++ 1 file changed, 867 insertions(+) create mode 100644 tutorials/explanation_Target_encoder.ipynb diff --git a/tutorials/explanation_Target_encoder.ipynb b/tutorials/explanation_Target_encoder.ipynb new file mode 100644 index 0000000..44d3300 --- /dev/null +++ b/tutorials/explanation_Target_encoder.ipynb @@ -0,0 +1,867 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "sys.path.insert(0, \"/home/patrick/Git/cobra/cobra/\")\n", + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + " Hi, welcome to Cobra!\n", + " You can find some tutorials that explain the functioning of cobra on the PythonPredictions GitHub:\n", + " https://github.com/PythonPredictions/cobra/tree/master/tutorials\n", + " \n", + "/home/patrick/anaconda3/envs/cobra_dev_env/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "from cobra.preprocessing import TargetEncoder " + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Defining the data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
numerical_varcategorical_varbinary_targetnumerical_targetnumerical_var_binned
018.727006lowFalse3.854165(11.0, 20.0]
147.535715lowTrue0.159663(39.0, 48.0]
236.599697middleFalse2.308938(30.0, 39.0]
329.932924middleTrue2.410255(30.0, 39.0]
47.800932lowFalse6.832635(1.0, 11.0]
\n", + "
" + ], + "text/plain": [ + " numerical_var categorical_var binary_target numerical_target \\\n", + "0 18.727006 low False 3.854165 \n", + "1 47.535715 low True 0.159663 \n", + "2 36.599697 middle False 2.308938 \n", + "3 29.932924 middle True 2.410255 \n", + "4 7.800932 low False 6.832635 \n", + "\n", + " numerical_var_binned \n", + "0 (11.0, 20.0] \n", + "1 (39.0, 48.0] \n", + "2 (30.0, 39.0] \n", + "3 (30.0, 39.0] \n", + "4 (1.0, 11.0] " + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "np.random.seed(42)\n", + "data = pd.DataFrame({\n", + " \"numerical_var\" : np.random.uniform(0,50, 15).tolist()+ [np.nan for _ in range(5)],\n", + " \"categorical_var\": np.random.choice([\"low\", \"middle\", \"high\"],15).tolist() + [np.nan for _ in range(5)],\n", + " \"binary_target\" : np.random.choice([True, False],20),\n", + " \"numerical_target\" : np.random.uniform(0,10, 20),\n", + "})\n", + "# binning of the numerical variable\n", + "data[\"numerical_var_binned\"] = pd.cut(data.numerical_var, bins=5, precision=0)\n", + "\n", + "data.head()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this data we have several missing values for the columns `numerical_var` and `categorical_var`. the Target encoder can due to this not assign an incidence value (for binary target) or a mean value (for numeric target) to those observations. The imputation strategy then defines how those encoded values should be estimated (possibilities: `\"min\", \"max\", \"mean\", \"median\"`). \n", + "Those missing encoded values are then replaced by the `\"min\", \"max\", \"mean\", \"median\"` of the encoded variable. " + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The target encoder's additive smoothing weight is set to 0. This disables smoothing and may make the encoding prone to overfitting. Increase the weight if needed.\n", + "Fitting target encoding...: 100%|██████████| 2/2 [00:00<00:00, 251.07it/s]\n", + "Applying target encoding...: 100%|██████████| 2/2 [00:00<00:00, 169.43it/s]\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
numerical_varcategorical_varbinary_targetnumerical_targetnumerical_var_binnednumerical_varned_enccategorical_var_enc
018.727006lowFalse3.854165(11.0, 20.0]0.500.4
147.535715lowTrue0.159663(39.0, 48.0]1.000.4
236.599697middleFalse2.308938(30.0, 39.0]0.750.8
329.932924middleTrue2.410255(30.0, 39.0]0.750.8
47.800932lowFalse6.832635(1.0, 11.0]0.400.4
57.799726lowFalse6.099967(1.0, 11.0]0.400.4
62.904181lowTrue8.331949(1.0, 11.0]0.400.4
743.308807highTrue1.733647(39.0, 48.0]1.000.8
830.055751highTrue3.910606(30.0, 39.0]0.750.8
935.403629highTrue1.822361(30.0, 39.0]0.750.8
101.029225middleTrue7.553614(1.0, 11.0]0.400.8
1148.495493highTrue4.251559(39.0, 48.0]1.000.8
1241.622132middleTrue2.079417(39.0, 48.0]1.000.8
1310.616956middleTrue5.677003(11.0, 20.0]0.500.8
149.091248highFalse0.313133(1.0, 11.0]0.400.8
15NaNNaNFalse8.422848NaN0.750.8
16NaNNaNTrue4.497541NaN0.750.8
17NaNNaNFalse3.951502NaN0.750.8
18NaNNaNFalse9.266589NaN0.750.8
19NaNNaNFalse7.272720NaN0.750.8
\n", + "
" + ], + "text/plain": [ + " numerical_var categorical_var binary_target numerical_target \\\n", + "0 18.727006 low False 3.854165 \n", + "1 47.535715 low True 0.159663 \n", + "2 36.599697 middle False 2.308938 \n", + "3 29.932924 middle True 2.410255 \n", + "4 7.800932 low False 6.832635 \n", + "5 7.799726 low False 6.099967 \n", + "6 2.904181 low True 8.331949 \n", + "7 43.308807 high True 1.733647 \n", + "8 30.055751 high True 3.910606 \n", + "9 35.403629 high True 1.822361 \n", + "10 1.029225 middle True 7.553614 \n", + "11 48.495493 high True 4.251559 \n", + "12 41.622132 middle True 2.079417 \n", + "13 10.616956 middle True 5.677003 \n", + "14 9.091248 high False 0.313133 \n", + "15 NaN NaN False 8.422848 \n", + "16 NaN NaN True 4.497541 \n", + "17 NaN NaN False 3.951502 \n", + "18 NaN NaN False 9.266589 \n", + "19 NaN NaN False 7.272720 \n", + "\n", + " numerical_var_binned numerical_varned_enc categorical_var_enc \n", + "0 (11.0, 20.0] 0.50 0.4 \n", + "1 (39.0, 48.0] 1.00 0.4 \n", + "2 (30.0, 39.0] 0.75 0.8 \n", + "3 (30.0, 39.0] 0.75 0.8 \n", + "4 (1.0, 11.0] 0.40 0.4 \n", + "5 (1.0, 11.0] 0.40 0.4 \n", + "6 (1.0, 11.0] 0.40 0.4 \n", + "7 (39.0, 48.0] 1.00 0.8 \n", + "8 (30.0, 39.0] 0.75 0.8 \n", + "9 (30.0, 39.0] 0.75 0.8 \n", + "10 (1.0, 11.0] 0.40 0.8 \n", + "11 (39.0, 48.0] 1.00 0.8 \n", + "12 (39.0, 48.0] 1.00 0.8 \n", + "13 (11.0, 20.0] 0.50 0.8 \n", + "14 (1.0, 11.0] 0.40 0.8 \n", + "15 NaN 0.75 0.8 \n", + "16 NaN 0.75 0.8 \n", + "17 NaN 0.75 0.8 \n", + "18 NaN 0.75 0.8 \n", + "19 NaN 0.75 0.8 " + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "t_encoder = TargetEncoder(imputation_strategy=\"median\")\n", + "encoded_data = t_encoder.fit_transform(data, column_names=[\"numerical_var_binned\", \"categorical_var\"], target_column=\"binary_target\" )\n", + "encoded_data" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "numerical_varned_enc 0.75\n", + "categorical_var_enc 0.80\n", + "dtype: float64" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "encoded_data.dropna()[[\"numerical_varned_enc\",\t\"categorical_var_enc\"]].median()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here above the `numerical_varned_enc` and `categorical_var_enc` contain the median value of the observations where `numerical_var`, `categorical_var` respectively are not missing.\n", + "\n", + "**So only the values where we have an encoded value are taken into account**" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The target encoder's additive smoothing weight is set to 0. This disables smoothing and may make the encoding prone to overfitting. Increase the weight if needed.\n", + "Fitting target encoding...: 100%|██████████| 2/2 [00:12<00:00, 6.05s/it]\n", + "Applying target encoding...: 100%|██████████| 2/2 [00:00<00:00, 119.55it/s]\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
numerical_varcategorical_varbinary_targetnumerical_targetnumerical_var_binnednumerical_varned_enccategorical_var_enc
018.727006lowFalse3.854165(11.0, 20.0]0.500.40
147.535715lowTrue0.159663(39.0, 48.0]1.000.40
236.599697middleFalse2.308938(30.0, 39.0]0.750.80
329.932924middleTrue2.410255(30.0, 39.0]0.750.80
47.800932lowFalse6.832635(1.0, 11.0]0.400.40
57.799726lowFalse6.099967(1.0, 11.0]0.400.40
62.904181lowTrue8.331949(1.0, 11.0]0.400.40
743.308807highTrue1.733647(39.0, 48.0]1.000.80
830.055751highTrue3.910606(30.0, 39.0]0.750.80
935.403629highTrue1.822361(30.0, 39.0]0.750.80
101.029225middleTrue7.553614(1.0, 11.0]0.400.80
1148.495493highTrue4.251559(39.0, 48.0]1.000.80
1241.622132middleTrue2.079417(39.0, 48.0]1.000.80
1310.616956middleTrue5.677003(11.0, 20.0]0.500.80
149.091248highFalse0.313133(1.0, 11.0]0.400.80
15NaNNaNFalse8.422848NaN0.550.55
16NaNNaNTrue4.497541NaN0.550.55
17NaNNaNFalse3.951502NaN0.550.55
18NaNNaNFalse9.266589NaN0.550.55
19NaNNaNFalse7.272720NaN0.550.55
\n", + "
" + ], + "text/plain": [ + " numerical_var categorical_var binary_target numerical_target \\\n", + "0 18.727006 low False 3.854165 \n", + "1 47.535715 low True 0.159663 \n", + "2 36.599697 middle False 2.308938 \n", + "3 29.932924 middle True 2.410255 \n", + "4 7.800932 low False 6.832635 \n", + "5 7.799726 low False 6.099967 \n", + "6 2.904181 low True 8.331949 \n", + "7 43.308807 high True 1.733647 \n", + "8 30.055751 high True 3.910606 \n", + "9 35.403629 high True 1.822361 \n", + "10 1.029225 middle True 7.553614 \n", + "11 48.495493 high True 4.251559 \n", + "12 41.622132 middle True 2.079417 \n", + "13 10.616956 middle True 5.677003 \n", + "14 9.091248 high False 0.313133 \n", + "15 NaN NaN False 8.422848 \n", + "16 NaN NaN True 4.497541 \n", + "17 NaN NaN False 3.951502 \n", + "18 NaN NaN False 9.266589 \n", + "19 NaN NaN False 7.272720 \n", + "\n", + " numerical_var_binned numerical_varned_enc categorical_var_enc \n", + "0 (11.0, 20.0] 0.50 0.40 \n", + "1 (39.0, 48.0] 1.00 0.40 \n", + "2 (30.0, 39.0] 0.75 0.80 \n", + "3 (30.0, 39.0] 0.75 0.80 \n", + "4 (1.0, 11.0] 0.40 0.40 \n", + "5 (1.0, 11.0] 0.40 0.40 \n", + "6 (1.0, 11.0] 0.40 0.40 \n", + "7 (39.0, 48.0] 1.00 0.80 \n", + "8 (30.0, 39.0] 0.75 0.80 \n", + "9 (30.0, 39.0] 0.75 0.80 \n", + "10 (1.0, 11.0] 0.40 0.80 \n", + "11 (39.0, 48.0] 1.00 0.80 \n", + "12 (39.0, 48.0] 1.00 0.80 \n", + "13 (11.0, 20.0] 0.50 0.80 \n", + "14 (1.0, 11.0] 0.40 0.80 \n", + "15 NaN 0.55 0.55 \n", + "16 NaN 0.55 0.55 \n", + "17 NaN 0.55 0.55 \n", + "18 NaN 0.55 0.55 \n", + "19 NaN 0.55 0.55 " + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "t_encoder = TargetEncoder(imputation_strategy=\"mean\")\n", + "encoded_data2 = t_encoder.fit_transform(data, column_names=[\"numerical_var_binned\", \"categorical_var\"], target_column=\"binary_target\" )\n", + "encoded_data2" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.55" + ] + }, + "execution_count": 49, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "encoded_data2.binary_target.mean()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**In the mean implementation the overall global mean over the target varaible is taken**\n", + "\n", + "Is this correct or should we here also take the mean over only the know ones ? " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "cobra_dev_env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.0" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} From e750ffdf19643139029a4feccd111fe9923d0740 Mon Sep 17 00:00:00 2001 From: patrickleonardy Date: Fri, 12 May 2023 14:43:38 +0200 Subject: [PATCH 5/7] add a more concrete explanation to the doctsting --- cobra/preprocessing/target_encoder.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cobra/preprocessing/target_encoder.py b/cobra/preprocessing/target_encoder.py index 09debbe..9342a5b 100644 --- a/cobra/preprocessing/target_encoder.py +++ b/cobra/preprocessing/target_encoder.py @@ -48,8 +48,13 @@ class TargetEncoder(BaseEstimator): In case there is a particular column which contains new categories, the encoding will lead to NULL values which should be imputed. Valid strategies then are to replace the NULL values with the global - mean or median of the train set or the min (resp. max) incidence of the - categories of that particular variable. + mean, median or the min (resp. max) incidence of the variable. + + Ex: By taking the mean strategy the mean of the known encoded variables + is computed and the missing encoded values would be imputed with this value. + + + weight : float Smoothing parameter (non-negative). The higher the value of the parameter, the bigger the contribution of the overall mean of targets From 65981db0660b593cf0bb3f593548566519137594 Mon Sep 17 00:00:00 2001 From: patrickleonardy Date: Fri, 12 May 2023 14:46:08 +0200 Subject: [PATCH 6/7] delete tutorial as not really necessary --- tutorials/explanation_Target_encoder.ipynb | 867 --------------------- 1 file changed, 867 deletions(-) delete mode 100644 tutorials/explanation_Target_encoder.ipynb diff --git a/tutorials/explanation_Target_encoder.ipynb b/tutorials/explanation_Target_encoder.ipynb deleted file mode 100644 index 44d3300..0000000 --- a/tutorials/explanation_Target_encoder.ipynb +++ /dev/null @@ -1,867 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "sys.path.insert(0, \"/home/patrick/Git/cobra/cobra/\")\n", - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n", - " Hi, welcome to Cobra!\n", - " You can find some tutorials that explain the functioning of cobra on the PythonPredictions GitHub:\n", - " https://github.com/PythonPredictions/cobra/tree/master/tutorials\n", - " \n", - "/home/patrick/anaconda3/envs/cobra_dev_env/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "import pandas as pd\n", - "import numpy as np\n", - "from cobra.preprocessing import TargetEncoder " - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Defining the data" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
numerical_varcategorical_varbinary_targetnumerical_targetnumerical_var_binned
018.727006lowFalse3.854165(11.0, 20.0]
147.535715lowTrue0.159663(39.0, 48.0]
236.599697middleFalse2.308938(30.0, 39.0]
329.932924middleTrue2.410255(30.0, 39.0]
47.800932lowFalse6.832635(1.0, 11.0]
\n", - "
" - ], - "text/plain": [ - " numerical_var categorical_var binary_target numerical_target \\\n", - "0 18.727006 low False 3.854165 \n", - "1 47.535715 low True 0.159663 \n", - "2 36.599697 middle False 2.308938 \n", - "3 29.932924 middle True 2.410255 \n", - "4 7.800932 low False 6.832635 \n", - "\n", - " numerical_var_binned \n", - "0 (11.0, 20.0] \n", - "1 (39.0, 48.0] \n", - "2 (30.0, 39.0] \n", - "3 (30.0, 39.0] \n", - "4 (1.0, 11.0] " - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.random.seed(42)\n", - "data = pd.DataFrame({\n", - " \"numerical_var\" : np.random.uniform(0,50, 15).tolist()+ [np.nan for _ in range(5)],\n", - " \"categorical_var\": np.random.choice([\"low\", \"middle\", \"high\"],15).tolist() + [np.nan for _ in range(5)],\n", - " \"binary_target\" : np.random.choice([True, False],20),\n", - " \"numerical_target\" : np.random.uniform(0,10, 20),\n", - "})\n", - "# binning of the numerical variable\n", - "data[\"numerical_var_binned\"] = pd.cut(data.numerical_var, bins=5, precision=0)\n", - "\n", - "data.head()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this data we have several missing values for the columns `numerical_var` and `categorical_var`. the Target encoder can due to this not assign an incidence value (for binary target) or a mean value (for numeric target) to those observations. The imputation strategy then defines how those encoded values should be estimated (possibilities: `\"min\", \"max\", \"mean\", \"median\"`). \n", - "Those missing encoded values are then replaced by the `\"min\", \"max\", \"mean\", \"median\"` of the encoded variable. " - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "The target encoder's additive smoothing weight is set to 0. This disables smoothing and may make the encoding prone to overfitting. Increase the weight if needed.\n", - "Fitting target encoding...: 100%|██████████| 2/2 [00:00<00:00, 251.07it/s]\n", - "Applying target encoding...: 100%|██████████| 2/2 [00:00<00:00, 169.43it/s]\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
numerical_varcategorical_varbinary_targetnumerical_targetnumerical_var_binnednumerical_varned_enccategorical_var_enc
018.727006lowFalse3.854165(11.0, 20.0]0.500.4
147.535715lowTrue0.159663(39.0, 48.0]1.000.4
236.599697middleFalse2.308938(30.0, 39.0]0.750.8
329.932924middleTrue2.410255(30.0, 39.0]0.750.8
47.800932lowFalse6.832635(1.0, 11.0]0.400.4
57.799726lowFalse6.099967(1.0, 11.0]0.400.4
62.904181lowTrue8.331949(1.0, 11.0]0.400.4
743.308807highTrue1.733647(39.0, 48.0]1.000.8
830.055751highTrue3.910606(30.0, 39.0]0.750.8
935.403629highTrue1.822361(30.0, 39.0]0.750.8
101.029225middleTrue7.553614(1.0, 11.0]0.400.8
1148.495493highTrue4.251559(39.0, 48.0]1.000.8
1241.622132middleTrue2.079417(39.0, 48.0]1.000.8
1310.616956middleTrue5.677003(11.0, 20.0]0.500.8
149.091248highFalse0.313133(1.0, 11.0]0.400.8
15NaNNaNFalse8.422848NaN0.750.8
16NaNNaNTrue4.497541NaN0.750.8
17NaNNaNFalse3.951502NaN0.750.8
18NaNNaNFalse9.266589NaN0.750.8
19NaNNaNFalse7.272720NaN0.750.8
\n", - "
" - ], - "text/plain": [ - " numerical_var categorical_var binary_target numerical_target \\\n", - "0 18.727006 low False 3.854165 \n", - "1 47.535715 low True 0.159663 \n", - "2 36.599697 middle False 2.308938 \n", - "3 29.932924 middle True 2.410255 \n", - "4 7.800932 low False 6.832635 \n", - "5 7.799726 low False 6.099967 \n", - "6 2.904181 low True 8.331949 \n", - "7 43.308807 high True 1.733647 \n", - "8 30.055751 high True 3.910606 \n", - "9 35.403629 high True 1.822361 \n", - "10 1.029225 middle True 7.553614 \n", - "11 48.495493 high True 4.251559 \n", - "12 41.622132 middle True 2.079417 \n", - "13 10.616956 middle True 5.677003 \n", - "14 9.091248 high False 0.313133 \n", - "15 NaN NaN False 8.422848 \n", - "16 NaN NaN True 4.497541 \n", - "17 NaN NaN False 3.951502 \n", - "18 NaN NaN False 9.266589 \n", - "19 NaN NaN False 7.272720 \n", - "\n", - " numerical_var_binned numerical_varned_enc categorical_var_enc \n", - "0 (11.0, 20.0] 0.50 0.4 \n", - "1 (39.0, 48.0] 1.00 0.4 \n", - "2 (30.0, 39.0] 0.75 0.8 \n", - "3 (30.0, 39.0] 0.75 0.8 \n", - "4 (1.0, 11.0] 0.40 0.4 \n", - "5 (1.0, 11.0] 0.40 0.4 \n", - "6 (1.0, 11.0] 0.40 0.4 \n", - "7 (39.0, 48.0] 1.00 0.8 \n", - "8 (30.0, 39.0] 0.75 0.8 \n", - "9 (30.0, 39.0] 0.75 0.8 \n", - "10 (1.0, 11.0] 0.40 0.8 \n", - "11 (39.0, 48.0] 1.00 0.8 \n", - "12 (39.0, 48.0] 1.00 0.8 \n", - "13 (11.0, 20.0] 0.50 0.8 \n", - "14 (1.0, 11.0] 0.40 0.8 \n", - "15 NaN 0.75 0.8 \n", - "16 NaN 0.75 0.8 \n", - "17 NaN 0.75 0.8 \n", - "18 NaN 0.75 0.8 \n", - "19 NaN 0.75 0.8 " - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "t_encoder = TargetEncoder(imputation_strategy=\"median\")\n", - "encoded_data = t_encoder.fit_transform(data, column_names=[\"numerical_var_binned\", \"categorical_var\"], target_column=\"binary_target\" )\n", - "encoded_data" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "numerical_varned_enc 0.75\n", - "categorical_var_enc 0.80\n", - "dtype: float64" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "encoded_data.dropna()[[\"numerical_varned_enc\",\t\"categorical_var_enc\"]].median()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here above the `numerical_varned_enc` and `categorical_var_enc` contain the median value of the observations where `numerical_var`, `categorical_var` respectively are not missing.\n", - "\n", - "**So only the values where we have an encoded value are taken into account**" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "The target encoder's additive smoothing weight is set to 0. This disables smoothing and may make the encoding prone to overfitting. Increase the weight if needed.\n", - "Fitting target encoding...: 100%|██████████| 2/2 [00:12<00:00, 6.05s/it]\n", - "Applying target encoding...: 100%|██████████| 2/2 [00:00<00:00, 119.55it/s]\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
numerical_varcategorical_varbinary_targetnumerical_targetnumerical_var_binnednumerical_varned_enccategorical_var_enc
018.727006lowFalse3.854165(11.0, 20.0]0.500.40
147.535715lowTrue0.159663(39.0, 48.0]1.000.40
236.599697middleFalse2.308938(30.0, 39.0]0.750.80
329.932924middleTrue2.410255(30.0, 39.0]0.750.80
47.800932lowFalse6.832635(1.0, 11.0]0.400.40
57.799726lowFalse6.099967(1.0, 11.0]0.400.40
62.904181lowTrue8.331949(1.0, 11.0]0.400.40
743.308807highTrue1.733647(39.0, 48.0]1.000.80
830.055751highTrue3.910606(30.0, 39.0]0.750.80
935.403629highTrue1.822361(30.0, 39.0]0.750.80
101.029225middleTrue7.553614(1.0, 11.0]0.400.80
1148.495493highTrue4.251559(39.0, 48.0]1.000.80
1241.622132middleTrue2.079417(39.0, 48.0]1.000.80
1310.616956middleTrue5.677003(11.0, 20.0]0.500.80
149.091248highFalse0.313133(1.0, 11.0]0.400.80
15NaNNaNFalse8.422848NaN0.550.55
16NaNNaNTrue4.497541NaN0.550.55
17NaNNaNFalse3.951502NaN0.550.55
18NaNNaNFalse9.266589NaN0.550.55
19NaNNaNFalse7.272720NaN0.550.55
\n", - "
" - ], - "text/plain": [ - " numerical_var categorical_var binary_target numerical_target \\\n", - "0 18.727006 low False 3.854165 \n", - "1 47.535715 low True 0.159663 \n", - "2 36.599697 middle False 2.308938 \n", - "3 29.932924 middle True 2.410255 \n", - "4 7.800932 low False 6.832635 \n", - "5 7.799726 low False 6.099967 \n", - "6 2.904181 low True 8.331949 \n", - "7 43.308807 high True 1.733647 \n", - "8 30.055751 high True 3.910606 \n", - "9 35.403629 high True 1.822361 \n", - "10 1.029225 middle True 7.553614 \n", - "11 48.495493 high True 4.251559 \n", - "12 41.622132 middle True 2.079417 \n", - "13 10.616956 middle True 5.677003 \n", - "14 9.091248 high False 0.313133 \n", - "15 NaN NaN False 8.422848 \n", - "16 NaN NaN True 4.497541 \n", - "17 NaN NaN False 3.951502 \n", - "18 NaN NaN False 9.266589 \n", - "19 NaN NaN False 7.272720 \n", - "\n", - " numerical_var_binned numerical_varned_enc categorical_var_enc \n", - "0 (11.0, 20.0] 0.50 0.40 \n", - "1 (39.0, 48.0] 1.00 0.40 \n", - "2 (30.0, 39.0] 0.75 0.80 \n", - "3 (30.0, 39.0] 0.75 0.80 \n", - "4 (1.0, 11.0] 0.40 0.40 \n", - "5 (1.0, 11.0] 0.40 0.40 \n", - "6 (1.0, 11.0] 0.40 0.40 \n", - "7 (39.0, 48.0] 1.00 0.80 \n", - "8 (30.0, 39.0] 0.75 0.80 \n", - "9 (30.0, 39.0] 0.75 0.80 \n", - "10 (1.0, 11.0] 0.40 0.80 \n", - "11 (39.0, 48.0] 1.00 0.80 \n", - "12 (39.0, 48.0] 1.00 0.80 \n", - "13 (11.0, 20.0] 0.50 0.80 \n", - "14 (1.0, 11.0] 0.40 0.80 \n", - "15 NaN 0.55 0.55 \n", - "16 NaN 0.55 0.55 \n", - "17 NaN 0.55 0.55 \n", - "18 NaN 0.55 0.55 \n", - "19 NaN 0.55 0.55 " - ] - }, - "execution_count": 33, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "t_encoder = TargetEncoder(imputation_strategy=\"mean\")\n", - "encoded_data2 = t_encoder.fit_transform(data, column_names=[\"numerical_var_binned\", \"categorical_var\"], target_column=\"binary_target\" )\n", - "encoded_data2" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.55" - ] - }, - "execution_count": 49, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "encoded_data2.binary_target.mean()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**In the mean implementation the overall global mean over the target varaible is taken**\n", - "\n", - "Is this correct or should we here also take the mean over only the know ones ? " - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "cobra_dev_env", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.0" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} From d1119b309111300bd4cd0694e7a6e53f18ee2451 Mon Sep 17 00:00:00 2001 From: patrickleonardy Date: Fri, 12 May 2023 14:59:53 +0200 Subject: [PATCH 7/7] explain the detail of the imputation only once --- cobra/preprocessing/preprocessor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cobra/preprocessing/preprocessor.py b/cobra/preprocessing/preprocessor.py index 0b9e171..dba6ff4 100644 --- a/cobra/preprocessing/preprocessor.py +++ b/cobra/preprocessing/preprocessor.py @@ -143,8 +143,8 @@ def from_params( Valid imputation strategies = mean, median, min or max In case there is a particular column which contains new categories, the encoding will lead to NULL values which should be imputed. - The imputation replaces the so found null values with the encoded value - according to the metric taken over all the encoded values for this variable. + For more information about how the imputation works go see the documentation of the + :class:`cobra.preprocessing.TargetEncoder` Returns -------