From 63ca2a07234a6a549010ec50fc174787cfea66c1 Mon Sep 17 00:00:00 2001 From: Anshuma Jain Date: Thu, 1 Oct 2020 15:43:39 -0700 Subject: [PATCH 1/4] fixed issue#9 --- metriks/ranking.py | 48 ++++++++++++++++++----------- tests/test_confusion_matrix_at_k.py | 2 ++ tests/test_mean_reciprocal_rank.py | 2 +- 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/metriks/ranking.py b/metriks/ranking.py index b61ca66..cea43d2 100644 --- a/metriks/ranking.py +++ b/metriks/ranking.py @@ -5,51 +5,57 @@ """ import numpy as np +from typing import Tuple +from numpy.ma import MaskedArray +from pytypes import typechecked -def check_arrays(y_true, y_prob): +@typechecked +def check_arrays(y_true: np.ndarray, y_prob: np.ndarray) -> None: # Make sure that inputs this conforms to our expectations assert isinstance(y_true, np.ndarray), AssertionError( 'Expect y_true to be a {expected}. Got {actual}' - .format(expected=np.ndarray, actual=type(y_true)) + .format(expected=np.ndarray, actual=type(y_true)) ) assert isinstance(y_prob, np.ndarray), AssertionError( 'Expect y_prob to be a {expected}. Got {actual}' - .format(expected=np.ndarray, actual=type(y_prob)) + .format(expected=np.ndarray, actual=type(y_prob)) ) assert y_true.shape == y_prob.shape, AssertionError( 'Shapes must match. Got y_true={true_shape}, y_prob={prob_shape}' - .format(true_shape=y_true.shape, prob_shape=y_prob.shape) + .format(true_shape=y_true.shape, prob_shape=y_prob.shape) ) assert len(y_true.shape) == 2, AssertionError( 'Shapes should be of rank 2. Got {rank}' - .format(rank=len(y_true.shape)) + .format(rank=len(y_true.shape)) ) uniques = np.unique(y_true) assert len(uniques) <= 2, AssertionError( 'Expected labels: [0, 1]. Got: {uniques}' - .format(uniques=uniques) + .format(uniques=uniques) ) -def check_k(n_items, k): +@typechecked +def check_k(n_items: int, k: int): # Make sure that inputs conform to our expectations assert isinstance(k, int), AssertionError( 'Expect k to be a {expected}. Got {actual}' - .format(expected=int, actual=type(k)) + .format(expected=int, actual=type(k)) ) assert 0 <= k <= n_items, AssertionError( 'Expect 0 <= k <= {n_items}. Got {k}' - .format(n_items=n_items, k=k) + .format(n_items=n_items, k=k) ) -def recall_at_k(y_true, y_prob, k): +@typechecked +def recall_at_k(y_true: np.ndarray, y_prob: np.ndarray, k: int) -> float: """ Calculates recall at k for binary classification ranking problems. Recall at k measures the proportion of total relevant items that are found in the @@ -118,7 +124,8 @@ def recall_at_k(y_true, y_prob, k): return recall -def precision_at_k(y_true, y_prob, k): +@typechecked +def precision_at_k(y_true: np.ndarray, y_prob: np.ndarray, k: int) -> float: """ Calculates precision at k for binary classification ranking problems. Precision at k measures the proportion of items in the top k (in ranked @@ -178,7 +185,8 @@ def precision_at_k(y_true, y_prob, k): return precision -def mean_reciprocal_rank(y_true, y_prob): +@typechecked +def mean_reciprocal_rank(y_true: np.ndarray, y_prob: np.ndarray) -> MaskedArray: """ Gets a positional score about how well you did at rank 1, rank 2, etc. The resulting vector is of size (n_items,) but element 0 corresponds to @@ -217,7 +225,8 @@ def mean_reciprocal_rank(y_true, y_prob): return ma.mean(axis=0) -def label_mean_reciprocal_rank(y_true, y_prob): +@typechecked +def label_mean_reciprocal_rank(y_true: np.ndarray, y_prob: np.ndarray) -> MaskedArray: """ Determines the average rank each label was placed across samples. Only labels that are relevant in the true data set are considered in the calculation. @@ -241,7 +250,8 @@ def label_mean_reciprocal_rank(y_true, y_prob): return ma.mean(axis=0) -def ndcg(y_true, y_prob, k=0): +@typechecked +def ndcg(y_true: np.ndarray, y_prob: np.ndarray, k=0) -> np.float64: """ A score for measuring the quality of a set of ranked results. The resulting score is between 0 and 1.0 - results that are relevant and appear earlier in the result set are given a heavier weight, so the @@ -322,7 +332,8 @@ def ndcg(y_true, y_prob, k=0): return np.mean(sample_ndcg, dtype=np.float64) -def generate_y_pred_at_k(y_prob, k): +@typechecked +def generate_y_pred_at_k(y_prob: np.ndarray, k: int) -> np.ndarray: """ Generates a matrix of binary predictions from a matrix of probabilities by evaluating the top k items (in ranked order by y_prob) as true. @@ -362,11 +373,13 @@ def generate_y_pred_at_k(y_prob, k): index_array = np.argsort(y_prob, axis=1) col_idx = np.arange(y_prob.shape[0]).reshape(-1, 1) y_pred = np.zeros(np.shape(y_prob)) - y_pred[col_idx, index_array[:, n_items-k:n_items]] = 1 + y_pred[col_idx, index_array[:, n_items - k:n_items]] = 1 return y_pred -def confusion_matrix_at_k(y_true, y_prob, k): +@typechecked +def confusion_matrix_at_k(y_true: np.ndarray, y_prob: np.ndarray, k: int) -> Tuple[ + np.ndarray, np.ndarray, np.ndarray, np.ndarray]: """ Generates binary predictions from probabilities by evaluating the top k items (in ranked order by y_prob) as true. Uses these binary predictions along with @@ -433,4 +446,3 @@ def confusion_matrix_at_k(y_true, y_prob, k): fn = np.count_nonzero((y_pred - 1) * y_true, axis=0) return tn, fp, fn, tp - diff --git a/tests/test_confusion_matrix_at_k.py b/tests/test_confusion_matrix_at_k.py index 088ba6f..90a7b00 100644 --- a/tests/test_confusion_matrix_at_k.py +++ b/tests/test_confusion_matrix_at_k.py @@ -80,6 +80,8 @@ def test_confusion_matrix_at_k_with_errors(): ) print(result) + print(type(result)) + print(type(result[0])) def test_confusion_matrix_at_k_perfect(): diff --git a/tests/test_mean_reciprocal_rank.py b/tests/test_mean_reciprocal_rank.py index a8c814b..700dbbe 100644 --- a/tests/test_mean_reciprocal_rank.py +++ b/tests/test_mean_reciprocal_rank.py @@ -117,7 +117,7 @@ def test_mrr_zeros(): np.testing.assert_allclose(result, expected) print(result) - + print(type(result)) if __name__ == '__main__': test_mrr_wikipedia() From e0396104475321e695e6fa3860460620679ef593 Mon Sep 17 00:00:00 2001 From: Anshuma Jain Date: Thu, 1 Oct 2020 15:44:06 -0700 Subject: [PATCH 2/4] fixed #9 issue --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9c70672..0545414 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ # Insert dependencies list here install_requires=[ - 'numpy', + 'numpy', 'pytypes' ], setup_requires=["setuptools-scm"], extras_require={ From e3c0c0b127118ec56c04a576e5d279e7b502b51d Mon Sep 17 00:00:00 2001 From: Anshuma Jain Date: Thu, 1 Oct 2020 15:57:13 -0700 Subject: [PATCH 3/4] Fixed pep8 warnings and doc comments. --- metriks/ranking.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/metriks/ranking.py b/metriks/ranking.py index cea43d2..22cd24b 100644 --- a/metriks/ranking.py +++ b/metriks/ranking.py @@ -15,42 +15,42 @@ def check_arrays(y_true: np.ndarray, y_prob: np.ndarray) -> None: # Make sure that inputs this conforms to our expectations assert isinstance(y_true, np.ndarray), AssertionError( 'Expect y_true to be a {expected}. Got {actual}' - .format(expected=np.ndarray, actual=type(y_true)) + .format(expected=np.ndarray, actual=type(y_true)) ) assert isinstance(y_prob, np.ndarray), AssertionError( 'Expect y_prob to be a {expected}. Got {actual}' - .format(expected=np.ndarray, actual=type(y_prob)) + .format(expected=np.ndarray, actual=type(y_prob)) ) assert y_true.shape == y_prob.shape, AssertionError( 'Shapes must match. Got y_true={true_shape}, y_prob={prob_shape}' - .format(true_shape=y_true.shape, prob_shape=y_prob.shape) + .format(true_shape=y_true.shape, prob_shape=y_prob.shape) ) assert len(y_true.shape) == 2, AssertionError( 'Shapes should be of rank 2. Got {rank}' - .format(rank=len(y_true.shape)) + .format(rank=len(y_true.shape)) ) uniques = np.unique(y_true) assert len(uniques) <= 2, AssertionError( 'Expected labels: [0, 1]. Got: {uniques}' - .format(uniques=uniques) + .format(uniques=uniques) ) @typechecked -def check_k(n_items: int, k: int): +def check_k(n_items: int, k: int) -> None: # Make sure that inputs conform to our expectations assert isinstance(k, int), AssertionError( 'Expect k to be a {expected}. Got {actual}' - .format(expected=int, actual=type(k)) + .format(expected=int, actual=type(k)) ) assert 0 <= k <= n_items, AssertionError( 'Expect 0 <= k <= {n_items}. Got {k}' - .format(n_items=n_items, k=k) + .format(n_items=n_items, k=k) ) @@ -74,7 +74,7 @@ def recall_at_k(y_true: np.ndarray, y_prob: np.ndarray, k: int) -> float: sorted order by y_prob Returns: - recall (~np.ndarray): The recall at k + recall (float): The recall at k Example: >>> y_true = np.array([ @@ -141,7 +141,7 @@ def precision_at_k(y_true: np.ndarray, y_prob: np.ndarray, k: int) -> float: sorted order by y_prob Returns: - precision_k (~np.ndarray): The precision at k + precision_k (float): The precision at k Example: >>> y_true = np.array([ From b27f3e92b2e4b5a55d47a306c3a594efdc545a87 Mon Sep 17 00:00:00 2001 From: Anshuma Jain Date: Fri, 2 Oct 2020 22:57:42 -0700 Subject: [PATCH 4/4] Made necessary changes to resolve PR review. --- metriks/ranking.py | 3 ++- setup.py | 3 ++- tests/test_confusion_matrix_at_k.py | 2 -- tests/test_label_mean_reciprocal_rank.py | 1 + tests/test_mean_reciprocal_rank.py | 2 +- tests/test_ndcg.py | 1 + 6 files changed, 7 insertions(+), 5 deletions(-) diff --git a/metriks/ranking.py b/metriks/ranking.py index 22cd24b..4f6204e 100644 --- a/metriks/ranking.py +++ b/metriks/ranking.py @@ -4,8 +4,9 @@ Metrics to use for ranking models. """ -import numpy as np from typing import Tuple + +import numpy as np from numpy.ma import MaskedArray from pytypes import typechecked diff --git a/setup.py b/setup.py index 0545414..61cd427 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,8 @@ # Insert dependencies list here install_requires=[ - 'numpy', 'pytypes' + 'numpy', + 'pytypes' ], setup_requires=["setuptools-scm"], extras_require={ diff --git a/tests/test_confusion_matrix_at_k.py b/tests/test_confusion_matrix_at_k.py index 90a7b00..088ba6f 100644 --- a/tests/test_confusion_matrix_at_k.py +++ b/tests/test_confusion_matrix_at_k.py @@ -80,8 +80,6 @@ def test_confusion_matrix_at_k_with_errors(): ) print(result) - print(type(result)) - print(type(result[0])) def test_confusion_matrix_at_k_perfect(): diff --git a/tests/test_label_mean_reciprocal_rank.py b/tests/test_label_mean_reciprocal_rank.py index 374e28d..f8a6b86 100644 --- a/tests/test_label_mean_reciprocal_rank.py +++ b/tests/test_label_mean_reciprocal_rank.py @@ -16,6 +16,7 @@ def test_label_mrr_with_errors(): expected = np.ma.array([1.0, (7/4) / 3, 2/3, 1/3]) np.testing.assert_allclose(result, expected) + print(result) diff --git a/tests/test_mean_reciprocal_rank.py b/tests/test_mean_reciprocal_rank.py index 700dbbe..a8c814b 100644 --- a/tests/test_mean_reciprocal_rank.py +++ b/tests/test_mean_reciprocal_rank.py @@ -117,7 +117,7 @@ def test_mrr_zeros(): np.testing.assert_allclose(result, expected) print(result) - print(type(result)) + if __name__ == '__main__': test_mrr_wikipedia() diff --git a/tests/test_ndcg.py b/tests/test_ndcg.py index 7f0144c..dafa738 100644 --- a/tests/test_ndcg.py +++ b/tests/test_ndcg.py @@ -207,6 +207,7 @@ def test_ndcg_0_over_0_error(): np.testing.assert_allclose([actual], [expected]) + if __name__ == '__main__': test_ndcg_perfect() test_ndcg_errors()