Merge pull request #14 from AstraZeneca/tests-restructure

Tests restructured for scorecards and added non groupped aggregations

Author: benedekrozemberczki

rexmex/dataset.py

@@ -13,9 +13,7 @@ class DatasetReader(object):
     """
 
     def __init__(self):
-        self.base_url = (
-            "https://raw.githubusercontent.com/AstraZeneca/rexmex/main/dataset/"
-        )
+        self.base_url = "https://raw.githubusercontent.com/AstraZeneca/rexmex/main/dataset/"
 
     def read_dataset(self, dataset: str = "erdos_renyi_example"):
         """

rexmex/metrics/classification.py

@@ -563,8 +563,6 @@ def pr_auc_score(y_true: np.array, y_score: np.array) -> float:
     Returns:
         pr_auc (float): The value of the precision-recall area under the curve.
     """
-    precision, recall, thresholds = sklearn.metrics.precision_recall_curve(
-        y_true, y_score
-    )
+    precision, recall, thresholds = sklearn.metrics.precision_recall_curve(y_true, y_score)
     pr_auc = sklearn.metrics.auc(recall, precision)
     return pr_auc

rexmex/metrics/rating.py

@@ -87,9 +87,7 @@ def root_mean_squared_error(y_true: np.array, y_score: np.array) -> float:
     return rmse
 
 
-def symmetric_mean_absolute_percentage_error(
-    y_true: np.array, y_score: np.array
-) -> float:
+def symmetric_mean_absolute_percentage_error(y_true: np.array, y_score: np.array) -> float:
     """
     Calculate the symmetric mean absolute percentage error (SMAPE) for a ground-truth prediction vector pair.
 
@@ -99,7 +97,5 @@ def symmetric_mean_absolute_percentage_error(
     Returns:
         smape (float): The value of the symmetric mean absolute percentage error.
     """
-    smape = 100 * np.mean(
-        np.abs(y_score - y_true) / ((np.abs(y_score) + np.abs(y_true)) / 2)
-    )
+    smape = 100 * np.mean(np.abs(y_score - y_true) / ((np.abs(y_score) + np.abs(y_true)) / 2))
     return smape

rexmex/metricset.py

@@ -1,5 +1,4 @@
-import pandas as pd
-from typing import List, Dict, Tuple
+from typing import List, Tuple
 
 from rexmex.utils import binarize, normalize
 
@@ -114,9 +113,7 @@ def __init__(self):
         self["pr_auc"] = pr_auc_score
         self["average_precision"] = average_precision_score
         self["f1_score"] = binarize(f1_score)
-        self["matthews_correlation_coefficent"] = binarize(
-            matthews_correlation_coefficient
-        )
+        self["matthews_correlation_coefficent"] = binarize(matthews_correlation_coefficient)
         self["fowlkes_mallows_index"] = binarize(fowlkes_mallows_index)
         self["precision"] = binarize(precision_score)
         self["recall"] = binarize(recall_score)

rexmex/scorecard.py

@@ -10,11 +10,9 @@ class ScoreCard(object):
     """
 
     def __init__(self, metric_set: rexmex.metricset.MetricSet):
-        self._metric_set = metric_set
+        self.metric_set = metric_set
 
-    def _get_performance_metrics(
-        self, y_true: np.array, y_score: np.array
-    ) -> pd.DataFrame:
+    def _get_performance_metrics(self, y_true: np.array, y_score: np.array) -> pd.DataFrame:
         """
         A method to get the performance metrics for a pair of vectors.
 
@@ -24,15 +22,11 @@ def _get_performance_metrics(
         Returns:
             performance_metrics (pd.DataFrame): The performance metrics calculated from the vectors.
         """
-        performance_metrics = {
-            name: [metric(y_true, y_score)] for name, metric in self._metric_set.items()
-        }
+        performance_metrics = {name: [metric(y_true, y_score)] for name, metric in self.metric_set.items()}
         performance_metrics = pd.DataFrame.from_dict(performance_metrics)
         return performance_metrics
 
-    def generate_report(
-        self, scores_to_evaluate: pd.DataFrame, groupping: List[str] = None
-    ) -> pd.DataFrame:
+    def generate_report(self, scores_to_evaluate: pd.DataFrame, groupping: List[str] = None) -> pd.DataFrame:
         """
         A method to calculate (aggregated) performance metrics based
         on a dataframe of ground truth and predictions. It assumes that the dataframe has the `y_true`
@@ -47,13 +41,9 @@ def generate_report(
         """
         if groupping is not None:
             scores_to_evaluate = scores_to_evaluate.groupby(groupping)
-            report = scores_to_evaluate.apply(
-                lambda group: self._get_performance_metrics(group.y_true, group.y_score)
-            )
+            report = scores_to_evaluate.apply(lambda group: self._get_performance_metrics(group.y_true, group.y_score))
         else:
-            report = self._get_performance_metrics(
-                scores_to_evaluate.y_true, scores_to_evaluate.y_score
-            )
+            report = self._get_performance_metrics(scores_to_evaluate.y_true, scores_to_evaluate.y_score)
         return report
 
     def __repr__(self):
@@ -66,4 +56,4 @@ def print_metrics(self):
         """
         Printing the name of metrics.
         """
-        print({k for k in self._metric_set.keys()})
+        print({k for k in self.metric_set.keys()})

tests/integration/test_aggregation.py

@@ -15,6 +15,9 @@ def test_classification(self):
         metric_set = ClassificationMetricSet()
         score_card = ScoreCard(metric_set)
 
+        performance_metrics = score_card.generate_report(self.scores)
+        assert performance_metrics.shape == (1, 11)
+
         performance_metrics = score_card.generate_report(
             self.scores, groupping=["source_group"]
         )
@@ -30,6 +33,9 @@ def test_regression(self):
         metric_set.normalize_metrics()
         score_card = ScoreCard(metric_set)
 
+        performance_metrics = score_card.generate_report(self.scores)
+        assert performance_metrics.shape == (1, 7)
+
         performance_metrics = score_card.generate_report(
             self.scores, groupping=["source_group"]
         )
@@ -44,6 +50,9 @@ def test_addition(self):
         metric_set = RatingMetricSet() + ClassificationMetricSet()
         score_card = ScoreCard(metric_set)
 
+        performance_metrics = score_card.generate_report(self.scores)
+        assert performance_metrics.shape == (1, 18)
+
         performance_metrics = score_card.generate_report(
             self.scores, groupping=["source_group"]
         )

tests/unit/test_scorecard.py

@@ -8,18 +8,18 @@
 
 
 class TestMetricSet(unittest.TestCase):
+    def setUp(self):
+        self.metric_set = ClassificationMetricSet()
+        self.score_card = ScoreCard(self.metric_set)
+
     def test_representation(self):
-        metric_set = ClassificationMetricSet()
-        score_card = ScoreCard(metric_set)
-        assert repr(score_card) == "ScoreCard()"
+        assert repr(self.score_card) == "ScoreCard()"
 
     def test_printing(self):
-        metric_set = ClassificationMetricSet()
-        metric_set.filter_metrics(["roc_auc", "pr_auc"])
-        score_card = ScoreCard(metric_set)
+        self.score_card.metric_set.filter_metrics(["roc_auc", "pr_auc"])
         captured = StringIO()
         sys.stdout = captured
-        score_card.print_metrics()
+        self.score_card.print_metrics()
         sys.stdout = sys.__stdout__
         out = captured.getvalue().strip("\n")
         assert out == str({"roc_auc", "pr_auc"})

tox.ini

@@ -21,4 +21,4 @@ deps =
     flake8
     flake8-black
 commands =
-    flake8 --select BLK100 rexmex/ tests/ setup.py
+    flake8 --select BLK120 rexmex/ tests/ setup.py