Merge pull request #4 from davidwarshaw/dev

 Metrics complete.

Author: David Warshaw

README.md

@@ -57,7 +57,7 @@ Each parent node, of at least one child, will generate a decision tree classific
   'labels': ['white'],
   'stage': 'light'}]
 ```
-The hmc.DecisionTreeHierarchicalClassifier is idiomatic to the sklearn tree.DecisionTreeClassifier. Fit, predict and score the same way. Traditional multi-classification accuracy is comparable.
+The hmc.DecisionTreeHierarchicalClassifier is idiomatic to the sklearn tree.DecisionTreeClassifier. Fit, predict and score the same way. Traditional multi-classification average accuracy is comparable.
 ```python
 from sklearn import tree
 dt = tree.DecisionTreeClassifier()
@@ -72,18 +72,62 @@ dth_accuracy = dth.score(X_test, y_test)
 ```
 ```python
 >>> dt_accuracy
-0.46561886051080548
+0.4400785854616896
 >>> dth_accuracy
-0.46758349705304519
+0.46561886051080548
 ```
-Hierarchically adjusted classification accuracy scoring is available in addition to traditional accuracy. This metric averages accuracy at each classification stage, penalizing the least harshly cases of the mis-classification of sibling nodes, and most harshly cases where true and predicted classes share no ancestors in the hierarchy.
+Additional hierarchical multi-classification specific metrics [2] are provided.
 ```python
-dth_accuracy_adjusted = dth.score_adjusted(X_test, y_test)
+import hmc.metrics as metrics
+
+>>> metrics.accuracy_score(ch, dth_predicted, y_test)
+0.46561886051080548
+>>> metrics.precision_score_ancestors(ch, dth_predicted, y_test)
+0.8108614232209738
+>>> metrics.recall_score_ancestors(ch, dth_predicted, y_test)
+0.7988929889298892
+>>> metrics.f1_score_ancestors(ch, dth_predicted, y_test)
+0.8048327137546468
+>>> metrics.precision_score_descendants(ch, dth_predicted, y_test)
+0.6160337552742616
+>>> metrics.recall_score_descendants(ch, dth_predicted, y_test)
+0.6576576576576577
+>>> metrics.f1_score_descendants(ch, dth_predicted, y_test)
+0.636165577342048
 ```
+Ancestor and Descendant precision and recall scores are calculated as the fraction of shared ancestor or descendant classes over the sum of either the predicted or true class for precision and recall respectively [3].
 ```python
->>> dth_accuracy_adjusted
-0.66115923150295042
-```
+true = ['dark', 'white', 'gray']
+
+pred_sibling = ['dark', 'white', 'black']
 
+>>> metrics.accuracy_score(ch, pred_sibling, true)
+0.66666666666666663
+>>> metrics.precision_score_ancestors(ch, pred_sibling, true)
+0.8
+>>> metrics.precision_score_descendants(ch, pred_sibling, true)
+0.6666666666666666
+
+pred_narrower = ['dark', 'white', 'ash']
+
+>>> metrics.accuracy_score(ch, pred_narrower, true)
+0.66666666666666663
+>>> metrics.precision_score_ancestors(ch, pred_narrower, true)
+1.0
+>>> metrics.precision_score_descendants(ch, pred_narrower, true)
+0.7777777777777778
+
+pred_broader = ['dark', 'white', 'dark']
+
+>>> metrics.accuracy_score(ch, pred_broader, true)
+0.66666666666666663
+>>> metrics.precision_score_ancestors(ch, pred_broader, true)
+0.8
+>>> metrics.precision_score_descendants(ch, pred_broader, true)
+1.0
+```
 
 1. Vens, C., Struyf, J., Schietgat, L., Džeroski, S., & Blockeel, H. (2008). Decision trees for hierarchical multi-label classification. Mach Learn Machine Learning, 73(2), 185-214.
+2. Sokolova, M., & Lapalme, G. (2009). A systematic analysis of performance measures for classification tasks. Information Processing & Management, 45(4), 427-437. doi:10.1016/j.ipm.2009.03.002
+3. Costa, E., Lorena, A., Carvalho, A., & Freitas, A. (2007). A review of performance evaluation measures for hierarchical classifiers. In Proceedings of the AAAI
+2007 workshop "Evaluation methods for machine learning" (pp. 1–6).

hmc/__init__.py

@@ -3,8 +3,13 @@
 from .hmc import DecisionTreeHierarchicalClassifier
 from .datasets import load_shades_class_hierachy
 from .datasets import load_shades_data
+from .metrics import accuracy_score
 
 __all__ = ["ClassHierarchy",
     "DecisionTreeHierarchicalClassifier",
     "load_shades_class_hierachy",
-    "load_shades_data"]
+    "load_shades_data",
+    "accuracy_score",
+    "precision_score_ancestors", "recall_score_ancestors",
+    "precision_score_descendants", "recall_score_descendants",
+    "f1_score_ancestors", "f1_score_descendants"]

hmc/hmc.py

@@ -10,6 +10,8 @@
 import numpy as np
 import pandas as pd
 
+import metrics
+
 __all__ = ["ClassHierarchy", "DecisionTreeHierarchicalClassifier"]
 
 # =============================================================================
@@ -40,6 +42,25 @@ def _get_children(self, parent):
         # Return a list of children nodes in alpha order
         return sorted([child for child, childs_parent in self.nodes.iteritems() if childs_parent == parent])
 
+    def _get_ancestors(self, child):
+        # Return a list of the ancestors of this node
+        # Not including root, not including the child
+        ancestors = []
+        while True:
+            child = self._get_parent(child)
+            if child == self.root:
+                break
+            ancestors.append(child)
+        return ancestors
+
+    def _get_descendants(self, parent):
+        # Return a list of the descendants of this node
+        # Not including the parent
+        descendants = []
+        self._depth_first(parent, descendants)
+        descendants.remove(parent)
+        return descendants
+
     def _is_descendant(self, parent, child):
         while child != self.class_hierarchy.root and child != parent:
             child = self.class_hierarchy._get_parent(child)
@@ -219,37 +240,5 @@ def score(self, X, y):
         """
         # Check that the trees have been fit
         self._check_fit()
-        classes = pd.DataFrame(self.predict(X), columns=['y_hat'], index=y.index)
-        classes['y'] = pd.DataFrame(y)
-        classes['correct'] = classes.apply(lambda row: 1 if row['y_hat'] == row['y'] else 0, axis=1)
-        return classes[['correct']].mean()[0]
-
-    def _score_stages(self, X, y):
-        y_hat = self._predict_stages(X)
-        y = pd.DataFrame(y)
-        y_classes = pd.DataFrame(index=y.index)
-
-        def assign_ancestor(classes, descendent):
-            while descendent not in classes and descendent != self.class_hierarchy.root:
-                descendent = self.class_hierarchy._get_parent(descendent)
-            if descendent == self.class_hierarchy.root and self.class_hierarchy.root not in classes:
-              descendent = ""
-            return descendent
-
-        accuracies = []
-        for stage in self.stages:
-            y_hat[stage['stage'] + "_true"] = y.apply(lambda row: assign_ancestor(stage['classes'], row[0]), axis=1)
-            y_hat[stage['stage'] + "_correct"] = y_hat.apply(lambda row: 1 if row[stage['stage'] + "_true"] == row[stage['stage']] else 0, axis=1)
-            y_hat[stage['stage'] + "_included"] = y_hat.apply(lambda row: 1 if len(row[stage['stage'] + "_true"]) > 0 else 0, axis=1)
-            accuracy = y_hat[[stage['stage'] + "_correct"]].sum()[0] / y_hat[[stage['stage'] + "_included"]].sum()[0]
-            accuracies.append(accuracy)
-        return accuracies
-
-    def score_adjusted(self, X, y):
-        """
-        Returns the hierachy adjusted mean accuracy on the given test data (X, y).
-        """
-        # Check that the trees have been fit
-        self._check_fit()
-        accuracies = self._score_stages(X, y)
-        return (1 / len(self.stages)) * sum(accuracies)
+        y_pred = pd.DataFrame(self.predict(X), columns=['y_hat'], index=y.index)
+        return metrics.accuracy_score(self.class_hierarchy, y, y_pred)

hmc/metrics.py

@@ -0,0 +1,92 @@
+"""
+Metrics for evaluating hierachical multi-classification performance.
+"""
+
+from __future__ import print_function
+from __future__ import division
+
+from sklearn import tree
+from sklearn import metrics as skmetrics
+from sklearn.utils import check_consistent_length
+from sklearn.utils import column_or_1d
+from sklearn.utils.multiclass import type_of_target
+
+from itertools import chain
+
+import numpy as np
+import pandas as pd
+
+def _check_targets_hmc(y_true, y_pred):
+    check_consistent_length(y_true, y_pred)
+    y_type = set([type_of_target(y_true), type_of_target(y_pred)])
+    if y_type == set(["binary", "multiclass"]):
+        y_type = set(["multiclass"])
+    if y_type != set(["multiclass"]):
+        raise ValueError("{0} is not supported".format(y_type))
+    y_true = column_or_1d(y_true)
+    y_pred = column_or_1d(y_pred)
+    return y_true, y_pred
+
+## General Scores
+# Average accuracy
+def accuracy_score(class_hierarchy, y_true, y_pred):
+    y_true, y_pred = _check_targets_hmc(y_true, y_pred)
+    return skmetrics.accuracy_score(y_true, y_pred)
+
+## Hierarchy Precision / Recall
+def _aggregate_class_sets(set_function, y_true, y_pred):
+    intersection_sum = 0
+    true_sum = 0
+    predicted_sum = 0
+    for true, pred in zip(y_true.tolist(), y_pred.tolist()):
+        true_set = set([true] + set_function(true))
+        pred_set = set([pred] + set_function(pred))
+        intersection_sum += len(true_set.intersection(pred_set))
+        true_sum += len(true_set)
+        predicted_sum += len(pred_set)
+    return (true_sum, predicted_sum, intersection_sum)
+
+# Ancestors Scores (Super Class)
+# Precision
+def precision_score_ancestors(class_hierarchy, y_true, y_pred):
+    y_true, y_pred = _check_targets_hmc(y_true, y_pred)
+    true_sum, predicted_sum, intersection_sum = _aggregate_class_sets(class_hierarchy._get_ancestors, y_true, y_pred)
+    return intersection_sum / predicted_sum
+
+# Recall
+def recall_score_ancestors(class_hierarchy, y_true, y_pred):
+    y_true, y_pred = _check_targets_hmc(y_true, y_pred)
+    true_sum, predicted_sum, intersection_sum = _aggregate_class_sets(class_hierarchy._get_ancestors, y_true, y_pred)
+    return intersection_sum / true_sum
+
+# Descendants Scores (Sub Class)
+# Precision
+def precision_score_descendants(class_hierarchy, y_true, y_pred):
+    y_true, y_pred = _check_targets_hmc(y_true, y_pred)
+    true_sum, predicted_sum, intersection_sum = _aggregate_class_sets(class_hierarchy._get_descendants, y_true, y_pred)
+    return intersection_sum / predicted_sum
+
+# Recall
+def recall_score_descendants(class_hierarchy, y_true, y_pred):
+    y_true, y_pred = _check_targets_hmc(y_true, y_pred)
+    true_sum, predicted_sum, intersection_sum = _aggregate_class_sets(class_hierarchy._get_descendants, y_true, y_pred)
+    return intersection_sum / true_sum
+
+# Hierarchy Fscore
+def _fbeta_score_class_sets(set_function, y_true, y_pred, beta=1):
+    y_true, y_pred = _check_targets_hmc(y_true, y_pred)
+    true_sum, predicted_sum, intersection_sum = _aggregate_class_sets(set_function, y_true, y_pred)
+    precision = intersection_sum / predicted_sum
+    recall = intersection_sum / true_sum
+    return ((beta ** 2 + 1) * precision * recall) / ((beta ** 2 * precision) + recall)
+
+def f1_score_ancestors(class_hierarchy, y_true, y_pred):
+    y_true, y_pred = _check_targets_hmc(y_true, y_pred)
+    return _fbeta_score_class_sets(class_hierarchy._get_ancestors, y_true, y_pred)
+
+def f1_score_descendants(class_hierarchy, y_true, y_pred):
+    y_true, y_pred = _check_targets_hmc(y_true, y_pred)
+    return _fbeta_score_class_sets(class_hierarchy._get_descendants, y_true, y_pred)
+
+# # Classification Report
+# def classification_report(class_hierarchy, y_true, y_pred):

setup.py

@@ -7,5 +7,5 @@
       description='Decision tree based hierachical multi-classifier',
       author='David Warshaw',
       author_email='[email protected]',
-      py_modules=['hmc', 'datasets'],
+      py_modules=['hmc', 'datasets', 'metrics'],
       requires=['sklearn', 'numpy', 'pandas'])

tests/__init__.py

@@ -1,2 +1,3 @@
 from .test_hmc import TestClassHierarchy
 from .test_hmc import TestDecisionTreeHierarchicalClassifier
+from .test_metrics import TestMetrics

tests/test_hmc.py

@@ -29,6 +29,16 @@ def test_get_children(self):
         ch = hmc.load_shades_class_hierachy()
         self.assertEqual(ch._get_children('dark'), ['black', 'gray'])
 
+    def test_get_ancestors(self):
+        ch = hmc.load_shades_class_hierachy()
+        self.assertEqual(ch._get_ancestors('ash'), ['gray', 'dark'])
+        self.assertEqual(len(ch._get_ancestors('colors')), 0)
+
+    def test_get_descendants(self):
+        ch = hmc.load_shades_class_hierachy()
+        self.assertEqual(ch._get_descendants('dark'), ['black', 'gray', 'ash', 'slate'])
+        self.assertEqual(len(ch._get_descendants('slate')), 0)
+
     def test_add_node(self):
         ch = hmc.load_shades_class_hierachy()
         old_number = len(ch.nodes_())
@@ -111,18 +121,6 @@ def test_score(self):
         # Hierachical classification should be at least as accurate as traditional classification
         self.assertTrue(accuracy >= accuracy_nonh)
 
-    def test_score_adjusted(self):
-        ch = hmc.load_shades_class_hierachy()
-        X, y = hmc.load_shades_data()
-        X_train, X_test, y_train, y_test = train_test_split(X, y,
-            test_size = 0.50, random_state = 0)
-        dt = hmc.DecisionTreeHierarchicalClassifier(ch)
-        dt = dt.fit(X_train, y_train)
-        accuracy = dt.score(X_test, y_test)
-        accuracy_adjusted = dt.score_adjusted(X_test, y_test)
-        # Adjusted accuracy should be at least as high as final class accuracy
-        self.assertTrue(accuracy_adjusted >= accuracy)
-
     def test_score_before_fit(self):
         ch = hmc.load_shades_class_hierachy()
         X, y = hmc.load_shades_data()

tests/test_metrics.py

@@ -0,0 +1,79 @@
+"""
+Tests for the hmc metrics module.
+"""
+
+import unittest
+
+import pandas as pd
+
+from sklearn import tree
+from sklearn.cross_validation import train_test_split
+from sklearn import metrics as skmetrics
+
+import hmc
+import hmc.metrics as metrics
+
+class TestMetrics(unittest.TestCase):
+
+    def setUp(self):
+        self.ch = hmc.load_shades_class_hierachy()
+        self.X, self.y = hmc.load_shades_data()
+        self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(self.X, self.y,
+            test_size=0.50, random_state=0)
+        self.dt = hmc.DecisionTreeHierarchicalClassifier(self.ch)
+        self.dt_nonh = tree.DecisionTreeClassifier()
+        self.dt = self.dt.fit(self.X_train, self.y_train)
+        self.dt_nonh = self.dt_nonh.fit(self.X_train, self.y_train)
+        self.y_pred = self.dt.predict(self.X_test)
+        self.y_pred_nonh = self.dt_nonh.predict(self.X_test)
+
+    ## General Scores
+    # Average accuracy
+    def test_accuracy_score(self):
+        accuracy = metrics.accuracy_score(self.ch, self.y_test, self.y_pred)
+        accuracy_sk = skmetrics.accuracy_score(self.y_test, self.y_pred)
+        # Hierachical classification should be at least as accurate as traditional classification
+        self.assertTrue(accuracy >= accuracy_sk)
+
+    ## Hierarchy Precision / Recall
+    # Ancestors Scores (Super Class)
+    # Precision
+    def test_precision_score_ancestors(self):
+        precision_ancestors = metrics.precision_score_ancestors(self.ch, self.y_test, self.y_pred)
+        precision_sk = skmetrics.precision_score(self.y_test, self.y_pred, average="macro")
+        self.assertTrue(precision_ancestors >= precision_sk)
+
+    # Recall
+    def test_recall_score_ancestors(self):
+        recall_ancestors = metrics.recall_score_ancestors(self.ch, self.y_test, self.y_pred)
+        recall_sk = skmetrics.recall_score(self.y_test, self.y_pred, average="macro")
+        self.assertTrue(recall_ancestors >= recall_sk)
+
+    # Descendants Scores (Sub Class)
+    # Precision
+    def test_precision_score_descendants(self):
+        precision_descendants = metrics.precision_score_descendants(self.ch, self.y_test, self.y_pred)
+        precision_sk = skmetrics.precision_score(self.y_test, self.y_pred, average="macro")
+        self.assertTrue(precision_descendants >= precision_sk)
+
+    # Recall
+    def test_recall_score_descendants(self):
+        recall_descendants = metrics.recall_score_descendants(self.ch, self.y_test, self.y_pred)
+        recall_sk = skmetrics.recall_score(self.y_test, self.y_pred, average="macro")
+        self.assertTrue(recall_descendants >= recall_sk)
+
+    # F1
+    # Ancestors
+    def test_f1_score_ancestors(self):
+        f1_ancestors = metrics.f1_score_ancestors(self.ch, self.y_test, self.y_pred)
+        f1_sk = skmetrics.f1_score(self.y_test, self.y_pred, average="macro")
+        self.assertTrue(f1_ancestors >= f1_sk)
+
+    # Descendants
+    def test_f1_score_descendants(self):
+        f1_descendants = metrics.f1_score_descendants(self.ch, self.y_test, self.y_pred)
+        f1_sk = skmetrics.f1_score(self.y_test, self.y_pred, average="macro")
+        self.assertTrue(f1_descendants >= f1_sk)
+
+if __name__ == '__main__':
+    unittest.main()