Merge pull request #44 from bashtage/test-single-mcs

TST: Add tests and protections for edge cases

Author: bashtage

README.md

@@ -128,7 +128,7 @@ ci = bs.conf_int(sharpe_ratio, 1000, method='percentile')
 * Stepwise (StepM)
 * Model Confidence Set (MCS)
 
-See the [multiple comparison example notebook](http://nbviewer.ipython.org/github/bashtage/arch/blob/master/examples/bootstrap_multiple_comparison.ipynb) 
+See the [multiple comparison example notebook](http://nbviewer.ipython.org/github/bashtage/arch/blob/master/examples/multiple-comparison_examples.ipynb) 
 for examples of the multiple comparison procedures.
 
 ## Requirements

arch/bootstrap/multiple_comparrison.py

@@ -112,7 +112,9 @@ class MCS(MultipleComparison):
     def __init__(self, losses, size, reps=1000, block_size=None, method='R',
                  bootstrap='stationary'):
         self.losses = ensure2d(losses, 'losses')
-        self._losses_arr = np.array(losses)
+        self._losses_arr = np.asarray(self.losses)
+        if self._losses_arr.shape[1] < 2:
+            raise ValueError('losses must have at least two columns')
         self.size = size
         self.reps = reps
         if block_size is None:
@@ -379,7 +381,7 @@ def compute(self):
         better_models = list(self.spa.better_models(self.size))
         all_better_models = better_models
         # 3. Stop if nothing superior
-        while better_models:
+        while better_models and (len(better_models) < self.k):
             # A. Use Selector to remove better models
             selector = np.ones(self.k, dtype=np.bool)
             if isinstance(self.models, pd.DataFrame):  # Columns
@@ -403,6 +405,9 @@ def superior_models(self):
         """
         Returns a list of the indices or column names of the superior models.
         """
+        if self._superior_models is None:
+            msg = 'compute must be callded before accessing superior_models'
+            raise RuntimeError(msg)
         return self._superior_models
 
 

arch/bootstrap/tests/test_multiple_comparrison.py

@@ -147,6 +147,14 @@ def test_bootstrap_selection(self):
         spa = SPA(self.benchmark, self.models, bootstrap='moving block')
         assert_true(isinstance(spa.bootstrap, MovingBlockBootstrap))
 
+    def test_single_model(self):
+        spa = SPA(self.benchmark, self.models[:,0])
+        spa.compute()
+
+        spa = SPA(self.benchmark_series, self.models_df.iloc[:,0])
+        spa.compute()
+
+
 
 class TestStepM(TestCase):
     @classmethod
@@ -213,6 +221,24 @@ def test_str_repr(self):
         expected = 'StepM(FWER (size): 0.05, studentization: none, bootstrap: ' + str(stepm.spa.bootstrap) + ')'
         assert_equal(expected, str(stepm))
 
+    def test_single_model(self):
+        stepm = StepM(self.benchmark, self.models[:,0], size=0.10)
+        stepm.compute()
+
+        stepm = StepM(self.benchmark_series, self.models_df.iloc[:,0])
+        stepm.compute()
+
+    def test_all_superior(self):
+        adj_models = self.models - 100.0
+        stepm = StepM(self.benchmark, adj_models, size=0.10)
+        stepm.compute()
+        assert_equal(len(stepm.superior_models), self.models.shape[1])
+
+    def test_errors(self):
+        stepm = StepM(self.benchmark, self.models, size=0.10)
+        with assert_raises(RuntimeError):
+            stepm.superior_models
+
 
 class TestMCS(TestCase):
     @classmethod
@@ -328,12 +354,14 @@ def test_smoke(self):
         assert_true(isinstance(mcs.pvalues, pd.DataFrame))
 
     def test_errors(self):
+        assert_raises(ValueError, MCS, self.losses[:,1], 0.05)
         mcs = MCS(self.losses, 0.05, reps=100, block_size=10, method='max', bootstrap='circular')
         mcs.compute()
         mcs = MCS(self.losses, 0.05, reps=100, block_size=10, method='max', bootstrap='moving block')
         mcs.compute()
         assert_raises(ValueError, MCS, self.losses, 0.05, bootstrap='unknown')
 
+
     def test_str_repr(self):
         mcs = MCS(self.losses, 0.05)
         expected = 'MCS(size: 0.05, bootstrap: ' + str(mcs.bootstrap) + ')'