[178390751]: remove the pairwise_significance legacy package

Author: ernestoarbitrio

src/cr/cube/cubepart.py

@@ -22,7 +22,7 @@
 from cr.cube.enums import CUBE_MEASURE as CM
 from cr.cube.min_base_size_mask import MinBaseSizeMask
 from cr.cube.matrix import Assembler
-from cr.cube.measures.pairwise_significance import PairwiseSignificance
+
 from cr.cube.noa.smoothing import SingleSidedMovingAvgSmoother
 from cr.cube.scalar import MeansScalar
 from cr.cube.stripe.assembler import StripeAssembler
@@ -1046,12 +1046,6 @@ def sums(self):
                 "`.sums` is undefined for a cube-result without a sum measure"
             )
 
-    @lazyproperty
-    def summary_pairwise_indices(self):
-        return PairwiseSignificance(
-            self, self._alpha, self._only_larger
-        ).summary_pairwise_indices
-
     @lazyproperty
     def table_base(self):
         """Scalar or 1D/2D np.float64 ndarray of unweighted-N for table.
@@ -1184,26 +1178,6 @@ def zscores(self):
 
     # ---implementation (helpers)-------------------------------------
 
-    @lazyproperty
-    def _alpha(self):
-        """float confidence-interval threshold for pairwise-t (sig) tests."""
-        # TODO: remove me when summary_pairwise_indices, summary_p_vals and
-        # summary_t_stats will be moved from measures/pairwise_significance.py to
-        # matrix/measures.py
-        return self._columns_dimension.pairwise_significance_spec.alpha_values[0]
-
-    @lazyproperty
-    def _alpha_alt(self):
-        """Alternate float confidence-interval threshold or None.
-
-        This is an optional secondary confidence interval allowing two-level
-        significance testing. Value is None if no alternate alpha was specified by user.
-        """
-        # TODO: remove me when summary_pairwise_indices, summary_p_vals and
-        # summary_t_stats will be moved from measures/pairwise_significance.py to
-        # matrix/measures.py
-        return self._columns_dimension.pairwise_significance_spec.alpha_values[1]
-
     @lazyproperty
     def _assembler(self):
         """The Assembler object for this slice.
@@ -1241,21 +1215,6 @@ def _dimensions(self):
             )
         )
 
-    @lazyproperty
-    def _only_larger(self):
-        """True if only the larger of reciprocal pairwise-t values should appear.
-
-        In general, pairwise-t tests are reciprocal. That is, if A is significant with
-        respect to B, then B is significant with respect to A. Having a letter in both
-        columns can produce a cluttered appearance. When this flag is set by the user,
-        only the cell in the reciprocal pair having the largest value gets a letter.
-        Defaults to True unless explicitly set False.
-        """
-        # TODO: remove me when summary_pairwise_indices, summary_p_vals and
-        # summary_t_stats will be moved from measures/pairwise_significance.py to
-        # matrix/measures.py
-        return self._columns_dimension.pairwise_significance_spec.only_larger
-
     @lazyproperty
     def _rows_dimension(self):
         return self._dimensions[0]

src/cr/cube/measures/__init__.py

@@ -1,82 +0,0 @@
-# encoding: utf-8
-
-"""T-score based P-values of pairwise comparison or columns of a contingency table."""
-
-import numpy as np
-from scipy.stats import t
-
-from cr.cube.util import lazyproperty
-
-
-class PairwiseSignificance(object):
-    """Implementation of p-vals and t-tests for each column proportions comparison."""
-
-    def __init__(self, slice_, alpha=0.05, only_larger=True):
-        self._slice = slice_
-        self._alpha = alpha
-        self._only_larger = only_larger
-
-    @lazyproperty
-    def summary_pairwise_indices(self):
-        """ndarray containing tuples of pairwise indices for the column summary."""
-        summary_pairwise_indices = np.empty(
-            self.values[0].summary_t_stats.shape[0], dtype=object
-        )
-        summary_pairwise_indices[:] = [
-            sig.summary_pairwise_indices for sig in self.values
-        ]
-        return summary_pairwise_indices
-
-    @lazyproperty
-    def values(self):
-        """list of _ColumnPairwiseSignificance tests.
-
-        Result has as many elements as there are coliumns in the slice. Each
-        significance test contains `p_vals` and `t_stats` significance tests.
-        """
-        return [
-            _ColumnPairwiseSignificance(
-                self._slice, col_idx, self._alpha, self._only_larger
-            )
-            for col_idx in range(self._slice.shape[1])
-        ]
-
-
-class _ColumnPairwiseSignificance(object):
-    """Value object providing matrix of T-score based pairwise-comparison P-values"""
-
-    def __init__(self, slice_, col_idx, alpha=0.05, only_larger=True):
-        self._slice = slice_
-        self._col_idx = col_idx
-        self._alpha = alpha
-        self._only_larger = only_larger
-
-    @lazyproperty
-    def summary_p_vals(self):
-        return 2 * (1 - t.cdf(abs(self.summary_t_stats), df=self._df))
-
-    @lazyproperty
-    def summary_pairwise_indices(self):
-        significance = self.summary_p_vals < self._alpha
-        if self._only_larger:
-            significance = np.logical_and(self.summary_t_stats < 0, significance)
-        return tuple(np.where(significance)[0])
-
-    @lazyproperty
-    def summary_t_stats(self):
-        col_margin_props = self._slice.columns_base / self._slice.table_margin
-        diff = col_margin_props - col_margin_props[self._col_idx]
-        var_props = (
-            col_margin_props * (1.0 - col_margin_props) / self._slice.table_margin
-        )
-        se_diff = np.sqrt(var_props + var_props[self._col_idx])
-        return diff / se_diff
-
-    @lazyproperty
-    def _df(self):
-        selected_unweighted_n = (
-            self._slice.columns_base[self._col_idx]
-            if self._slice.columns_base.ndim < 2
-            else self._slice.columns_base[:, self._col_idx][:, None]
-        )
-        return self._slice.columns_base + selected_unweighted_n - 2

src/cr/cube/measures/pairwise_significance.py

@@ -489,26 +489,6 @@ def test_cat_x_cat_pruning_and_hs_scale_means_pairwise_t_tests(self):
             [1.0, 0.1046981, 0.059721, 0.2918845]
         )
 
-    def test_cat_x_cat_summary_pairwise_indices(self):
-        # Only larger
-        slice_ = Cube(CR.PAIRWISE_HIROTSU_OCCUPATION_X_ILLNESS).partitions[0]
-        pairwise_indices = slice_.summary_pairwise_indices
-        expected_indices = np.array([(2,), (0, 2), ()], dtype=tuple)
-        np.testing.assert_array_equal(pairwise_indices, expected_indices)
-
-        # Larger and smaller
-        transforms = {
-            "columns_dimension": {
-                "pairwise_significance": {"pairwise_indices": {"only_larger": False}},
-            },
-        }
-        slice_ = Cube(
-            CR.PAIRWISE_HIROTSU_OCCUPATION_X_ILLNESS, transforms=transforms
-        ).partitions[0]
-        pairwise_indices = slice_.summary_pairwise_indices
-        expected_indices = np.array([(1, 2), (0, 2), (0, 1)], dtype="i,i")
-        np.testing.assert_array_equal(pairwise_indices, expected_indices)
-
     def test_cat_x_cat_wgtd_pairwise_t_tests(self):
         """The weights on this cube demonstrate much higher variance (less
         extreme t values, and higher associated p-values) than if weighted_n

tests/integration/test_pairwise_significance.py

@@ -159,11 +159,6 @@ def it_knows_whether_it_is_empty(self, shape, expected_value, shape_prop_):
 
         assert is_empty is expected_value
 
-    def but_it_returns_None_when_no_secondary_alpha_specified(self, cube_):
-        cube_.alpha_values = (0.05, None)
-
-        assert _Slice(cube_, None, None, None, None).pairwise_indices_alt is None
-
     def it_knows_the_population_fraction(self, cube_):
         cube_.population_fraction = 0.5
         slice_ = _Slice(cube_, None, None, None, None)
@@ -198,14 +193,11 @@ def it_provides_the_scale_mean_pairwise_indices(
         self,
         _assembler_prop_,
         assembler_,
-        PairwiseSignificance_,
         dimension_,
         _dimensions_prop_,
     ):
         assembler_.scale_mean_pairwise_indices = ((2,), (0,), ())
         _assembler_prop_.return_value = assembler_
-        PairwiseSignificance_.alpha_values = (0.05, None)
-        dimension_.pairwise_significance_spec = PairwiseSignificance_
         _dimensions_prop_.return_value = (None, dimension_)
         slice_ = _Slice(None, None, None, None, None)
 
@@ -225,14 +217,11 @@ def it_provides_the_secondary_scale_mean_pairwise_indices(
         self,
         _assembler_prop_,
         assembler_,
-        PairwiseSignificance_,
         dimension_,
         _dimensions_prop_,
     ):
         assembler_.scale_mean_pairwise_indices_alt = ((2,), (0,), ())
         _assembler_prop_.return_value = assembler_
-        PairwiseSignificance_.alpha_values = (0.05, 0.01)
-        dimension_.pairwise_significance_spec = PairwiseSignificance_
         _dimensions_prop_.return_value = (None, dimension_)
         slice_ = _Slice(None, None, None, None, None)
 
@@ -293,61 +282,6 @@ def it_constructs_its_assembler_instance_to_help(
         Assembler_.assert_called_once_with(cube_, (dimension_, dimension_), slice_idx)
         assert assembler is assembler_
 
-    def it_knows_the_primary_alpha_value_to_help(
-        self, dimension_, _dimensions_prop_, PairwiseSignificance_
-    ):
-        """alpha is the primary confidence-interval threshold specified by the user."""
-        PairwiseSignificance_.alpha_values = (0.042, 0.084)
-        dimension_.pairwise_significance_spec = PairwiseSignificance_
-        _dimensions_prop_.return_value = (None, dimension_)
-        slice_ = _Slice(None, None, None, None, None)
-
-        assert slice_._alpha == 0.042
-
-    @pytest.mark.parametrize(
-        "alpha_values, expected_value", (((0.042, 0.084), 0.084), ((0.042, None), None))
-    )
-    def it_knows_the_secondary_alpha_value_to_help(
-        self,
-        dimension_,
-        _dimensions_prop_,
-        PairwiseSignificance_,
-        alpha_values,
-        expected_value,
-    ):
-        PairwiseSignificance_.alpha_values = alpha_values
-        dimension_.pairwise_significance_spec = PairwiseSignificance_
-        _dimensions_prop_.return_value = (None, dimension_)
-        slice_ = _Slice(None, None, None, None, None)
-
-        assert slice_._alpha_alt == expected_value
-
-    def but_it_returns_None_when_no_secondary_alpha_specified(
-        self,
-        dimension_,
-        _dimensions_prop_,
-        PairwiseSignificance_,
-    ):
-        PairwiseSignificance_.alpha_values = (0.05, None)
-        dimension_.pairwise_significance_spec = PairwiseSignificance_
-        _dimensions_prop_.return_value = (None, dimension_)
-        slice_ = _Slice(None, None, None, None, None)
-
-        assert slice_._alpha_alt is None
-
-    def it_knows_the_only_larger_flag_state_to_help(
-        self,
-        dimension_,
-        _dimensions_prop_,
-        PairwiseSignificance_,
-    ):
-        PairwiseSignificance_.only_larger = False
-        dimension_.pairwise_significance_spec = PairwiseSignificance_
-        _dimensions_prop_.return_value = (None, dimension_)
-        slice_ = _Slice(None, None, None, None, None)
-
-        assert slice_._only_larger is False
-
     # fixture components ---------------------------------------------
 
     @pytest.fixture
@@ -374,10 +308,6 @@ def _dimensions_prop_(self, request):
     def dimension_types_prop_(self, request):
         return property_mock(request, _Slice, "dimension_types")
 
-    @pytest.fixture
-    def PairwiseSignificance_(self, request):
-        return class_mock(request, "cr.cube.cubepart.PairwiseSignificance")
-
     @pytest.fixture
     def shape_prop_(self, request):
         return property_mock(request, _Slice, "shape")