[178390751]: address comments 3part

Author: ernestoarbitrio

src/cr/cube/matrix/assembler.py

@@ -251,32 +251,28 @@ def means(self):
     @lazyproperty
     def pairwise_indices(self):
         """2D ndarray of tuple of int column-idxs pairwise-t threshold."""
-        p_vals, t_stats = (
+        indices_matrix = (
             self._measures.pairwise_indices_for_subvar.values
             if self._cube.has_overlaps
             else self._measures.pairwise_indices.values
         )
-        return self._assemble_pw_indices(
-            p_vals, t_stats, self._alpha, self._only_larger
-        )
+        return self._assemble_matrix(self._remap_indices(indices_matrix))
 
     @lazyproperty
     def pairwise_indices_alt(self):
-        """Optional 2D ndarray of tuple of int column-idxs.
+        """Optional 2D ndarray of tuple of int column-idxs pairwise-t alt threshold.
 
         Represents the pairwise significance array of tuples considering the alternative
         threshold value.
         """
         if self._alpha_alt is None:
             return None
-        p_vals, t_stats = (
-            self._measures.pairwise_indices_for_subvar.values
+        indices_matrix = (
+            self._measures.pairwise_indices_for_subvar_alt.values
             if self._cube.has_overlaps
-            else self._measures.pairwise_indices.values
-        )
-        return self._assemble_pw_indices(
-            p_vals, t_stats, self._alpha_alt, self._only_larger
+            else self._measures.pairwise_indices_alt.values
         )
+        return self._assemble_matrix(self._remap_indices(indices_matrix))
 
     @lazyproperty
     def pairwise_means_indices(self):
@@ -287,14 +283,13 @@ def pairwise_means_indices(self):
 
         Raises `ValueError` if the cube-result does not include `means` cube-measures.
         """
-        p_vals, t_stats = self._measures.pairwise_means_indices.values
-        return self._assemble_pw_indices(
-            p_vals, t_stats, self._alpha, self._only_larger
+        return self._assemble_matrix(
+            self._remap_indices(self._measures.pairwise_means_indices.values)
         )
 
     @lazyproperty
     def pairwise_means_indices_alt(self):
-        """Optional 2D. ndarray of tuple of int column-idxs.
+        """Optional 2D ndarray of tuple of int column-idxs.
 
         Represents the pairwise mean significance array of tuples considering the
         alternative threshold value.
@@ -303,9 +298,8 @@ def pairwise_means_indices_alt(self):
         """
         if self._alpha_alt is None:
             return None
-        p_vals, t_stats = self._measures.pairwise_means_indices.values
-        return self._assemble_pw_indices(
-            p_vals, t_stats, self._alpha_alt, self._only_larger
+        return self._assemble_matrix(
+            self._remap_indices(self._measures.pairwise_means_indices_alt.values)
         )
 
     @lazyproperty
@@ -574,20 +568,22 @@ def rows_scale_median(self):
 
     @lazyproperty
     def scale_mean_pairwise_indices(self):
-        """..."""
-        p_vals, t_stats = self._measures.scale_mean_pairwise_indices.values
-        return self._assemble_pw_scale_mean_indices(
-            p_vals, t_stats, self._alpha, self._only_larger
+        """1D ndarray of tuple of int pairwise-t threshold for scale means."""
+        return self._remap_indices(
+            self._assemble_marginal(self._measures.scale_mean_pairwise_indices)
         )
 
     @lazyproperty
     def scale_mean_pairwise_indices_alt(self):
-        """1D optional ndarray of tuple of int pairwise-t threshold for scale means."""
+        """Optional 1D ndarray of tuple of int pairwise-t threshold for scale means.
+
+        Represents the pairwise significance for scale means considering the
+        alternative threshold value.
+        """
         if self._alpha_alt is None:
             return None
-        p_vals, t_stats = self._measures.scale_mean_pairwise_indices.values
-        return self._assemble_pw_scale_mean_indices(
-            p_vals, t_stats, self._alpha_alt, self._only_larger
+        return self._remap_indices(
+            self._assemble_marginal(self._measures.scale_mean_pairwise_indices_alt)
         )
 
     @lazyproperty
@@ -726,11 +722,6 @@ def zscores(self):
         """
         return self._assemble_matrix(self._measures.zscores.blocks)
 
-    @lazyproperty
-    def _alpha(self):
-        """float confidence-interval threshold for pairwise-t (sig) tests."""
-        return self._columns_dimension.pairwise_significance_spec.alpha_values[0]
-
     @lazyproperty
     def _alpha_alt(self):
         """Alternate float confidence-interval threshold or None.
@@ -774,88 +765,6 @@ def _assemble_matrix(self, blocks):
         # --- desired output.
         return np.block(blocks)[np.ix_(self._row_order, self._column_order)]
 
-    def _assemble_pw_indices(self, p_vals, t_stats, alpha, only_larger):
-        """Return assembled 2D ndarray ot tuple of pairwise int indices.
-
-        Significance indices are calculated iterating over each ordered pvals and
-        t_stats column and assembled into a 2D array.
-
-        This assembler method generates a 2D array of tuples of integers starting from
-        t_stats and p_vals significance matrices:
-        pvals:
-        [[
-           [0.1  0.2  0.3 ]    pvals_col_0_selected
-           [0.4  0.5  0.6 ]
-         ]
-         [
-           [0.01 0.02 0.3 ]    pvals_col_1_selected
-           [0.02 0.03 0.04]
-         ]]
-
-        t_stats:
-        [[
-           [0.1  0.2  0.3 ]    t_stats_col_0_selected
-           [0.4  0.5  0.6 ]
-         ]
-         [
-           [0.01 0.02 0.3 ]    t_stats_col_1_selected
-           [0.02 0.03 0.04]
-         ]]
-        Each pvals and tstats matrix is used to calculate the 1D array of tuple of int
-        that represents the significance for the selected column:
-        e.g. [(0,),(), (1,2)]
-        Iterating over each significance matrix the method assemble the complete matrix
-        of significant indices:
-        [[(0,), (), (1,2)],
-         [(), (), (0,1]]
-        """
-
-        def pairwise_indices(p_vals, t_stats):
-            """1D ndarray containing tuples of int pairwise indices of each column."""
-            significance = p_vals < alpha
-            if only_larger:
-                significance = np.logical_and(t_stats < 0, significance)
-            col_significance = np.empty((len(significance),), dtype=object)
-            col_significance[:] = [
-                tuple(np.where(sig_row)[0]) for sig_row in significance
-            ]
-            return col_significance
-
-        return np.array(
-            [
-                pairwise_indices(
-                    p_vals[i][np.ix_(self._row_order, self._column_order)],
-                    t_stats[i][np.ix_(self._row_order, self._column_order)],
-                )
-                for i in self._column_order
-            ]
-        ).T
-
-    def _assemble_pw_scale_mean_indices(self, p_vals, t_stats, alpha, only_larger):
-        """List of tuples indicating the significance of scale means values
-
-        Considering this output: [(3,), (2, 3), (3,), ()] and scale_means values
-        [26, 30, 21, 11], each element contains a tuple of other element
-        indices that are significantly different from the present element's
-        index in a two-tailed test with alpha=.05 by default. The element at
-        index 0 (26) indicates that it differs significantly only from the
-        element at index 3 (11)
-        """
-
-        def scale_mean_pairwise_indices(pvals, tstats):
-            """Tuple of int pairwise indices of each column."""
-            significance = pvals < alpha
-            if only_larger:
-                significance = np.logical_and(tstats < 0, significance)
-            return tuple(np.where(significance)[0])
-
-        return [
-            scale_mean_pairwise_indices(
-                p_vals[i][self._column_order], t_stats[i][self._column_order]
-            )
-            for i in self._column_order
-        ]
-
     def _assemble_vector(self, base_vector, subtotals, order, diffs_nan=False):
         """Return 1D ndarray of `base_vector` with inserted `subtotals`, in `order`.
 
@@ -928,10 +837,48 @@ def _measures(self):
         """SecondOrderMeasures collection object for this cube-result."""
         return SecondOrderMeasures(self._cube, self._dimensions, self._slice_idx)
 
-    @lazyproperty
-    def _only_larger(self):
-        """True if only the larger of reciprocal pairwise-t values should appear."""
-        return self._columns_dimension.pairwise_significance_spec.only_larger
+    def _remap_indices(self, indices_matrix):
+        """Remap pairwise indices from payload order to sort order.
+
+        Returns the significance indices of columns shifted according to the final
+        column order and the eventual column insertions positions.
+
+        Considering a column_order == (0, 1, -1, 2, 3) with negative indices mapped to
+        payload position of insertions
+
+        The column-order mapped to the payload positions will be (0, 1, 4, 2, 3) with
+        (0, 1, 2, 3, 4*) as related sorted position (4* is a col insertion).
+
+        So, (1, 3, 4) should map to (1, 4, 2)
+        """
+        n_col = len(self._columns_dimension.element_ids) + len(
+            self._columns_dimension.insertion_ids
+        )
+        col_order = tuple((n_col + idx) % n_col for idx in self._column_order)
+        # --- Mapping between payload-order indices and sorted positions (indices):
+        # ---
+        # --- {
+        # ---     0: 0,
+        # ---     1: 1,
+        # ---     2: 3,
+        # ---     3: 4,
+        # ---     4: 2  # (-1)
+        # --- }
+        order_map = {k: v for v, k in enumerate(col_order)}
+        remapped_indices = np.array(
+            [
+                tuple(
+                    sorted([order_map[idx] for idx in idxs if idx in order_map.keys()])
+                )
+                for idxs in indices_matrix.ravel()
+            ],
+            dtype=object,
+        )
+        return (
+            remapped_indices.reshape(indices_matrix.shape)
+            if remapped_indices.size > 0
+            else indices_matrix
+        )
 
     @lazyproperty
     def _row_order(self):