Merge pull request #745 from DashAISoftware/develop

0.9.6

Author: cristian-tamblay

.github/workflows/publish.yml

@@ -251,7 +251,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install build python-appimage
           sudo apt-get update
-          sudo apt-get install -y libfuse2 imagemagick
+          sudo apt-get install -y libfuse2 imagemagick librsvg2-bin
       - name: Build wheel (frontend bundled)
         run: python -m build --wheel
       - name: Verify frontend is included in wheel
@@ -262,8 +262,11 @@ jobs:
           fi
       - name: Prepare AppImage recipe
         run: |
-          # Icon referenced by dashai.desktop (Icon=dashai); take the first frame of the .ico
-          convert installer/dashAI.ico[0] appimage/dashai.png
+          # Icon referenced by dashai.desktop (Icon=dashai). Rasterize the
+          # scalable SVG isotype to a large square PNG so desktops have a
+          # high-resolution icon (the .ico only carried small frames).
+          rsvg-convert -h 512 DashAI/front/public/dashai-isotype.svg -o /tmp/dashai-isotype.png
+          convert /tmp/dashai-isotype.png -background none -gravity center -extent 512x512 appimage/dashai.png
           # Append the freshly built wheel (with bundled frontend) to the recipe requirements
           WHEEL=$(ls "$PWD"/dist/*.whl | head -n1)
           echo "$WHEEL" >> appimage/requirements.txt

DashAI/back/converters/base_converter.py

@@ -72,6 +72,9 @@ def get_metadata(cls) -> Dict[str, Any]:
         meta["color"] = cls.COLOR if cls.COLOR else "rgb(255, 255, 255)"
         meta["supervised"] = cls.SUPERVISED
         meta["changes_row_count"] = cls.CHANGES_ROW_COUNT
+        meta["n_components_features_bounded"] = getattr(
+            cls, "N_COMPONENTS_FEATURES_BOUNDED", False
+        )
 
         # Serialize allowed_types class references → class name strings for the frontend
         raw_types = meta.get("allowed_types", [])

DashAI/back/converters/category/dimensionality_reduction.py

@@ -26,3 +26,4 @@ class DimensionalityReductionConverter(BaseConverter):
     )
     ICON: Final[str] = Icon.Layers.value
     COLOR: Final[str] = "rgb(255, 99, 132)"
+    N_COMPONENTS_FEATURES_BOUNDED: bool = True

DashAI/back/converters/category/feature_selection.py

@@ -1,8 +1,12 @@
-from typing import Final
+from typing import TYPE_CHECKING, Final, Union
 
 from DashAI.back.converters.base_converter import BaseConverter
 from DashAI.back.core.utils import MultilingualString
 from DashAI.back.static.icons import Icon
+from DashAI.back.types.dashai_data_type import DashAIDataType
+
+if TYPE_CHECKING:
+    from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset
 
 
 class FeatureSelectionConverter(BaseConverter):
@@ -15,6 +19,10 @@ class FeatureSelectionConverter(BaseConverter):
 
     Use these converters to reduce overfitting, speed up training, and improve
     model interpretability by retaining only the most informative features.
+
+    These converters only drop columns; the retained columns keep their
+    original values untouched, so their data types must be preserved instead of
+    being coerced to float.
     """
 
     CATEGORY = MultilingualString(
@@ -26,3 +34,57 @@ class FeatureSelectionConverter(BaseConverter):
     )
     ICON: Final[str] = Icon.FilterList.value
     COLOR: Final[str] = "rgb(255, 206, 86)"
+
+    def fit(
+        self, x: "DashAIDataset", y: Union["DashAIDataset", None] = None
+    ) -> "FeatureSelectionConverter":
+        """Fit the selector while remembering the input column types.
+
+        Feature selection only keeps a subset of the input columns without
+        modifying their values, so the original types are captured here to be
+        returned later by ``get_output_type``. Types are recorded during ``fit``
+        (rather than ``transform``) because scikit-learn auto-wraps ``transform``
+        on subclasses and would coerce its output back to a pandas DataFrame.
+
+        Parameters
+        ----------
+        x : DashAIDataset
+            The input dataset to fit the selector on.
+        y : DashAIDataset, optional
+            Target values for the supervised selectors. Defaults to None.
+
+        Returns
+        -------
+        FeatureSelectionConverter
+            The fitted selector instance (self).
+        """
+        if hasattr(x, "types") and x.types is not None:
+            self._input_types = dict(x.types)
+        return super().fit(x, y)
+
+    def get_output_type(self, column_name: str = None) -> DashAIDataType:
+        """Return the original DashAI data type of a retained column.
+
+        Since feature selection leaves the retained columns' values unchanged,
+        the output type matches the input type of that column.
+
+        Parameters
+        ----------
+        column_name : str, optional
+            The name of the retained column. Defaults to None.
+
+        Returns
+        -------
+        DashAIDataType
+            The original type of the column. Falls back to ``float64`` when the
+            input type is unknown (feature selectors only operate on numbers).
+        """
+        input_types = getattr(self, "_input_types", None)
+        if input_types is not None and column_name in input_types:
+            return input_types[column_name]
+
+        import pyarrow as pa
+
+        from DashAI.back.types.value_types import Float
+
+        return Float(arrow_type=pa.float64())

DashAI/back/converters/hugging_face/embedding.py

@@ -247,4 +247,9 @@ def _process_batch(self, batch: "DashAIDataset") -> "DashAIDataset":
                     pa.array(embeddings_np[:, i].tolist(), type=pa.float32()),
                 )
 
+        # Remove original text columns — they are replaced by their emb_* counterparts
+        for column in batch.column_names:
+            col_idx = result_table.column_names.index(column)
+            result_table = result_table.remove_column(col_idx)
+
         return DashAIDataset(result_table)

DashAI/back/converters/hugging_face/tokenizer.py

@@ -185,6 +185,11 @@ def _process_batch(self, batch: "DashAIDataset") -> "DashAIDataset":
                     pa.array(input_ids[:, i].tolist(), type=pa.int64()),
                 )
 
+        # Remove original text columns — they are replaced by their tok_* counterparts
+        for column in batch.column_names:
+            col_idx = result_table.column_names.index(column)
+            result_table = result_table.remove_column(col_idx)
+
         return DashAIDataset(result_table)
 
     def get_output_type(self, column_name: Optional[str] = None) -> DashAIDataType:

DashAI/back/converters/scikit_learn/additive_chi_2_sampler.py

@@ -78,7 +78,7 @@ class AdditiveChi2Sampler(
     )
     DISPLAY_NAME = MultilingualString(
         en="Additive Chi² Sampler",
-        es="Muestreador Chi²",
+        es="Muestreador Chi² Aditivo",
         pt="Amostrador Qui-2 Aditivo",
         de="Additiver Chi²-Stichprobennehmer",
         zh="加性卡方采样器",

DashAI/back/converters/scikit_learn/fast_ica.py

@@ -208,6 +208,7 @@ class FastICA(DimensionalityReductionConverter, SklearnWrapper, FastICAOperation
     """
 
     SCHEMA = FastICASchema
+    N_COMPONENTS_FEATURES_BOUNDED: bool = False
     DESCRIPTION = MultilingualString(
         en="FastICA: a fast algorithm for Independent Component Analysis.",
         es=(

DashAI/back/converters/scikit_learn/generic_univariate_select.py

@@ -14,7 +14,6 @@
 )
 from DashAI.back.core.schema_fields.base_schema import BaseSchema
 from DashAI.back.core.utils import MultilingualString
-from DashAI.back.types.dashai_data_type import DashAIDataType
 from DashAI.back.types.value_types import Float, Integer
 
 
@@ -83,21 +82,3 @@ class GenericUnivariateSelect(
     )
     IMAGE_PREVIEW = "generic_univariate_select.png"
     metadata = {"allowed_types": [Float, Integer], "allowed_dtypes": []}
-
-    def get_output_type(self, column_name: str = None) -> DashAIDataType:
-        """Return the DashAI data type produced by this converter for a column.
-
-        Parameters
-        ----------
-        column_name : str, optional
-            Not used; all output columns share the
-            same type. Defaults to None.
-
-        Returns
-        -------
-        DashAIDataType
-            A Float type backed by ``pyarrow.float64()``.
-        """
-        import pyarrow as pa
-
-        return Float(arrow_type=pa.float64())

DashAI/back/converters/scikit_learn/missing_indicator.py

@@ -1,3 +1,5 @@
+from typing import TYPE_CHECKING, Union
+
 from sklearn.impute import MissingIndicator as MissingIndicatorOperation
 
 from DashAI.back.converters.category.basic_preprocessing import (
@@ -9,6 +11,11 @@
 from DashAI.back.types.dashai_data_type import DashAIDataType
 from DashAI.back.types.value_types import Integer
 
+if TYPE_CHECKING:
+    import pandas as pd
+
+    from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset
+
 
 class MissingIndicatorSchema(BaseSchema):
     """Schema for configuring the MissingIndicator converter.
@@ -75,7 +82,122 @@ def __init__(self, **kwargs):
             Configuration keyword arguments matching the converter's
             schema fields. Forwarded to the underlying scikit-learn class.
         """
+        # Force indicators for all selected features so the user always sees the
+        # new column, even when a feature has no missing values (all-False indicator).
+        kwargs.setdefault("features", "all")
         super().__init__(**kwargs)
+        # SklearnWrapper.__init__ sets set_output(transform="pandas"), which causes
+        # sklearn's __init_subclass__ wrapper to intercept our custom transform and
+        # attempt to rename its output using get_feature_names_out() (which returns
+        # only the indicator column count, not the combined output count).
+        # Reset to "default" so the wrapper returns our DashAIDataset as-is.
+        if hasattr(self, "set_output"):
+            self.set_output(transform="default")
+
+    @staticmethod
+    def _normalize_missing(frame: "pd.DataFrame") -> "pd.DataFrame":
+        """Return a copy of *frame* where object-column missing values are float NaN.
+
+        HuggingFace/PyArrow stores missing strings as ``None`` (Python), but
+        sklearn's ``_get_mask`` uses ``x != x`` which is ``False`` for ``None``
+        (only ``float('nan') != float('nan')`` is ``True``).  We also treat
+        empty strings as missing to match the dataset-filter behaviour.
+        """
+        import numpy as np
+
+        frame = frame.copy()
+        for col in frame.select_dtypes(include="object").columns:
+            frame[col] = frame[col].replace("", np.nan)
+            frame[col] = frame[col].where(frame[col].notna(), np.nan)
+        return frame
+
+    def fit(
+        self, x: "DashAIDataset", y: Union["DashAIDataset", None] = None
+    ) -> "MissingIndicator":
+        """Fit after normalising missing values so sklearn detects them."""
+        from DashAI.back.dataloaders.classes.dashai_dataset import to_dashai_dataset
+
+        x_pandas = x.to_pandas() if hasattr(x, "to_pandas") else x
+        x_clean_ds = to_dashai_dataset(self._normalize_missing(x_pandas))
+        if hasattr(x, "types"):
+            x_clean_ds.types = x.types.copy()
+        return super().fit(x_clean_ds, y)
+
+    def transform(
+        self, x: "DashAIDataset", y: Union["DashAIDataset", None] = None
+    ) -> "DashAIDataset":
+        """Transform x by appending missing-value indicator columns.
+
+        Keeps the original columns intact and appends one boolean indicator
+        column per feature that had missing values during fit.  Indicator
+        columns are named ``missingindicator_<original_col_name>`` so that
+        ``_rebuild_dataset_with_transformed_columns`` treats them as *new*
+        columns rather than replacements, preserving the original data.
+        """
+        import numpy as np
+        import pandas as pd
+
+        from DashAI.back.dataloaders.classes.dashai_dataset import to_dashai_dataset
+
+        x_pandas = x.to_pandas() if hasattr(x, "to_pandas") else x
+
+        # Normalise missing values before sklearn sees the data (None and ""
+        # are both treated as missing, matching dataset-filter behaviour).
+        x_for_sklearn = self._normalize_missing(x_pandas)
+
+        sklearn_cls = next(
+            (
+                cls
+                for cls in type(self).__mro__
+                if "sklearn" in cls.__module__
+                and "DashAI" not in cls.__module__
+                and "transform" in cls.__dict__
+            ),
+            None,
+        )
+        if sklearn_cls is None:
+            raise RuntimeError(
+                "No sklearn class with a 'transform' method found in the MRO."
+            )
+
+        indicators = sklearn_cls.__dict__["transform"](self, x_for_sklearn)
+
+        # features_ contains the column indices for which indicators are produced.
+        # With features='all' (default), this always equals all input column indices.
+        if hasattr(self, "features_") and len(self.features_) > 0:
+            indicator_col_names = [
+                f"missingindicator_{x_pandas.columns[i]}" for i in self.features_
+            ]
+        else:
+            indicator_col_names = [
+                f"missingindicator_{col}" for col in x_pandas.columns
+            ]
+
+        if isinstance(indicators, np.ndarray):
+            indicators_df = pd.DataFrame(
+                indicators,
+                columns=indicator_col_names,
+                index=x_pandas.index,
+            )
+        else:
+            indicators_df = indicators.copy()
+            indicators_df.columns = indicator_col_names
+
+        combined_df = pd.concat([x_pandas, indicators_df], axis=1)
+        converted_dataset = to_dashai_dataset(combined_df)
+
+        output_type = self.get_output_type()
+        for col in indicator_col_names:
+            if col in converted_dataset.column_names:
+                converted_dataset.types[col] = output_type
+
+        # Preserve original column types from the input dataset
+        if hasattr(x, "types"):
+            for col in x_pandas.columns:
+                if col in x.types and col in converted_dataset.column_names:
+                    converted_dataset.types[col] = x.types[col]
+
+        return converted_dataset
 
     def get_output_type(self, column_name: str = None) -> DashAIDataType:
         """Return the DashAI data type produced by this converter for a column.