diff --git a/DashAI/back/exploration/complexity_measures.py b/DashAI/back/exploration/complexity_measures.py new file mode 100644 index 000000000..d618739a0 --- /dev/null +++ b/DashAI/back/exploration/complexity_measures.py @@ -0,0 +1,391 @@ +"""Geometrical complexity measures for classification datasets. + +This module implements a subset of the measures surveyed in Lorena et al. +(2019), "How Complex Is Your Classification Problem? A Survey on Measuring +Classification Complexity". They describe how hard a classification problem is +from the geometry of the data alone, without training any classifier. + +Three measures are provided: + +``F1`` + Maximum Fisher discriminant ratio. Feature-based: looks for the single most + discriminative feature. +``N1`` + Fraction of borderline points, obtained from a minimum spanning tree. + Neighbourhood-based: measures the size of the class boundary. +``N2`` + Ratio of intra-class to extra-class nearest neighbour distances. + Neighbourhood-based: measures how tight classes are relative to their + separation. + +All three are normalised to ``[0, 1]`` where **lower means easier to +separate**, following the convention used in the survey. + +This module intentionally has no DashAI imports. The measures are plain +functions over ``(x, y)`` arrays so they can be reused unchanged if the project +later introduces a dedicated data-metric component type. +""" + +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +if TYPE_CHECKING: + from numpy import ndarray + +# Measures computed by :func:`compute_class_overlap`, in reporting order. +MEASURE_NAMES: Tuple[str, ...] = ("F1", "N1", "N2") + +# Above this many rows the pairwise distance matrix needed by N1 and N2 stops +# being practical, so the sample is reduced before computing them. +DEFAULT_MAX_SAMPLES: int = 2000 + + +def _validate(x: "ndarray", y: "ndarray") -> None: + """Check that the arrays can support a complexity measure. + + Parameters + ---------- + x : ndarray + Feature matrix of shape ``(n_samples, n_features)``. + y : ndarray + Class labels of shape ``(n_samples,)``. + + Raises + ------ + ValueError + If the shapes disagree, the matrix is empty, or fewer than two classes + are present. + """ + import numpy as np + + if x.ndim != 2: + raise ValueError(f"x must be two-dimensional, got shape {x.shape}.") + if len(x) != len(y): + raise ValueError( + "x and y must have the same number of rows, given: " + f"len(x) = {len(x)} and len(y) = {len(y)}." + ) + if len(x) == 0: + raise ValueError("x is empty; no complexity measure can be computed.") + if len(np.unique(y)) < 2: + raise ValueError( + "At least two classes are required to measure class overlap, found " + f"{len(np.unique(y))}." + ) + + +def _drop_missing(x: "ndarray", y: "ndarray") -> Tuple["ndarray", "ndarray", int]: + """Remove rows holding a missing value in any feature or in the label. + + Parameters + ---------- + x : ndarray + Feature matrix. + y : ndarray + Class labels. + + Returns + ------- + Tuple[ndarray, ndarray, int] + The filtered matrix, the filtered labels, and the number of rows that + were dropped. + """ + import numpy as np + + finite = np.isfinite(x).all(axis=1) + labelled = np.array([value is not None and value == value for value in y]) + keep = finite & labelled + return x[keep], y[keep], int((~keep).sum()) + + +def _min_max_scale(x: "ndarray") -> "ndarray": + """Scale every feature to ``[0, 1]``. + + Distance-based measures are not scale invariant, so features are put on a + common range before N1 and N2 are computed. Constant features collapse to + zero and therefore stop contributing to the distances. + + Parameters + ---------- + x : ndarray + Feature matrix. + + Returns + ------- + ndarray + The scaled matrix, as float. + """ + x = x.astype(float, copy=True) + minimum = x.min(axis=0) + spread = x.max(axis=0) - minimum + spread[spread == 0] = 1.0 + return (x - minimum) / spread + + +def _stratified_subsample( + x: "ndarray", y: "ndarray", max_samples: int, random_state: Optional[int] +) -> Tuple["ndarray", "ndarray"]: + """Reduce the sample while keeping the class proportions. + + Every class keeps at least two members whenever it had them, so that the + intra-class nearest neighbour used by N2 still exists after subsampling. + + Parameters + ---------- + x : ndarray + Feature matrix. + y : ndarray + Class labels. + max_samples : int + Target number of rows. The result may be slightly larger when the + per-class minimum forces it. + random_state : Optional[int] + Seed for the selection, or ``None`` for a non-deterministic draw. + + Returns + ------- + Tuple[ndarray, ndarray] + The reduced matrix and labels. + """ + import numpy as np + + if len(x) <= max_samples: + return x, y + + rng = np.random.default_rng(random_state) + keep: List[int] = [] + classes, counts = np.unique(y, return_counts=True) + ratio = max_samples / len(x) + + for label, count in zip(classes, counts, strict=True): + indices = np.flatnonzero(y == label) + quota = max(min(2, int(count)), int(round(int(count) * ratio))) + keep.extend(rng.choice(indices, size=quota, replace=False).tolist()) + + keep_array = np.array(sorted(keep)) + return x[keep_array], y[keep_array] + + +def fisher_discriminant_ratio(x: "ndarray", y: "ndarray") -> float: + """Compute F1, the maximum Fisher discriminant ratio. + + For each feature the ratio contrasts the separation between class means + against the spread inside the classes, weighted by class size. The measure + keeps the largest ratio over all features, so it answers "is there at least + one feature that separates these classes on its own?". + + The value is ``1 / (1 + max_ratio)``: it approaches 0 when some feature + separates the classes cleanly and approaches 1 when no single feature does. + + Parameters + ---------- + x : ndarray + Feature matrix of shape ``(n_samples, n_features)``. + y : ndarray + Class labels of shape ``(n_samples,)``. + + Returns + ------- + float + F1 in ``[0, 1]``. Lower means easier to separate. + """ + import numpy as np + + _validate(x, y) + x = x.astype(float, copy=False) + classes = np.unique(y) + + numerator = np.zeros(x.shape[1], dtype=float) + denominator = np.zeros(x.shape[1], dtype=float) + + means = {} + sizes = {} + for label in classes: + members = x[y == label] + means[label] = members.mean(axis=0) + sizes[label] = len(members) + denominator += ((members - means[label]) ** 2).sum(axis=0) + + for position, first in enumerate(classes): + for second in classes[position + 1 :]: + weight = sizes[first] * sizes[second] + numerator += weight * (means[first] - means[second]) ** 2 + + # A zero denominator means the feature has no within-class spread at all. + # If its class means still differ the feature separates perfectly, which is + # an infinite ratio; if they do not, the feature carries no information. + safe_denominator = np.where(denominator > 0, denominator, 1.0) + ratios = np.where( + denominator > 0, + numerator / safe_denominator, + np.where(numerator > 0, np.inf, 0.0), + ) + + best = float(np.max(ratios)) + if np.isinf(best): + return 0.0 + return float(1.0 / (1.0 + best)) + + +def borderline_points(x: "ndarray", y: "ndarray") -> float: + """Compute N1, the fraction of points on a class boundary. + + A minimum spanning tree is built over the points using Euclidean distance. + Every edge that joins two different classes marks both of its endpoints as + borderline. N1 is the share of points marked this way, so it estimates how + long and how populated the boundary between classes is. + + Parameters + ---------- + x : ndarray + Feature matrix, already scaled. + y : ndarray + Class labels. + + Returns + ------- + float + N1 in ``[0, 1]``. Lower means a smaller boundary, so easier to + separate. + """ + import numpy as np + from scipy.sparse.csgraph import minimum_spanning_tree + from scipy.spatial.distance import pdist, squareform + + _validate(x, y) + distances = squareform(pdist(x.astype(float, copy=False))) + + # minimum_spanning_tree reads a zero as "no edge", so coincident points + # would lose their edge. Nudging those zeros keeps the tree connected. + off_diagonal = ~np.eye(len(x), dtype=bool) + positive = distances[off_diagonal & (distances > 0)] + epsilon = float(positive.min()) * 1e-6 if positive.size else 1e-12 + distances[off_diagonal & (distances == 0)] = epsilon + + tree = minimum_spanning_tree(distances).tocoo() + borderline = set() + for source, target in zip(tree.row, tree.col, strict=True): + if y[source] != y[target]: + borderline.add(int(source)) + borderline.add(int(target)) + + return float(len(borderline) / len(x)) + + +def intra_extra_nn_ratio(x: "ndarray", y: "ndarray") -> float: + """Compute N2, the intra-class over extra-class nearest neighbour ratio. + + For every point the distance to its closest same-class neighbour is + compared against the distance to its closest different-class neighbour. + Summing both over the dataset gives a ratio that is small when classes are + tight and far apart, and large when they interleave. + + The result is ``r / (1 + r)`` so that it stays in ``[0, 1]``. + + Points belonging to a class with a single member are skipped, since they + have no same-class neighbour to measure against. + + Parameters + ---------- + x : ndarray + Feature matrix, already scaled. + y : ndarray + Class labels. + + Returns + ------- + float + N2 in ``[0, 1]``. Lower means easier to separate. + + Raises + ------ + ValueError + If no point has both a same-class and a different-class neighbour. + """ + import numpy as np + from scipy.spatial.distance import pdist, squareform + + _validate(x, y) + distances = squareform(pdist(x.astype(float, copy=False))) + np.fill_diagonal(distances, np.inf) + + intra_total = 0.0 + extra_total = 0.0 + measured = 0 + + for index in range(len(x)): + same = y == y[index] + same[index] = False + if not same.any(): + continue + intra_total += float(distances[index][same].min()) + extra_total += float(distances[index][~same].min()) + measured += 1 + + if measured == 0: + raise ValueError( + "No point has both a same-class and a different-class neighbour, " + "so N2 cannot be computed." + ) + if extra_total == 0: + return 1.0 + + ratio = intra_total / extra_total + return float(ratio / (1.0 + ratio)) + + +def compute_class_overlap( + x: "ndarray", + y: "ndarray", + max_samples: int = DEFAULT_MAX_SAMPLES, + random_state: Optional[int] = None, +) -> Dict[str, Any]: + """Compute F1, N1 and N2 for a labelled dataset. + + Rows with a missing feature or label are dropped, features are scaled to + ``[0, 1]`` for the two distance-based measures, and the sample is reduced + when it exceeds ``max_samples``. + + Parameters + ---------- + x : ndarray + Numeric feature matrix of shape ``(n_samples, n_features)``. + y : ndarray + Class labels of shape ``(n_samples,)``. + max_samples : int + Row budget for N1 and N2, which both need a pairwise distance matrix. + F1 always uses every available row, since it is linear in the sample. + random_state : Optional[int] + Seed used when the sample has to be reduced. + + Returns + ------- + Dict[str, Any] + Dictionary with the keys ``"measures"`` (one float per measure name), + ``"n_samples"`` (rows kept after dropping missing values), + ``"n_features"``, ``"n_classes"``, ``"n_dropped_rows"`` and + ``"n_samples_used_for_distances"``. + """ + import numpy as np + + x = np.asarray(x, dtype=float) + y = np.asarray(y) + + _validate(x, y) + x, y, dropped = _drop_missing(x, y) + _validate(x, y) + + scaled = _min_max_scale(x) + reduced_x, reduced_y = _stratified_subsample(scaled, y, max_samples, random_state) + + return { + "measures": { + "F1": fisher_discriminant_ratio(x, y), + "N1": borderline_points(reduced_x, reduced_y), + "N2": intra_extra_nn_ratio(reduced_x, reduced_y), + }, + "n_samples": int(len(x)), + "n_features": int(x.shape[1]), + "n_classes": int(len(np.unique(y))), + "n_dropped_rows": dropped, + "n_samples_used_for_distances": int(len(reduced_x)), + } diff --git a/DashAI/back/exploration/data_complexity_explorer.py b/DashAI/back/exploration/data_complexity_explorer.py new file mode 100644 index 000000000..cb45edc3f --- /dev/null +++ b/DashAI/back/exploration/data_complexity_explorer.py @@ -0,0 +1,29 @@ +from typing import Final + +from DashAI.back.core.utils import MultilingualString +from DashAI.back.exploration.base_explorer import BaseExplorer +from DashAI.back.static.icons import Icon + + +class DataComplexityExplorer(BaseExplorer): + """Base class for explorers that measure how hard a dataset is to learn. + + Complexity explorers describe the geometry of a supervised problem without + training any model: how much the classes overlap, how large the boundary + between them is, how tight each class is relative to its neighbours. They + answer a question that comes before model evaluation, namely whether the + data supports the task at all. + + Subclass this and implement `launch_exploration`, `save_notebook`, and + `get_results` to create a new complexity explorer. + """ + + CATEGORY: Final[str] = MultilingualString( + en="Data Complexity", + es="Complejidad de los Datos", + pt="Complexidade dos Dados", + de="Datenkomplexität", + zh="数据复杂度", + ) + ICON: Final[str] = Icon.Layers.value + COLOR: Final[str] = "rgb(230, 126, 34)" diff --git a/DashAI/back/exploration/explorers/class_overlap.py b/DashAI/back/exploration/explorers/class_overlap.py new file mode 100644 index 000000000..5d511776e --- /dev/null +++ b/DashAI/back/exploration/explorers/class_overlap.py @@ -0,0 +1,447 @@ +from typing import TYPE_CHECKING, Any, Dict, List + +from DashAI.back.core.schema_fields import ( + int_field, + none_type, + schema_field, + string_field, + union_type, +) +from DashAI.back.core.utils import MultilingualString +from DashAI.back.dependencies.database.models import Explorer, Notebook +from DashAI.back.exploration.base_explorer import ( + NON_NUMERIC_DTYPES, + BaseExplorerSchema, +) +from DashAI.back.exploration.complexity_measures import ( + DEFAULT_MAX_SAMPLES, + compute_class_overlap, +) +from DashAI.back.exploration.data_complexity_explorer import DataComplexityExplorer +from DashAI.back.types.categorical import Categorical +from DashAI.back.types.value_types import Float, Integer + +if TYPE_CHECKING: + from pathlib import Path + + from DashAI.back.dataloaders.classes.dashai_dataset import DashAIDataset + +# A target with more distinct values than this is almost certainly continuous, +# and the measures are only defined for discrete classes. +MAX_CLASSES = 50 + + +class ClassOverlapExplorerSchema(BaseExplorerSchema): + """Schema for ClassOverlapExplorer configuration. + + ``target_column`` names the column holding the class labels; it is + mandatory, since the measures are supervised. The remaining selected + columns are treated as the numeric features that describe each point. + + N1 and N2 both need a pairwise distance matrix, which grows with the square + of the sample. ``max_samples`` caps the number of rows used for those two + measures, drawing a class-proportional subsample when the dataset is + larger; ``random_state`` makes that draw reproducible. F1 is linear in the + sample and always uses every available row. + """ + + target_column: schema_field( + union_type(string_field(), int_field(ge=0)), + "", + description=MultilingualString( + en=( + "Required. Name or index of the column holding the class " + "labels. It is added to the exploration automatically, so it " + "does not need to be selected as a column." + ), + es=( + "Obligatorio. Nombre o índice de la columna que contiene las " + "clases. Se agrega a la exploración automáticamente, así que " + "no hace falta seleccionarla como columna." + ), + pt=( + "Obrigatório. Nome ou índice da coluna que contém as classes. " + "É adicionada à exploração automaticamente, portanto não " + "precisa ser selecionada como coluna." + ), + de=( + "Erforderlich. Name oder Index der Spalte mit den " + "Klassenbezeichnungen. Sie wird der Exploration automatisch " + "hinzugefügt und muss nicht als Spalte ausgewählt werden." + ), + zh=("必填。包含类别标签的列名或索引。该列会自动加入探索，无需作为列选中。"), + ), + alias=MultilingualString( + en="Target column", + es="Columna objetivo", + pt="Coluna alvo", + de="Zielspalte", + zh="目标列", + ), + ) # type: ignore + max_samples: schema_field( + int_field(gt=1), + DEFAULT_MAX_SAMPLES, + description=MultilingualString( + en=( + "Maximum number of rows used for the distance-based measures " + "N1 and N2. Larger datasets are subsampled keeping the class " + "proportions." + ), + es=( + "Número máximo de filas usadas en las medidas basadas en " + "distancia N1 y N2. Los datasets más grandes se submuestrean " + "manteniendo las proporciones de clase." + ), + pt=( + "Número máximo de linhas usadas nas medidas baseadas em " + "distância N1 e N2. Conjuntos maiores são subamostrados " + "mantendo as proporções das classes." + ), + de=( + "Maximale Anzahl Zeilen für die distanzbasierten Maße N1 und " + "N2. Größere Datensätze werden unter Beibehaltung der " + "Klassenanteile unterabgetastet." + ), + zh="用于基于距离的度量N1和N2的最大行数。更大的数据集将按类别比例进行子采样。", + ), + alias=MultilingualString( + en="Maximum samples", + es="Muestras máximas", + pt="Amostras máximas", + de="Maximale Stichproben", + zh="最大样本数", + ), + ) # type: ignore + random_state: schema_field( + none_type(int_field(ge=0)), + 0, + description=MultilingualString( + en=("Seed used when the dataset has to be subsampled."), + es=("Semilla usada cuando el dataset debe ser submuestreado."), + pt=("Semente usada quando o conjunto precisa ser subamostrado."), + de=("Startwert für die Unterabtastung des Datensatzes."), + zh="数据集需要子采样时使用的随机种子。", + ), + alias=MultilingualString( + en="Random state", + es="Semilla aleatoria", + pt="Semente aleatória", + de="Zufallsstartwert", + zh="随机种子", + ), + ) # type: ignore + + +class ClassOverlapExplorer(DataComplexityExplorer): + """Explorer that measures how much the classes of a dataset overlap. + + It reports three of the geometrical complexity measures surveyed in Lorena + et al. (2019), computed from the data alone, with no model involved: + + ``F1`` + Maximum Fisher discriminant ratio. Near 0 when at least one feature + separates the classes on its own, near 1 when none does. + ``N1`` + Fraction of points sitting on a class boundary, taken from a minimum + spanning tree over the sample. A large value means a long, populated + frontier between classes. + ``N2`` + Ratio between the distance to the nearest same-class neighbour and the + distance to the nearest different-class neighbour. A large value means + the classes interleave rather than forming compact groups. + + All three are normalised so that **lower means easier to separate**. + + Use this explorer before training to tell an unpromising dataset from an + unpromising model: if the classes already overlap heavily, a disappointing + accuracy is a property of the data rather than of the learner, and the + useful next step is feature engineering or relabelling rather than model + search. + """ + + DISPLAY_NAME = MultilingualString( + en="Class Overlap", + es="Solapamiento de Clases", + pt="Sobreposição de Classes", + de="Klassenüberlappung", + zh="类别重叠", + ) + DESCRIPTION = MultilingualString( + en=( + "Measures how much the classes overlap using the F1, N1 and N2 " + "complexity measures. Lower values mean the classes are easier to " + "separate. No model is trained." + ), + es=( + "Mide cuánto se solapan las clases usando las medidas de " + "complejidad F1, N1 y N2. Valores más bajos indican clases más " + "fáciles de separar. No se entrena ningún modelo." + ), + pt=( + "Mede o quanto as classes se sobrepõem usando as medidas de " + "complexidade F1, N1 e N2. Valores menores indicam classes mais " + "fáceis de separar. Nenhum modelo é treinado." + ), + de=( + "Misst die Überlappung der Klassen mit den Komplexitätsmaßen F1, " + "N1 und N2. Niedrigere Werte bedeuten leichter trennbare Klassen. " + "Es wird kein Modell trainiert." + ), + zh=( + "使用F1、N1和N2复杂度度量衡量类别重叠程度。数值越低表示类别越容易分离。" + "不训练任何模型。" + ), + ) + + SCHEMA = ClassOverlapExplorerSchema + metadata: Dict[str, Any] = { + "allowed_types": [Float, Integer, Categorical], + "allowed_dtypes": [], + "non_allowed_dtypes": NON_NUMERIC_DTYPES, + "input_cardinality": {"min": 1}, + } + + def __init__(self, **kwargs) -> None: + """Initialize ClassOverlapExplorer with its measurement parameters. + + Parameters + ---------- + **kwargs + Keyword arguments matching ``ClassOverlapExplorerSchema`` fields: + target_column (str | int): Name or index of the label column. + max_samples (int): Row budget for the distance-based measures. + random_state (int | None): Seed used when subsampling. + """ + self.target_column = kwargs.get("target_column") + self.max_samples = kwargs.get("max_samples", DEFAULT_MAX_SAMPLES) + self.random_state = kwargs.get("random_state") + super().__init__(**kwargs) + + def prepare_dataset( + self, loaded_dataset: "DashAIDataset", columns: List[Dict[str, Any]] + ) -> "DashAIDataset": + """Extend column selection to include the target column. + + The target is configured through the schema rather than picked as one + of the explored columns, so it has to be appended to the selection + before the dataset is narrowed down. + + Parameters + ---------- + loaded_dataset : DashAIDataset + The full dataset. + columns : List[Dict[str, Any]] + Explicitly selected column descriptors. + + Returns + ------- + DashAIDataset + Dataset containing the selected feature columns plus the target. + + Raises + ------ + ValueError + If no target column was configured. + """ + if self.target_column is None or self.target_column == "": + raise ValueError( + "A target column is required to measure class overlap. Set it " + "in the explorer parameters to the name or index of the column " + "holding the class labels." + ) + + explorer_columns = [col["columnName"] for col in columns] + dataset_columns = loaded_dataset.column_names + + if isinstance(self.target_column, int): + index = self.target_column + if index >= len(dataset_columns): + raise ValueError( + f"Target column index {index} is out of range for a " + f"dataset with {len(dataset_columns)} columns." + ) + name = dataset_columns[index] + if name not in explorer_columns: + columns.append({"id": index, "columnName": name}) + else: + name = self.target_column + if name not in dataset_columns: + raise ValueError(f"Target column '{name}' is not in the dataset.") + if name not in explorer_columns: + columns.append({"columnName": name}) + + self.target_column = name + return super().prepare_dataset(loaded_dataset, columns) + + def launch_exploration( + self, dataset: "DashAIDataset", __explorer_info__: Explorer + ) -> Any: + """Compute the complexity measures and lay them out as a table. + + Parameters + ---------- + dataset : DashAIDataset + Dataset holding the feature columns and the target column. + __explorer_info__ : Explorer + The explorer database record (unused). + + Returns + ------- + Any + A ``pandas.DataFrame`` indexed by row label, with a ``"value"`` + column and a ``"detail"`` column describing each entry. + + Raises + ------ + ValueError + If the target is missing, holds too many distinct values to be a + class label, or leaves no numeric feature column behind. + """ + import pandas as pd + + frame = dataset.to_pandas() + + if self.target_column not in frame.columns: + raise ValueError( + f"Target column '{self.target_column}' is not in the prepared dataset." + ) + + labels = frame[self.target_column] + distinct = labels.nunique(dropna=True) + if distinct > MAX_CLASSES: + raise ValueError( + f"The target column '{self.target_column}' has {distinct} " + f"distinct values, which exceeds the limit of {MAX_CLASSES}. " + "These measures apply to classification targets, so the target " + "should be the column holding the class labels, not a feature. " + "Note that an index refers to the position of the column in " + "the dataset, not among the columns selected for the explorer." + ) + + features = frame.drop(columns=[self.target_column]) + features = features.select_dtypes(include="number") + if features.shape[1] == 0: + raise ValueError( + "No numeric feature column is left after removing the target. " + "The distance-based measures need numeric features." + ) + + summary = compute_class_overlap( + features.to_numpy(), + labels.to_numpy(), + max_samples=self.max_samples, + random_state=self.random_state, + ) + + rows = [ + ( + "F1", + round(summary["measures"]["F1"], 6), + "Maximum Fisher discriminant ratio. Lower is easier.", + ), + ( + "N1", + round(summary["measures"]["N1"], 6), + "Fraction of points on a class boundary. Lower is easier.", + ), + ( + "N2", + round(summary["measures"]["N2"], 6), + "Intra over extra class nearest neighbour ratio. Lower is easier.", + ), + ("Samples", summary["n_samples"], "Rows kept after dropping missing."), + ("Features", summary["n_features"], "Numeric feature columns used."), + ("Classes", summary["n_classes"], "Distinct labels in the target."), + ( + "Rows dropped", + summary["n_dropped_rows"], + "Rows removed for holding a missing value.", + ), + ( + "Samples for N1 and N2", + summary["n_samples_used_for_distances"], + "Rows used by the distance-based measures.", + ), + ] + + # The column holds both measures and counts. Building it as object + # keeps the counts as integers; a plain numeric column would upcast + # them and render "10000.0" beside the measures. + return pd.DataFrame( + { + "value": pd.Series( + [value for _, value, _ in rows], + index=[name for name, _, _ in rows], + dtype=object, + ), + "detail": [detail for _, _, detail in rows], + } + ) + + def save_notebook( + self, + __notebook_info__: Notebook, + explorer_info: Explorer, + save_path: "Path", + result: Any, + ) -> str: + """Save the measures table to a JSON file on disk. + + Parameters + ---------- + __notebook_info__ : Notebook + The notebook database record (unused). + explorer_info : Explorer + The explorer record used for filename generation. + save_path : Path + Directory where the file will be saved. + result : Any + The ``pandas.DataFrame`` returned by ``launch_exploration``. + + Returns + ------- + str + The path of the saved JSON file as a POSIX string. + """ + import os + from pathlib import Path + + import pandas as pd + + filename = f"{explorer_info.id}.json" + path = Path(os.path.join(save_path, filename)) + + assert isinstance(result, pd.DataFrame) + result.to_json(path) + return path.as_posix() + + def get_results( + self, exploration_path: str, options: Dict[str, Any] + ) -> Dict[str, Any]: + """Load and return the saved measures table for the frontend. + + Parameters + ---------- + exploration_path : str + Path to the JSON file saved by ``save_notebook``. + options : Dict[str, Any] + Rendering options from the frontend (unused). + + Returns + ------- + Dict[str, Any] + Dictionary with keys ``"data"`` (nested dict of the table, keyed by + column then row, which is the orientation the artifact conversion + expects), ``"type"`` (``"tabular"``) and ``"config"``. + """ + import json + + # Read with json rather than pandas. The table mixes measures with + # counts in one column, and pandas would coerce the whole column to + # float, turning a sample count of 10000 into 10000.0. The file on + # disk already carries the right types. + with open(exploration_path, "r", encoding="utf-8") as file: + columns = json.load(file) + + return {"type": "tabular", "data": columns, "config": {"orient": "dict"}} diff --git a/DashAI/back/initial_components.py b/DashAI/back/initial_components.py index 27bc9ed23..d77ca3d2d 100644 --- a/DashAI/back/initial_components.py +++ b/DashAI/back/initial_components.py @@ -131,6 +131,7 @@ # Explorers from DashAI.back.exploration.explorers.box_plot import BoxPlotExplorer +from DashAI.back.exploration.explorers.class_overlap import ClassOverlapExplorer from DashAI.back.exploration.explorers.corr_matrix import CorrelationMatrixExplorer from DashAI.back.exploration.explorers.cov_matrix import CovarianceMatrixExplorer from DashAI.back.exploration.explorers.density_heatmap import DensityHeatmapExplorer @@ -676,6 +677,7 @@ def get_initial_components(): TimeSeriesPlotExplorer, ParallelCategoriesExplorer, ParallelCordinatesExplorer, + ClassOverlapExplorer, # Converters ColumnRemover, NanRemover, diff --git a/tests/back/exploration/test_complexity_measures.py b/tests/back/exploration/test_complexity_measures.py new file mode 100644 index 000000000..9e05b9c23 --- /dev/null +++ b/tests/back/exploration/test_complexity_measures.py @@ -0,0 +1,221 @@ +import numpy as np +import pytest + +from DashAI.back.exploration.complexity_measures import ( + MEASURE_NAMES, + borderline_points, + compute_class_overlap, + fisher_discriminant_ratio, + intra_extra_nn_ratio, +) + + +def _separable(seed=0, per_class=40): + """Two blobs far enough apart that no point sits on the boundary.""" + rng = np.random.default_rng(seed) + X = np.vstack( + [ + rng.normal(0.0, 0.2, (per_class, 2)), + rng.normal(10.0, 0.2, (per_class, 2)), + ] + ) + y = np.array([0] * per_class + [1] * per_class) + return X, y + + +def _noise(seed=0, samples=80): + """Labels drawn independently of the features, so nothing is learnable.""" + rng = np.random.default_rng(seed) + return rng.normal(0.0, 1.0, (samples, 3)), rng.integers(0, 2, samples) + + +# --- range and direction --- + + +@pytest.mark.parametrize( + "measure", + [fisher_discriminant_ratio, borderline_points, intra_extra_nn_ratio], +) +def test_measures_stay_in_the_unit_interval(measure): + for builder in (_separable, _noise): + X, y = builder() + value = measure(X, y) + assert 0.0 <= value <= 1.0 + + +@pytest.mark.parametrize( + "measure", + [fisher_discriminant_ratio, borderline_points, intra_extra_nn_ratio], +) +def test_separable_data_scores_lower_than_noise(measure): + """Lower must mean easier, which is the convention the whole module uses.""" + easy = measure(*_separable()) + hard = measure(*_noise()) + assert easy < hard + + +# --- F1 --- + + +def test_f1_is_zero_when_a_feature_separates_perfectly(): + X = np.array([[0.0], [0.0], [1.0], [1.0]]) + y = np.array([0, 0, 1, 1]) + # No within-class spread and different means: an infinite ratio, so F1 = 0. + assert fisher_discriminant_ratio(X, y) == 0.0 + + +def test_f1_finds_the_single_informative_feature_among_noise(): + rng = np.random.default_rng(1) + informative = np.r_[np.zeros(40), np.ones(40) * 8] + X = np.column_stack([informative, rng.normal(0.0, 1.0, (80, 5))]) + y = np.array([0] * 40 + [1] * 40) + assert fisher_discriminant_ratio(X, y) < 0.01 + + +def test_f1_is_one_when_the_classes_share_a_mean_and_spread(): + X = np.array([[0.0], [1.0], [0.0], [1.0]]) + y = np.array([0, 0, 1, 1]) + # Identical means cancel the numerator, so the ratio is 0 and F1 is 1. + assert fisher_discriminant_ratio(X, y) == pytest.approx(1.0) + + +# --- N1 --- + + +def test_n1_is_small_for_well_separated_blobs(): + # Only the pair of points bridging the two blobs can be borderline. + assert borderline_points(*_separable()) <= 2 / 80 + + +def test_n1_flags_every_point_when_classes_alternate(): + # Points alternate along a line, so every tree edge joins two classes. + X = np.arange(10, dtype=float).reshape(-1, 1) + y = np.array([0, 1] * 5) + assert borderline_points(X, y) == 1.0 + + +def test_n1_handles_coincident_points(): + """Duplicate rows give a zero distance, which must not break the tree.""" + X = np.array([[0.0], [0.0], [0.0], [5.0], [5.0], [5.0]]) + y = np.array([0, 0, 0, 1, 1, 1]) + assert 0.0 <= borderline_points(X, y) <= 1.0 + + +# --- N2 --- + + +def test_n2_is_small_for_well_separated_blobs(): + assert intra_extra_nn_ratio(*_separable()) < 0.05 + + +def test_n2_skips_classes_with_a_single_member(): + X = np.array([[0.0], [0.1], [0.2], [9.0]]) + y = np.array([0, 0, 0, 1]) + # The lone member of class 1 has no same-class neighbour; the rest do. + assert 0.0 <= intra_extra_nn_ratio(X, y) <= 1.0 + + +def test_n2_raises_when_no_point_has_a_same_class_neighbour(): + X = np.array([[0.0], [1.0]]) + y = np.array([0, 1]) + with pytest.raises(ValueError, match="same-class and a different-class"): + intra_extra_nn_ratio(X, y) + + +# --- validation --- + + +@pytest.mark.parametrize( + "measure", + [fisher_discriminant_ratio, borderline_points, intra_extra_nn_ratio], +) +def test_measures_reject_a_single_class(measure): + with pytest.raises(ValueError, match="At least two classes"): + measure(np.zeros((6, 2)), np.zeros(6)) + + +@pytest.mark.parametrize( + "measure", + [fisher_discriminant_ratio, borderline_points, intra_extra_nn_ratio], +) +def test_measures_reject_mismatched_lengths(measure): + with pytest.raises(ValueError, match="same number of rows"): + measure(np.zeros((6, 2)), np.zeros(5)) + + +def test_measures_reject_an_empty_matrix(): + with pytest.raises(ValueError, match="empty"): + fisher_discriminant_ratio(np.zeros((0, 2)), np.zeros(0)) + + +# --- compute_class_overlap --- + + +def test_compute_class_overlap_reports_every_measure(): + X, y = _separable() + summary = compute_class_overlap(X, y, random_state=0) + assert set(summary["measures"]) == set(MEASURE_NAMES) + assert summary["n_samples"] == 80 + assert summary["n_features"] == 2 + assert summary["n_classes"] == 2 + assert summary["n_dropped_rows"] == 0 + + +def test_compute_class_overlap_drops_rows_with_missing_values(): + X, y = _separable() + X = X.astype(float) + X[0, 0] = np.nan + X[7, 1] = np.nan + summary = compute_class_overlap(X, y, random_state=0) + assert summary["n_dropped_rows"] == 2 + assert summary["n_samples"] == 78 + + +def test_compute_class_overlap_subsamples_above_the_budget(): + rng = np.random.default_rng(2) + X = rng.normal(0.0, 1.0, (300, 2)) + y = rng.integers(0, 2, 300) + summary = compute_class_overlap(X, y, max_samples=60, random_state=0) + assert summary["n_samples"] == 300 + # F1 keeps the full sample; only the distance-based measures are reduced. + assert summary["n_samples_used_for_distances"] <= 66 + + +def test_compute_class_overlap_is_reproducible_with_a_seed(): + rng = np.random.default_rng(3) + X = rng.normal(0.0, 1.0, (200, 2)) + y = rng.integers(0, 2, 200) + first = compute_class_overlap(X, y, max_samples=50, random_state=7) + second = compute_class_overlap(X, y, max_samples=50, random_state=7) + assert first["measures"] == second["measures"] + + +def test_compute_class_overlap_is_scale_invariant_for_distance_measures(): + """Min-max scaling means a unit change must not move N1 or N2.""" + X, y = _separable() + plain = compute_class_overlap(X, y, random_state=0)["measures"] + stretched = compute_class_overlap(X * 1000.0, y, random_state=0)["measures"] + assert plain["N1"] == pytest.approx(stretched["N1"]) + assert plain["N2"] == pytest.approx(stretched["N2"]) + + +def test_compute_class_overlap_supports_more_than_two_classes(): + rng = np.random.default_rng(4) + X = np.vstack( + [ + rng.normal(0.0, 0.3, (30, 2)), + rng.normal(6.0, 0.3, (30, 2)), + rng.normal(12.0, 0.3, (30, 2)), + ] + ) + y = np.array([0] * 30 + [1] * 30 + [2] * 30) + summary = compute_class_overlap(X, y, random_state=0) + assert summary["n_classes"] == 3 + assert all(value < 0.1 for value in summary["measures"].values()) + + +def test_compute_class_overlap_accepts_string_labels(): + X, y = _separable() + labels = np.where(y == 0, "cat", "dog") + summary = compute_class_overlap(X, labels, random_state=0) + assert summary["n_classes"] == 2